diff --git a/.gitignore b/.gitignore index 3341cf2853..6157c1787e 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,25 @@ test/config.js # Doxygen generated documentation output HtmlDocumentation + +# Xcode user-specific project settings +# Xcode +.DS_Store +*/build/* +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 +xcuserdata +profile +*.moved-aside +DerivedData +.idea/ +*.hmap + +# Intel Parallel Studio 2013 XE +My Amplifier XE Results - RippleD diff --git a/BeastConfig.h b/BeastConfig.h index 2d08bcb6e6..4c35072c81 100644 --- a/BeastConfig.h +++ b/BeastConfig.h @@ -1,25 +1,123 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco -#ifndef BEAST_BEASTCONFIG_HEADER -#define BEAST_BEASTCONFIG_HEADER + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. -// beast_core flags: + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== -#ifndef BEAST_FORCE_DEBUG - //#define BEAST_FORCE_DEBUG +#ifndef BEAST_BEASTCONFIG_H_INCLUDED +#define BEAST_BEASTCONFIG_H_INCLUDED + +/** Configuration file for Beast. + + This sets various configurable options for Beast. In order to compile you + must place a copy of this file in a location where your build environment + can find it, and then customize its contents to suit your needs. + + @file BeastConfig.h +*/ + +//------------------------------------------------------------------------------ + +/** Config: BEAST_FORCE_DEBUG + + Normally, BEAST_DEBUG is set to 1 or 0 based on compiler and project + settings, but if you define this value, you can override this to force it + to be true or false. +*/ +#ifndef BEAST_FORCE_DEBUG +//#define BEAST_FORCE_DEBUG 0 #endif -#ifndef BEAST_LOG_ASSERTIONS - //#define BEAST_LOG_ASSERTIONS 1 +//------------------------------------------------------------------------------ + +/** Config: BEAST_LOG_ASSERTIONS + + If this flag is enabled, the the bassert and bassertfalse macros will always + use Logger::writeToLog() to write a message when an assertion happens. + + Enabling it will also leave this turned on in release builds. When it's + disabled, however, the bassert and bassertfalse macros will not be compiled + in a release build. + + @see bassert, bassertfalse, Logger +*/ +#ifndef BEAST_LOG_ASSERTIONS +//#define BEAST_LOG_ASSERTIONS 0 #endif -#ifndef BEAST_CHECK_MEMORY_LEAKS - //#define BEAST_CHECK_MEMORY_LEAKS +//------------------------------------------------------------------------------ + +/** Config: BEAST_CHECK_MEMORY_LEAKS + + Enables a memory-leak check for certain objects when the app terminates. + See the LeakChecked class for more details about enabling leak checking for + specific classes. +*/ +#ifndef BEAST_CHECK_MEMORY_LEAKS +//#define BEAST_CHECK_MEMORY_LEAKS 1 #endif -#ifndef BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES - //#define BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES +//------------------------------------------------------------------------------ + +/** Config: BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES + + In a Visual C++ build, this can be used to stop the required system libs + being automatically added to the link stage. +*/ +#ifndef BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES +//#define BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES 1 #endif -#define RIPPLE_USE_NAMESPACE 0 +//------------------------------------------------------------------------------ + +/** Config: BEAST_INCLUDE_ZLIB_CODE + + This can be used to disable Beast's embedded 3rd-party zlib code. + You might need to tweak this if you're linking to an external zlib library in your app, + but for normal apps, this option should be left alone. + + If you disable this, you might also want to set a value for BEAST_ZLIB_INCLUDE_PATH, to + specify the path where your zlib headers live. +*/ +#ifndef BEAST_INCLUDE_ZLIB_CODE +//#define BEAST_INCLUDE_ZLIB_CODE 0 +#endif + +#ifndef BEAST_ZLIB_INCLUDE_PATH +#define BEAST_ZLIB_INCLUDE_PATH +#endif + +//------------------------------------------------------------------------------ + +/** Config: BEAST_BOOST_IS_AVAILABLE + + This activates boost specific features and improvements. +*/ +#ifndef BEAST_BOOST_IS_AVAILABLE +#define BEAST_BOOST_IS_AVAILABLE 1 +#endif + +/** Bind source configuration. + + Set one of these to manually force a particular implementation of bind(). + If nothing is chosen then beast will use whatever is appropriate for your + environment based on what is available. +*/ +//#define BEAST_BIND_USES_STD 1 +//#define BEAST_BIND_USES_TR1 1 +//#define BEAST_BIND_USES_BOOST 1 #endif diff --git a/Builds/QtCreator/.gitignore b/Builds/QtCreator/.gitignore new file mode 100644 index 0000000000..20e0d48afd --- /dev/null +++ b/Builds/QtCreator/.gitignore @@ -0,0 +1,5 @@ +# QTCreator + +Makefile +*.user + diff --git a/Builds/QtCreator/rippled.pro b/Builds/QtCreator/rippled.pro new file mode 100644 index 0000000000..946376b4a8 --- /dev/null +++ b/Builds/QtCreator/rippled.pro @@ -0,0 +1,96 @@ + +# Ripple protocol buffers + +PROTOS = ../../src/cpp/ripple/ripple.proto +PROTOS_DIR = ../../build/proto + +# Google Protocol Buffers support + +protobuf_h.name = protobuf header +protobuf_h.input = PROTOS +protobuf_h.output = $${PROTOS_DIR}/${QMAKE_FILE_BASE}.pb.h +protobuf_h.depends = ${QMAKE_FILE_NAME} +protobuf_h.commands = protoc --cpp_out=$${PROTOS_DIR} --proto_path=${QMAKE_FILE_PATH} ${QMAKE_FILE_NAME} +protobuf_h.variable_out = HEADERS +QMAKE_EXTRA_COMPILERS += protobuf_h + +protobuf_cc.name = protobuf implementation +protobuf_cc.input = PROTOS +protobuf_cc.output = $${PROTOS_DIR}/${QMAKE_FILE_BASE}.pb.cc +protobuf_cc.depends = $${PROTOS_DIR}/${QMAKE_FILE_BASE}.pb.h +protobuf_cc.commands = $$escape_expand(\\n) +#protobuf_cc.variable_out = SOURCES +QMAKE_EXTRA_COMPILERS += protobuf_cc + +# Ripple compilation + +DESTDIR = ../../build/QtCreator +OBJECTS_DIR = ../../build/QtCreator/obj + +TEMPLATE = app +CONFIG += console thread warn_off +CONFIG -= qt gui + +linux-gg++:QMAKE_CXXFLAGS += \ + -Wall \ + -Wno-sign-compare \ + -Wno-char-subscripts \ + -Wno-invalid-offsetof \ + -Wno-unused-parameter \ + -Wformat \ + -O0 \ + -pthread + +INCLUDEPATH += \ + "../.." \ + "../../Subtrees" \ + "../../Subtrees/leveldb/" \ + "../../Subtrees/leveldb/port" \ + "../../Subtrees/leveldb/include" \ + $${PROTOS_DIR} + +OTHER_FILES += \ + $$files(../../Subtrees/beast/*) \ + $$files(../../Subtrees/beast/modules/beast_basics/diagnostic/*) + +# $$files(../../Subtrees/beast/modules/beast_core/, true) +# $$files(../../modules/*, true) \ +# $$files(../../src/cpp/ripple/*, true) \ + +UI_HEADERS_DIR += ../../modules/ripple_basics + +SOURCES += \ + ../../Subtrees/beast/modules/beast_basics/beast_basics.cpp \ + ../../Subtrees/beast/modules/beast_core/beast_core.cpp \ + ../../modules/ripple_app/ripple_app_pt1.cpp \ + ../../modules/ripple_app/ripple_app_pt2.cpp \ + ../../modules/ripple_app/ripple_app_pt3.cpp \ + ../../modules/ripple_app/ripple_app_pt4.cpp \ + ../../modules/ripple_app/ripple_app_pt5.cpp \ + ../../modules/ripple_app/ripple_app_pt6.cpp \ + ../../modules/ripple_app/ripple_app_pt7.cpp \ + ../../modules/ripple_app/ripple_app_pt8.cpp \ + ../../modules/ripple_basics/ripple_basics.cpp \ + ../../modules/ripple_basio/ripple_basio.cpp \ + ../../modules/ripple_core/ripple_core.cpp \ + ../../modules/ripple_client/ripple_client.cpp \ + ../../modules/ripple_data/ripple_data.cpp \ + ../../modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp \ + ../../modules/ripple_json/ripple_json.cpp \ + ../../modules/ripple_leveldb/ripple_leveldb.cpp \ + ../../modules/ripple_mdb/ripple_mdb.c \ + ../../modules/ripple_net/ripple_net.cpp \ + ../../modules/ripple_sqlite/ripple_sqlite.c \ + ../../modules/ripple_websocket/ripple_websocket.cpp + +LIBS += \ + -lboost_date_time-mt\ + -lboost_filesystem-mt \ + -lboost_program_options-mt \ + -lboost_regex-mt \ + -lboost_system-mt \ + -lboost_thread-mt \ + -lboost_random-mt \ + -lprotobuf \ + -lssl \ + -lrt diff --git a/Builds/VisualStudio2012/Ripple.sln b/Builds/VisualStudio2012/Ripple.sln index 5995866ca7..0d7f6c5a08 100644 --- a/Builds/VisualStudio2012/Ripple.sln +++ b/Builds/VisualStudio2012/Ripple.sln @@ -1,10 +1,10 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Express 2012 for Windows Desktop -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "RippleD", "RippleD.vcxproj", "{19465545-42EE-42FA-9CC8-F8975F8F1CC7}" -EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "beast", "..\..\Subtrees\beast\Builds\VisualStudio2012\beast.vcxproj", "{73C5A0F0-7629-4DE7-9194-BE7AC6C19535}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "RippleD", "RippleD.vcxproj", "{B7F39ECD-473C-484D-BC34-31F8362506A5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -13,20 +13,22 @@ Global Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {19465545-42EE-42FA-9CC8-F8975F8F1CC7}.Debug|Win32.ActiveCfg = Debug|Win32 - {19465545-42EE-42FA-9CC8-F8975F8F1CC7}.Debug|Win32.Build.0 = Debug|Win32 - {19465545-42EE-42FA-9CC8-F8975F8F1CC7}.Debug|x64.ActiveCfg = Debug|x64 - {19465545-42EE-42FA-9CC8-F8975F8F1CC7}.Debug|x64.Build.0 = Debug|x64 - {19465545-42EE-42FA-9CC8-F8975F8F1CC7}.Release|Win32.ActiveCfg = Release|Win32 - {19465545-42EE-42FA-9CC8-F8975F8F1CC7}.Release|Win32.Build.0 = Release|Win32 - {19465545-42EE-42FA-9CC8-F8975F8F1CC7}.Release|x64.ActiveCfg = Release|x64 - {19465545-42EE-42FA-9CC8-F8975F8F1CC7}.Release|x64.Build.0 = Release|x64 {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Debug|Win32.ActiveCfg = Debug|Win32 {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Debug|Win32.Build.0 = Debug|Win32 - {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Debug|x64.ActiveCfg = Debug|Win32 + {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Debug|x64.ActiveCfg = Debug|x64 + {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Debug|x64.Build.0 = Debug|x64 {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Release|Win32.ActiveCfg = Release|Win32 {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Release|Win32.Build.0 = Release|Win32 - {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Release|x64.ActiveCfg = Release|Win32 + {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Release|x64.ActiveCfg = Release|x64 + {73C5A0F0-7629-4DE7-9194-BE7AC6C19535}.Release|x64.Build.0 = Release|x64 + {B7F39ECD-473C-484D-BC34-31F8362506A5}.Debug|Win32.ActiveCfg = Debug|Win32 + {B7F39ECD-473C-484D-BC34-31F8362506A5}.Debug|Win32.Build.0 = Debug|Win32 + {B7F39ECD-473C-484D-BC34-31F8362506A5}.Debug|x64.ActiveCfg = Debug|x64 + {B7F39ECD-473C-484D-BC34-31F8362506A5}.Debug|x64.Build.0 = Debug|x64 + {B7F39ECD-473C-484D-BC34-31F8362506A5}.Release|Win32.ActiveCfg = Release|Win32 + {B7F39ECD-473C-484D-BC34-31F8362506A5}.Release|Win32.Build.0 = Release|Win32 + {B7F39ECD-473C-484D-BC34-31F8362506A5}.Release|x64.ActiveCfg = Release|x64 + {B7F39ECD-473C-484D-BC34-31F8362506A5}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Builds/VisualStudio2012/RippleD.props b/Builds/VisualStudio2012/RippleD.props index 04758bdfe3..2f398635ce 100644 --- a/Builds/VisualStudio2012/RippleD.props +++ b/Builds/VisualStudio2012/RippleD.props @@ -12,10 +12,10 @@ - BOOST_TEST_ALTERNATIVE_INIT_API;BOOST_TEST_NO_MAIN;_WIN32_WINNT=0x0600;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;WIN32;%(PreprocessorDefinitions) + _VARIADIC_MAX=10;_WIN32_WINNT=0x0600;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;WIN32;%(PreprocessorDefinitions) true Level3 - $(RepoDir);$(RepoDir)\src\cpp\protobuf\src;$(RepoDir)\src\cpp\protobuf\vsprojects;$(RepoDir)\build\proto;$(RepoDir)\Subtrees;$(RepoDir)\Subtrees\leveldb;$(RepoDir)\Subtrees\leveldb\include;$(RepoDir)\Subtrees\beast;%(AdditionalIncludeDirectories) + $(RepoDir);$(RepoDir)\src\cpp\protobuf\src;$(RepoDir)\src\cpp\protobuf\vsprojects;$(RepoDir)\build\proto;$(RepoDir)\Subtrees;$(RepoDir)\Subtrees\leveldb;$(RepoDir)\Subtrees\leveldb\include;%(AdditionalIncludeDirectories) /bigobj %(AdditionalOptions) Async diff --git a/Builds/VisualStudio2012/RippleD.vcxproj b/Builds/VisualStudio2012/RippleD.vcxproj index ba7f912ff1..ba9ba1223a 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj +++ b/Builds/VisualStudio2012/RippleD.vcxproj @@ -18,167 +18,174 @@ x64 - - {19465545-42EE-42FA-9CC8-F8975F8F1CC7} - Win32Proj - newcoin - - - - Application - true - MultiByte - v110 - - - Application - true - MultiByte - v110 - - - Application - false - true - MultiByte - v110 - - - Application - false - true - MultiByte - v110 - - - - - - - - - - - - - - - - - - - - - - - true - - - true - - - false - - - false - - - - NotUsing - Disabled - _DEBUG;_CONSOLE;%(PreprocessorDefinitions) - ProgramDatabase - false - MultiThreadedDebug - - - Console - true - %(AdditionalLibraryDirectories) - ssleay32MTd.lib;libeay32MTd.lib;%(AdditionalDependencies) - - - - - - - - - - - - - NotUsing - Disabled - _DEBUG;_CONSOLE;%(PreprocessorDefinitions) - ProgramDatabase - false - MultiThreadedDebug - - - Console - true - %(AdditionalLibraryDirectories) - ssleay32MTd.lib;libeay32MTd.lib;%(AdditionalDependencies) - - - - - - - - - - - - - NotUsing - MaxSpeed - true - true - NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - MultiThreaded - - - Console - true - true - true - %(AdditionalLibraryDirectories) - ssleay32MT.lib;libeay32MT.lib;%(AdditionalDependencies) - - - - - NotUsing - MaxSpeed - true - true - NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - MultiThreaded - - - Console - true - true - true - %(AdditionalLibraryDirectories) - ssleay32MT.lib;libeay32MT.lib;%(AdditionalDependencies) - - + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + + + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + true true true true + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true true @@ -209,6 +216,12 @@ true true + + true + true + true + true + true true @@ -216,13 +229,199 @@ true - - - Level4 - Level4 - Level4 - Level4 + + true + true + true + true + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true true @@ -236,85 +435,668 @@ true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + true true true true + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + true true true true + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true true + true + true true @@ -346,12 +1128,6 @@ true true - - true - true - true - true - true true @@ -382,12 +1158,6 @@ true true - - true - true - true - true - true true @@ -490,6 +1260,12 @@ true true + + true + true + true + true + true true @@ -568,958 +1344,36 @@ true true - - true - true - true - true - - - true - true - true - true - - - - - - - true - true - true - true - - - true - true - true - true - - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - - - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - true - true - true - true - - - - + + + + + + + + - + + + + - + @@ -1528,12 +1382,18 @@ + + + + + + @@ -1543,8 +1403,9 @@ - + + @@ -1555,11 +1416,10 @@ - - + @@ -1570,237 +1430,82 @@ + + + + + + - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + - + + + + + + + + @@ -1808,16 +1513,9 @@ - - - - - - - @@ -1825,12 +1523,10 @@ - - - + @@ -1848,6 +1544,7 @@ + @@ -1868,6 +1565,8 @@ + + @@ -1900,6 +1599,19 @@ + + + Document + protoc --cpp_out=$(RepoDir)\build\proto -I=$(SrcDir) "$(SrcDir)\%(Filename)%(Extension)" + $(RepoDir)\build\proto\%(Filename).pb.h;$(RepoDir)\build\proto\%(Filename).pb.cc + protoc --cpp_out=$(RepoDir)\build\proto -I=$(SrcDir) "$(SrcDir)\%(Filename)%(Extension)" + $(RepoDir)\build\proto\%(Filename).pb.h;$(RepoDir)\build\proto\%(Filename).pb.cc + protoc --cpp_out=$(RepoDir)\build\proto -I=$(SrcDir) "$(SrcDir)\%(Filename)%(Extension)" + $(RepoDir)\build\proto\%(Filename).pb.h;$(RepoDir)\build\proto\%(Filename).pb.cc + protoc --cpp_out=$(RepoDir)\build\proto -I=$(SrcDir) "$(SrcDir)\%(Filename)%(Extension)" + $(RepoDir)\build\proto\%(Filename).pb.h;$(RepoDir)\build\proto\%(Filename).pb.cc + + @@ -1907,34 +1619,160 @@ - - Document - protoc --cpp_out=$(RepoDir)\build\proto -I=$(SrcDir) "$(SrcDir)\%(Filename)%(Extension)" - protoc --cpp_out=$(SrcDir) -I=$(SrcDir) "$(SrcDir)\%(Filename)%(Extension)" - $(RepoDir)\build\proto\%(Filename).pb.h;$(RepoDir)\build\proto\%(Filename).pb.cc - $(SrcDir)\%(Filename).pb.h;$(SrcDir)\%(Filename).pb.cc - protoc --cpp_out=$(RepoDir)\build\proto -I=$(SrcDir) "$(SrcDir)\%(Filename)%(Extension)" - protoc --cpp_out=$(SrcDir) -I=$(SrcDir) "$(SrcDir)\%(Filename)%(Extension)" - $(RepoDir)\build\proto\%(Filename).pb.h;$(RepoDir)\build\proto\%(Filename).pb.cc - $(SrcDir)\%(Filename).pb.h;$(SrcDir)\%(Filename).pb.cc - - - - true - true - - - - - - - - + + + {B7F39ECD-473C-484D-BC34-31F8362506A5} + Win32Proj + RippleD + + + + Application + true + v110 + MultiByte + + + Application + true + Intel C++ Compiler XE 13.0 + MultiByte + + + Application + false + v110 + true + MultiByte + + + Application + false + Intel C++ Compiler XE 13.0 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + true + + + true + + + false + + + false + + + + + + Disabled + _CRTDBG_MAP_ALLOC;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ProgramDatabase + false + MultiThreadedDebug + false + true + false + + + Console + true + ssleay32MT.lib;libeay32MT.lib;%(AdditionalDependencies) + + + + + + + Disabled + _CRTDBG_MAP_ALLOC;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + MultiThreadedDebug + false + true + false + + + Console + true + ssleay32MT.lib;libeay32MT.lib;%(AdditionalDependencies) + + + + + + + MaxSpeed + false + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + MultiThreaded + true + false + Speed + false + + + Console + true + true + true + ssleay32MT.lib;libeay32MT.lib;%(AdditionalDependencies) + Default + + + + + + + MaxSpeed + false + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + MultiThreaded + true + false + Speed + false + false + + + Console + true + true + true + ssleay32MT.lib;libeay32MT.lib;%(AdditionalDependencies) + Default + + diff --git a/Builds/VisualStudio2012/RippleD.vcxproj.filters b/Builds/VisualStudio2012/RippleD.vcxproj.filters index c5f063b9ed..1487bad876 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2012/RippleD.vcxproj.filters @@ -1,1765 +1,1657 @@  - - {475c5b33-c9b5-415f-89df-fb9961f3b57c} + + {ddf42dfe-eaad-4275-8cb7-5c02be627366} - - {290b7b39-a4e6-4b8b-b464-d1e458562fdd} + + {9766e229-8795-43d8-871e-60de90501333} - - {83e0c1a9-5ac4-4c68-ba62-45e24bf60c85} + + {bcc68889-41a4-4090-ac3f-12a3ba1d8961} - - {1e155698-fbdf-4b1a-861f-396ca1690486} + + {f669c93c-60da-4674-a693-9eebcddacb6f} - - {90f03f10-67b3-405b-9395-ca514b3d2554} + + {3debb18d-0f76-4208-accb-f96cb9ce6dab} - - {08051cd0-8c55-4aea-a839-fab10847ae8e} + + {1f093002-9955-4915-a17b-398d55c9fc76} - - {2c99e10c-fd93-4859-ad41-b4878eb7e149} + + {086839ba-d691-4ca8-b2ef-40aee8842957} - - {5aa322df-2bf4-4409-a39a-cb0069eb5e87} + + {ca8e7c6c-0e6d-4915-b7b4-0a0b21f91d16} - - {5249322a-6711-40e8-b1fb-12873aa39495} + + {e0dd916d-72b2-46cd-840a-fb5e924a470a} - - {50c3843c-4a67-4cc2-a1d9-c45692849b48} + + {0d065d55-0a7b-480b-beee-67dad6ee6684} - - {f57e4d02-7a69-4861-9f4b-160f13695e54} + + {df861e00-baa6-43d4-bbbf-df61d66ee414} - - {27858e65-9580-4a67-b0c5-054e970516af} + + {487f6b35-d0a3-4b34-85c1-94e2aad4c9ff} - - {a152ed99-bd05-4c37-8ae7-82888b1bb469} + + {e119423b-04d2-4ce8-9794-1c2abd6a9cb5} - - {84e43f43-816e-4ccd-80c2-38b322904894} + + {6ad1f6a2-1710-43ac-96d4-f6b54fd8379e} - - {96cbc9ff-0118-4844-bb4c-05aef58a60b5} + + {f238bebe-dfcc-48f3-9d89-9c5c7c72ebed} - - {9f33883e-d53d-469f-ab51-653ac3e8780b} + + {2bf8c9ef-13f7-477a-8a78-2a52a26bb4f8} - - {670d727d-6c99-4893-b0bb-e620975e8b8d} + + {1b631697-2450-473a-9eb6-cc6cfaf66668} - - {e398b116-a7fa-4850-8b8e-6de95efdea4e} + + {42324d78-07a6-4782-b938-3f620904addf} - - {c30623e7-e0ab-4a8e-a213-aeaceb4327e2} + + {c817e7d7-36d7-410e-ad16-f51c5eb024af} - - {0d38ac4f-f094-4b17-9f4c-ac4011ea3bca} + + {c8fec1f5-496a-4a1e-bc8e-8096163c7766} - - {8d0241d2-071c-4d6c-a15e-980cc51b26ce} + + {548037f2-eb8a-41bd-95dc-05f58cdbc041} - - {095d33d8-dbf7-44d0-a675-6722b02bb3be} + + {514a3062-9305-4285-8007-3ebb955eaab3} - - {2f3572a9-2882-4656-ab93-82b7761c9e3d} + + {3af7c606-c5b5-4d1f-951b-d5d99997818f} - - {ba524810-8446-4c50-ad59-d284747ba220} + + {a837f3ce-75b7-4e6c-b8b1-728b6a1216bd} - - {82b7f371-8bc6-474b-b10c-18c004467b46} + + {9d43b540-fdec-484d-a8bd-0dc65d3b0c55} - - {786657f9-b1b3-483c-a8e6-863cc2e02eb8} + + {99ac4d07-04a7-4ce3-96c7-b8ea578f1a61} - - {97c96b68-70fd-4679-89fc-c1c8c87c265e} + + {6967f835-a1b0-47e3-9bd9-6fc8bd0f3df7} - - {1a379c07-ccf1-4636-8018-2cfc0685edf0} + + {fe2722a9-6550-4098-91eb-1575f7419a42} - - {86d0831e-74e5-443b-b5f9-d6b39d5b4c97} + + {e1f02660-3d99-4ef3-b79a-2a8da16e2ec7} - - {c7f863ce-204b-4a15-b236-24fe1e3430e5} + + {5280f25b-a657-45f7-a0e6-822891d0f20f} - - {3f351c55-360d-40bc-a136-4944ce572efd} + + {8710b403-2cfa-4991-b8b4-8214d1947528} - - {1d36a65a-de7c-41f9-85dd-93b29d67606e} + + {223ac5ce-c9a0-4196-9b75-4f5fbe8bfa00} - - {c441b367-d096-401f-a3f0-5ac24290d7b7} + + {46bda3c0-166b-4188-8f6b-da6f6ff638fc} - - {78b0aa30-f238-4e41-821f-ce84dde82ceb} + + {01ac5649-14bd-4c61-ac6b-169742a86036} - - {6097a179-ddad-4c69-9a34-2e3fc2c9fa1d} + + {85ea08c2-f8a6-477b-8d59-9a7339761d20} - - {bfd76b72-9979-469b-8fd1-834e0622728a} + + {cf19fa31-c40b-4203-b497-63a8c3dcb282} - - {5b22fcf9-c9f0-4260-9a17-7b448e30bb57} + + {7c14e9df-6d8d-4ba3-b208-f89c1d0b6b30} - - {babbb25a-e143-4140-bc6f-16b659d8098b} + + {66d2d40a-72bd-41f5-9cf9-1f464a40bb04} - - {8b88138b-e5bd-492b-bd34-012b9f43e544} + + {ec6b9e0e-4a92-4c06-833d-441d51b0b163} - - {90a5527e-0de0-4d5f-a731-b6b196a013b5} + + {d46790be-df64-4e31-b51c-ac9e95a24747} - - {857187f0-ff25-4ebe-af30-544b7b7f503a} + + {7b3efe80-291a-49bc-b32e-085b5af47f98} - - {90f3b8c4-01a1-41f1-969b-39ec650c00cd} + + {29393027-f1ed-4700-bbd1-c880091ab96b} - - {9d37d3a1-e8ac-49c5-845b-702a191f28e0} + + {e54bae0e-09f1-483e-bcf7-fb6b17b8d561} - - {9cd9dfb0-daa3-474e-b1c7-b048f5b85e71} + + {c6a852db-1174-4bf6-a726-ebaabf595743} - - {6bbe236f-79ff-4e30-928c-bf5f302b3a21} + + {c977169f-d68b-4a59-aa0b-f7d49157d9ac} - - {82d79c26-4932-4a48-b134-09969f45d75a} + + {c84fc3af-f487-4eba-af78-d4be009f76d1} - - {ce56fa36-0012-44ab-a6dd-da0391f32ba3} + + {febf2e7e-f071-4a6c-9b81-68498fc8ea57} - - {f7a586fa-b21a-4c7b-b87e-5ac62f2758e4} + + {82a6d3f3-fa01-4cc6-9730-f928e61b9929} + + + {bd48a825-192f-4d5e-9230-661fdd65f9b6} + + + {6a769530-8edf-4836-afc8-8836fe315603} + + + {571acd5b-065c-4202-8de3-8692735171dc} - - 0. Third Party Code\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\contracts - - - 1. Modules\ripple_app\_unfactored\contracts - - - 1. Modules\ripple_basics - - - 2. %28Unused%29\ripple_client - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\contracts - - - 1. Modules\ripple_app\_unfactored\contracts - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json - - - 1. Modules\ripple_basics\containers - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_data - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_basics\types - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_data\utility - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_basics\containers - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_app\refactored\transactions - - - 1. Modules\ripple_app\refactored\consensus - - - 1. Modules\ripple_app\refactored\consensus - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored\ledger - - - 1. Modules\ripple_app\refactored\ledger - - - 1. Modules\ripple_app\refactored\ledger - - - 1. Modules\ripple_app\refactored\ledger - - - 1. Modules\ripple_app - - - 1. Modules\ripple_app - - - 1. Modules\ripple_app - - - 1. Modules\ripple_app - - - 1. Modules\ripple_app - - - 1. Modules\ripple_app\_unfactored - - - 1. Modules\ripple_app\_unfactored - - - 1. Modules\ripple_basics\utility + + [0] Subtrees\sqlite - 0. Third Party Code\beast + [0] Subtrees\beast - 0. Third Party Code\beast + [0] Subtrees\beast - - 0. Third Party Code\LevelDB\db + + [0] Subtrees\protobuf - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_app - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_app - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_app - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_app - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_app - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_basics\containers - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_basics\containers - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_basics\containers - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_basics\types - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_basics\utility - - 0. Third Party Code\LevelDB\db + + [1] Ripple\ripple_basics\utility - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\port - - - 0. Third Party Code\LevelDB\port - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 1. Modules\ripple_websocket - - - 1. Modules\ripple_websocket\autosocket - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_app\_unfactored\db - - - 1. Modules\ripple_app\_unfactored\db - - - 1. Modules\ripple_app\_unfactored\db - - - 1. Modules\ripple_sqlite - - - 1. Modules\ripple_leveldb - - - 1. Modules\ripple_core - - - 1. Modules\ripple_websocket\autosocket - - - 1. Modules\ripple_core\functional - - - 1. Modules\ripple_core\functional - - - 1. Modules\ripple_core\functional - - - 1. Modules\ripple_core\functional + + [1] Ripple\ripple_basics\utility - 1. Modules\ripple_basics\utility + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics - 1. Modules\ripple_core\functional + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional - 1. Modules\ripple_core\functional + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json + + + [1] Ripple\ripple_leveldb + + + [1] Ripple\ripple_sqlite + + + [1] Ripple\ripple_websocket\autosocket + + + [1] Ripple\ripple_websocket\autosocket + + + [1] Ripple\ripple_websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\port + + + [0] Subtrees\leveldb\port + + + [1] Ripple\ripple_client + + + [1] Ripple\ripple_app\_contracts + + + [1] Ripple\ripple_app\_contracts + + + [1] Ripple\ripple_app\_contracts + + + [1] Ripple\ripple_app\_contracts + + + [1] Ripple\ripple_app\_misc\_consensus + + + [1] Ripple\ripple_app\_misc\_consensus + + + [1] Ripple\ripple_app\_data + + + [1] Ripple\ripple_app\_data + + + [1] Ripple\ripple_app\_data + + + [1] Ripple\ripple_app\_data + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_main + + + [1] Ripple\ripple_app\_main - 1. Modules\ripple_app\refactored + [1] Ripple\ripple_app\_main + + + [1] Ripple\ripple_app\_main + + + [1] Ripple\ripple_app\_main + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_network + + + [1] Ripple\ripple_app\_network + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions - 1. Modules\ripple_app + [1] Ripple\ripple_app - 1. Modules\ripple_app + [1] Ripple\ripple_app - 1. Modules\ripple_app + [1] Ripple\ripple_app - 1. Modules\ripple_app + [1] Ripple\ripple_app + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_basio + + + [1] Ripple\ripple_basio\boost + + + [1] Ripple\ripple_net + + + [1] Ripple\ripple_net\basics + + + [1] Ripple\ripple_net\basics + + + [1] Ripple\ripple_app\basics + + + [1] Ripple\ripple_net\basics + + + [1] Ripple\ripple_net\basics + + + [1] Ripple\ripple_app\_network + + + [1] Ripple\ripple_basio\boost + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_mdb + + + [1] Ripple\ripple_hyperleveldb - - 0. Third Party Code\protobuf\protobuf + + [0] Subtrees\sqlite - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\protobuf - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\io - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 0. Third Party Code\protobuf\stubs - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\contracts - - - 1. Modules\ripple_app\_unfactored\contracts - - - 1. Modules\ripple_basics - - - 2. %28Unused%29\ripple_client - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_app\_unfactored\contracts - - - 1. Modules\ripple_app\_unfactored\contracts - - - 1. Modules\ripple_app\_unfactored\transactions - - - 1. Modules\ripple_basics\containers - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json\DEPRECATED - - - 1. Modules\ripple_json - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_basics\containers - - - 1. Modules\ripple_basics\containers - - - 1. Modules\ripple_basics\containers - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_basics\types - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_data - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_data\crypto - - - 1. Modules\ripple_basics\utility - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\_unfactored\ledger - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\network - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_app\_unfactored\rpc - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_basics\types - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_basics\types - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_data\utility - - - 1. Modules\ripple_data\utility - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored\shamap - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_data\utility - - - 1. Modules\ripple_app\refactored\transactions - - - 1. Modules\ripple_app\refactored\consensus - - - 1. Modules\ripple_app\refactored\consensus - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_app\refactored\pathing - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_data\protocol - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored - - - 1. Modules\ripple_app\refactored\ledger - - - 1. Modules\ripple_app\refactored\ledger - - - 1. Modules\ripple_app\refactored\ledger - - - 1. Modules\ripple_app\refactored\ledger + + [0] Subtrees\sqlite - 1. Modules\ripple_app + [1] Ripple\ripple_app - - 1. Modules\ripple_app\_unfactored + + [1] Ripple\ripple_basics\containers + + + [1] Ripple\ripple_basics\containers + + + [1] Ripple\ripple_basics\containers + + + [1] Ripple\ripple_basics\containers + + + [1] Ripple\ripple_basics\system + + + [1] Ripple\ripple_basics\system + + + [1] Ripple\ripple_basics\system + + + [1] Ripple\ripple_basics\types + + + [1] Ripple\ripple_basics\types + + + [1] Ripple\ripple_basics\types + + + [1] Ripple\ripple_basics\utility - 1. Modules\ripple_basics\utility + [1] Ripple\ripple_basics\utility - - 1. Modules\ripple_data\protocol + + [1] Ripple\ripple_basics\utility - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\db - - - 0. Third Party Code\LevelDB\port\win - - - 0. Third Party Code\LevelDB\port - - - 0. Third Party Code\LevelDB\port - - - 0. Third Party Code\LevelDB\port - - - 0. Third Party Code\LevelDB\port - - - 0. Third Party Code\LevelDB\port - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\table - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 0. Third Party Code\LevelDB\util - - - 1. Modules\ripple_websocket - - - 1. Modules\ripple_websocket\autosocket - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_websocket\subtree - - - 1. Modules\ripple_app\_unfactored\db - - - 1. Modules\ripple_app\_unfactored\db - - - 1. Modules\ripple_sqlite - - - 1. Modules\ripple_leveldb - - - 1. Modules\ripple_core - - - 1. Modules\ripple_core\functional - - - 1. Modules\ripple_core\functional - - - 1. Modules\ripple_core\functional - - - 1. Modules\ripple_core\functional + + [1] Ripple\ripple_basics\utility - 1. Modules\ripple_basics\utility + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_basics - 1. Modules\ripple_core\functional + [1] Ripple\ripple_core\functional - 1. Modules\ripple_core\functional + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core\functional + + + [1] Ripple\ripple_core + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\crypto + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\utility + + + [1] Ripple\ripple_data + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json + + + [1] Ripple\ripple_leveldb + + + [1] Ripple\ripple_sqlite + + + [1] Ripple\ripple_websocket\autosocket + + + [1] Ripple\ripple_websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\websocket + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\db + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\util + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\table + + + [0] Subtrees\leveldb\port\win + + + [0] Subtrees\leveldb\port + + + [0] Subtrees\leveldb\port + + + [0] Subtrees\leveldb\port + + + [0] Subtrees\leveldb\port + + + [0] Subtrees\leveldb\port + + + [0] Subtrees\leveldb\port + + + [1] Ripple\ripple_client + + + [1] Ripple\ripple_app\_contracts + + + [1] Ripple\ripple_app\_contracts + + + [1] Ripple\ripple_app\_contracts + + + [1] Ripple\ripple_app\_contracts + + + [1] Ripple\ripple_app\_misc\_consensus + + + [1] Ripple\ripple_app\_misc\_consensus + + + [1] Ripple\ripple_app\_data + + + [1] Ripple\ripple_app\_data + + + [1] Ripple\ripple_app\_data + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_ledger + + + [1] Ripple\ripple_app\_main + + + [1] Ripple\ripple_app\_main + + + [1] Ripple\ripple_app\_main + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_network + + + [1] Ripple\ripple_app\_network + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_misc + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_pathing + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_rpc + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\_shamap + + + [1] Ripple\ripple_app\basics + + + [1] Ripple\ripple_app\basics + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions + + + [1] Ripple\ripple_app\_transactions - 1. Modules\ripple_app\refactored + [1] Ripple\ripple_app\_main + + + [1] Ripple\ripple_basics\utility + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_data\protocol + + + [1] Ripple\ripple_basio + + + [1] Ripple\ripple_basio + + + [1] Ripple\ripple_basio + + + [1] Ripple\ripple_basio\boost + + + [1] Ripple\ripple_net + + + [1] Ripple\ripple_net\basics + + + [1] Ripple\ripple_net\basics + + + [1] Ripple\ripple_app\basics + + + [1] Ripple\ripple_net\basics + + + [1] Ripple\ripple_net\basics + + + [1] Ripple\ripple_app\_network + + + [1] Ripple\ripple_app\_peers + + + [1] Ripple\ripple_basio\boost + + + [1] Ripple\ripple_app\_data + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_app\node + + + [1] Ripple\ripple_mdb + + + [1] Ripple\ripple_hyperleveldb - - - - 0. Third Party Code\protobuf\protobuf - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json - - - 1. Modules\ripple_json\json - - - - - - + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + [1] Ripple\ripple_json\json + + + + + + + diff --git a/LEVELDB.txt b/LEVELDB.txt deleted file mode 100644 index 9685fd2082..0000000000 --- a/LEVELDB.txt +++ /dev/null @@ -1,14 +0,0 @@ - -To use LevelDB, follow these steps: - -1) Add the following to your rippled.cfg file: -[node_db] -LevelDB - -2) If you have no databases, you can just start rippled. If you have -databases you don't care about, just remove them. Otherwise, start rippled -with '--import' to import your existing hashnode database into LevelDB. -After you import the hashnode database, either delete (or move) the -'hashnode.db', 'hashnode.db-shm' and 'hashnode.db-wal' files from your -database directory. Do not touch the LevelDB files in the 'hashnode' -directory. diff --git a/LICENSE b/LICENSE index 44b4f56b05..98798675e5 100644 --- a/LICENSE +++ b/LICENSE @@ -10,7 +10,10 @@ Copyright (c) 2011-2012 Jed McCaleb Some code: Copyright (c) 2013 Vinnie Falco - Contributions from Vinnie Falco provided under the terms of the ISC License: +Copyright (c) 2013 Bob Way +Copyright (c) 2013 Eric Lombrozo + Contributions from Vinnie Falco, Bob Way, and Eric Lombrozo + provided under the terms of the ISC License: Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. diff --git a/RippleD.props b/RippleD.props deleted file mode 100644 index 7466630c5c..0000000000 --- a/RippleD.props +++ /dev/null @@ -1,34 +0,0 @@ - - - - - . - $(RepoDir)\src\cpp\ripple - - - $(SolutionDir)build\VisualStudio2012\$(Configuration).$(Platform)\ - $(SolutionDir)build\obj\VisualStudio2012\$(Configuration).$(Platform)\ - rippled - - - - USE_LEVELDB;BOOST_TEST_ALTERNATIVE_INIT_API;BOOST_TEST_NO_MAIN;_WIN32_WINNT=0x0600;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;WIN32;%(PreprocessorDefinitions) - true - Level3 - $(RepoDir);$(RepoDir)\src\cpp\leveldb;$(RepoDir)\src\cpp\leveldb\include;$(RepoDir)\src\cpp\protobuf\src;$(RepoDir)\src\cpp\protobuf\vsprojects;$(RepoDir)\build\proto;$(RepoDir)\Subtrees\beast;%(AdditionalIncludeDirectories) - /bigobj %(AdditionalOptions) - Async - - - Shlwapi.lib;%(AdditionalDependencies) - - - - - $(RepoDir) - - - $(SrcDir) - - - \ No newline at end of file diff --git a/SConstruct b/SConstruct index e5e3779d30..5f011bd8e3 100644 --- a/SConstruct +++ b/SConstruct @@ -9,17 +9,17 @@ import os import platform import re -LevelDB = bool(1) - OSX = bool(platform.mac_ver()[0]) FreeBSD = bool('FreeBSD' == platform.system()) Linux = bool('Linux' == platform.system()) Ubuntu = bool(Linux and 'Ubuntu' == platform.linux_distribution()[0]) if OSX or Ubuntu: - CTAGS = '/usr/bin/ctags' + CTAGS = 'ctags' +elif FreeBSD: + CTAGS = 'exctags' else: - CTAGS = '/usr/bin/exuberant-ctags' + CTAGS = 'exuberant-ctags' # # scons tools @@ -35,6 +35,13 @@ GCC_VERSION = re.split('\.', commands.getoutput(env['CXX'] + ' -dumpversion')) #env.Replace(CC = 'clang') #env.Replace(CXX = 'clang++') +# Use a newer gcc on FreeBSD +if FreeBSD: + env.Replace(CC = 'gcc46') + env.Replace(CXX = 'g++46') + env.Append(CCFLAGS = ['-Wl,-rpath=/usr/local/lib/gcc46']) + env.Append(LINKFLAGS = ['-Wl,-rpath=/usr/local/lib/gcc46']) + # # Builder for CTags # @@ -47,6 +54,16 @@ else: # Use openssl env.ParseConfig('pkg-config --cflags --libs openssl') +# Use protobuf +env.ParseConfig('pkg-config --cflags --libs protobuf') + +# Beast uses kvm on FreeBSD +if FreeBSD: + env.Append ( + LIBS = [ + 'kvm' + ] + ) # The required version of boost is documented in the README file. # @@ -55,6 +72,7 @@ env.ParseConfig('pkg-config --cflags --libs openssl') # If a threading library is included the platform can be whitelisted. # # FreeBSD and Ubuntu non-mt libs do link with pthreads. + if FreeBSD or Ubuntu: env.Append( LIBS = [ @@ -103,13 +121,6 @@ INCLUDE_PATHS = [ COMPILED_FILES = [ 'Subtrees/beast/modules/beast_core/beast_core.cpp', 'Subtrees/beast/modules/beast_basics/beast_basics.cpp', - 'modules/ripple_basics/ripple_basics.cpp', - 'modules/ripple_core/ripple_core.cpp', - 'modules/ripple_data/ripple_data.cpp', - 'modules/ripple_json/ripple_json.cpp', - 'modules/ripple_leveldb/ripple_leveldb.cpp', - 'modules/ripple_websocket/ripple_websocket.cpp', - 'modules/ripple_sqlite/ripple_sqlite.c', 'modules/ripple_app/ripple_app_pt1.cpp', 'modules/ripple_app/ripple_app_pt2.cpp', 'modules/ripple_app/ripple_app_pt3.cpp', @@ -117,7 +128,18 @@ COMPILED_FILES = [ 'modules/ripple_app/ripple_app_pt5.cpp', 'modules/ripple_app/ripple_app_pt6.cpp', 'modules/ripple_app/ripple_app_pt7.cpp', - 'modules/ripple_app/ripple_app_pt8.cpp' + 'modules/ripple_app/ripple_app_pt8.cpp', + 'modules/ripple_basics/ripple_basics.cpp', + 'modules/ripple_basio/ripple_basio.cpp', + 'modules/ripple_core/ripple_core.cpp', + 'modules/ripple_data/ripple_data.cpp', + 'modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp', + 'modules/ripple_json/ripple_json.cpp', + 'modules/ripple_leveldb/ripple_leveldb.cpp', + 'modules/ripple_mdb/ripple_mdb.c', + 'modules/ripple_net/ripple_net.cpp', + 'modules/ripple_websocket/ripple_websocket.cpp', + 'modules/ripple_sqlite/ripple_sqlite.c' ] #------------------------------------------------------------------------------- @@ -146,25 +168,18 @@ if not FreeBSD: ] ) -# Apparently, pkg-config --libs protobuf on bsd fails to provide this necessary include dir. -if FreeBSD: - env.Append(LINKFLAGS = ['-I/usr/local/include']) - env.Append(CXXFLAGS = ['-DOS_FREEBSD']) - env.Append( LIBS = [ 'rt', # for clock_nanosleep in beast - 'protobuf', 'z' ] ) DEBUGFLAGS = ['-g', '-DDEBUG'] -BOOSTFLAGS = ['-DBOOST_TEST_DYN_LINK', '-DBOOST_FILESYSTEM_NO_DEPRECATED'] env.Append(LINKFLAGS = ['-rdynamic', '-pthread']) env.Append(CCFLAGS = ['-pthread', '-Wall', '-Wno-sign-compare', '-Wno-char-subscripts']) -env.Append(CXXFLAGS = ['-O0', '-pthread', '-Wno-invalid-offsetof', '-Wformat']+BOOSTFLAGS+DEBUGFLAGS) +env.Append(CXXFLAGS = ['-O0', '-pthread', '-Wno-invalid-offsetof', '-Wformat']+DEBUGFLAGS) # RTTI is required for Beast and CountedObject. # @@ -173,6 +188,10 @@ env.Append(CXXFLAGS = ['-frtti']) if (int(GCC_VERSION[0]) > 4 or (int(GCC_VERSION[0]) == 4 and int(GCC_VERSION[1]) >= 7)): env.Append(CXXFLAGS = ['-std=c++11']) +# FreeBSD doesn't support O_DSYNC +if FreeBSD: + env.Append(CPPFLAGS = ['-DMDB_DSYNC=O_SYNC']) + if OSX: env.Append(LINKFLAGS = ['-L/usr/local/opt/openssl/lib']) env.Append(CXXFLAGS = ['-I/usr/local/opt/openssl/include']) @@ -186,7 +205,7 @@ TAG_SRCS = copy.copy(COMPILED_FILES) # Derive the object files from the source files. OBJECT_FILES = [] -OBJECT_FILES += PROTO_SRCS +OBJECT_FILES.append(PROTO_SRCS[0]) for file in COMPILED_FILES: OBJECT_FILES.append('build/obj/' + file) diff --git a/Subtrees/README.md b/Subtrees/README.md index 75c170dc28..51435b4def 100644 --- a/Subtrees/README.md +++ b/Subtrees/README.md @@ -10,9 +10,11 @@ http://blogs.atlassian.com/2013/05/alternatives-to-git-submodule-git-subtree/ ``` -git@github.com:vinniefalco/LevelDB.git +git@github.com:ripple/LevelDB.git ``` Branch ``` @@ -21,9 +23,11 @@ ripple-fork ## websocket +Ripple's fork of websocketpp has some incompatible changes and Ripple specific includes. + Repository ``` -git@github.com:vinniefalco/websocketpp.git +git@github.com:ripple/websocketpp.git ``` Branch ``` @@ -32,9 +36,12 @@ ripple-fork ## protobuf +Ripple's fork of protobuf doesn't have any actual changes, but since the upstream +repository uses SVN, we have created a Git version to use with the git-subtree command. + Repository ``` -git@github.com:vinniefalco/protobuf.git +git@github.com:ripple/protobuf.git ``` Branch ``` diff --git a/Subtrees/beast/Builds/VisualStudio2012/Beast.props b/Subtrees/beast/Builds/VisualStudio2012/Beast.props new file mode 100644 index 0000000000..4e9ca3e391 --- /dev/null +++ b/Subtrees/beast/Builds/VisualStudio2012/Beast.props @@ -0,0 +1,13 @@ + + + + + + + + Level4 + _CRTDBG_MAP_ALLOC;%(PreprocessorDefinitions) + + + + \ No newline at end of file diff --git a/Subtrees/beast/Builds/VisualStudio2012/BeastConfig.h b/Subtrees/beast/Builds/VisualStudio2012/BeastConfig.h index 56a26c54dd..0a19d0402d 100644 --- a/Subtrees/beast/Builds/VisualStudio2012/BeastConfig.h +++ b/Subtrees/beast/Builds/VisualStudio2012/BeastConfig.h @@ -17,35 +17,107 @@ */ //============================================================================== -#ifndef BEAST_BEASTCONFIG_HEADER -#define BEAST_BEASTCONFIG_HEADER +#ifndef BEAST_BEASTCONFIG_H_INCLUDED +#define BEAST_BEASTCONFIG_H_INCLUDED -// beast_core flags: +/** Configuration file for Beast. -#ifndef BEAST_FORCE_DEBUG - //#define BEAST_FORCE_DEBUG + This sets various configurable options for Beast. In order to compile you + must place a copy of this file in a location where your build environment + can find it, and then customize its contents to suit your needs. + + @file BeastConfig.h +*/ + +//------------------------------------------------------------------------------ + +/** Config: BEAST_FORCE_DEBUG + + Normally, BEAST_DEBUG is set to 1 or 0 based on compiler and project + settings, but if you define this value, you can override this to force it + to be true or false. +*/ +#ifndef BEAST_FORCE_DEBUG +//#define BEAST_FORCE_DEBUG 0 #endif -#ifndef BEAST_LOG_ASSERTIONS - //#define BEAST_LOG_ASSERTIONS 1 +//------------------------------------------------------------------------------ + +/** Config: BEAST_LOG_ASSERTIONS + + If this flag is enabled, the the bassert and bassertfalse macros will always + use Logger::writeToLog() to write a message when an assertion happens. + + Enabling it will also leave this turned on in release builds. When it's + disabled, however, the bassert and bassertfalse macros will not be compiled + in a release build. + + @see bassert, bassertfalse, Logger +*/ +#ifndef BEAST_LOG_ASSERTIONS +//#define BEAST_LOG_ASSERTIONS 0 #endif -#ifndef BEAST_CHECK_MEMORY_LEAKS - //#define BEAST_CHECK_MEMORY_LEAKS +//------------------------------------------------------------------------------ + +/** Config: BEAST_CHECK_MEMORY_LEAKS + + Enables a memory-leak check for certain objects when the app terminates. + See the LeakChecked class for more details about enabling leak checking for + specific classes. +*/ +#ifndef BEAST_CHECK_MEMORY_LEAKS +//#define BEAST_CHECK_MEMORY_LEAKS 1 #endif -#ifndef BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES - //#define BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES +//------------------------------------------------------------------------------ + +/** Config: BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES + + In a Visual C++ build, this can be used to stop the required system libs + being automatically added to the link stage. +*/ +#ifndef BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES +//#define BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES 1 #endif -// beast_basics flags +//------------------------------------------------------------------------------ -#ifndef BEAST_USE_BOOST -#define BEAST_USE_BOOST 0 +/** Config: BEAST_INCLUDE_ZLIB_CODE + + This can be used to disable Beast's embedded 3rd-party zlib code. + You might need to tweak this if you're linking to an external zlib library in your app, + but for normal apps, this option should be left alone. + + If you disable this, you might also want to set a value for BEAST_ZLIB_INCLUDE_PATH, to + specify the path where your zlib headers live. +*/ +#ifndef BEAST_INCLUDE_ZLIB_CODE +//#define BEAST_INCLUDE_ZLIB_CODE 0 #endif -#ifndef BEAST_USE_LEAKCHECKED -#define BEAST_USE_LEAKCHECKED BEAST_CHECK_MEMORY_LEAKS +#ifndef BEAST_ZLIB_INCLUDE_PATH +#define BEAST_ZLIB_INCLUDE_PATH #endif +//------------------------------------------------------------------------------ + +/** Config: BEAST_BOOST_IS_AVAILABLE + + This activates boost specific features and improvements. +*/ +#ifndef BEAST_BOOST_IS_AVAILABLE +#define BEAST_BOOST_IS_AVAILABLE 0 +#endif + +/** Bind source configuration. + + Set one of these to manually force a particular implementation of bind(). + If nothing is chosen then beast will use whatever is appropriate for your + environment based on what is available. +*/ +//#define BEAST_BIND_USES_STD 1 +//#define BEAST_BIND_USES_TR1 1 +//#define BEAST_BIND_USES_BOOST 1 + #endif diff --git a/Subtrees/beast/Builds/VisualStudio2012/beast.vcxproj b/Subtrees/beast/Builds/VisualStudio2012/beast.vcxproj index 04af470a2b..2b91845bd7 100644 --- a/Subtrees/beast/Builds/VisualStudio2012/beast.vcxproj +++ b/Subtrees/beast/Builds/VisualStudio2012/beast.vcxproj @@ -5,10 +5,18 @@ Debug Win32 + + Debug + x64 + Release Win32 + + Release + x64 + @@ -18,71 +26,60 @@ true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true - - - - - - - - - - - + - - - - - - - - - - - - - - + @@ -91,7 +88,6 @@ - @@ -102,14 +98,25 @@ + + + + + + + + + + + @@ -122,19 +129,27 @@ + + + + + + - + - + + + @@ -157,6 +172,7 @@ + @@ -184,6 +200,7 @@ + @@ -191,6 +208,7 @@ + @@ -216,438 +234,675 @@ true + true true + true - - true - true - - - true - true - - - true - true - - + true true + true + true true + true true - - - true - true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true - - - true - true - - - true - true - - - true - true - - - true - true + true true + true true - - - true - true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true true + true true + true true + true true + true true + true true + true true + true true + true + + + true + true + true + true true + true true + true true + true true + true + + + true + true + true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true + + + true + true + true + true true + true true + true true + true true + true true + true true + true true + true true + true + + + true + true + true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true true + true + + + {73C5A0F0-7629-4DE7-9194-BE7AC6C19535} Win32Proj @@ -660,6 +915,12 @@ v110 Unicode + + StaticLibrary + true + v110 + Unicode + StaticLibrary false @@ -667,14 +928,31 @@ true Unicode + + StaticLibrary + false + v110 + true + Unicode + + + + + + + + + + + @@ -682,7 +960,19 @@ - Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + $(ProjectDir) + + + Windows + true + + + + + + Disabled WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) $(ProjectDir) @@ -694,7 +984,23 @@ - Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + $(ProjectDir) + + + Windows + true + true + true + + + + MaxSpeed diff --git a/Subtrees/beast/Builds/VisualStudio2012/beast.vcxproj.filters b/Subtrees/beast/Builds/VisualStudio2012/beast.vcxproj.filters index f2f88ba0bc..51db6d1515 100644 --- a/Subtrees/beast/Builds/VisualStudio2012/beast.vcxproj.filters +++ b/Subtrees/beast/Builds/VisualStudio2012/beast.vcxproj.filters @@ -95,9 +95,6 @@ {e3a8f3eb-7f0f-4b81-b978-0dd0823f583b} - - {3e9389c0-c8f0-4657-ab11-cbbea889d3be} - {ba11b980-76dd-49a4-b2c7-878e9f08f8ce} @@ -113,12 +110,12 @@ {9e850052-6ab7-4a65-911d-adfde81ceb5f} - - {811c5374-8959-4df9-aba9-a7e27b85046e} - {f58dddf7-fe43-49a2-8e57-91feba586119} + + {69e28551-92ea-420b-a465-75ed248e3b59} + @@ -223,9 +220,6 @@ beast_core\memory - - beast_core\memory - beast_core\memory @@ -241,9 +235,6 @@ beast_core\memory - - beast_core\memory - beast_core\memory @@ -473,57 +464,15 @@ beast_basics - - beast_basics\containers - - - beast_basics\containers - - - beast_basics\containers - - - beast_basics\containers - - - beast_basics\containers - beast_basics\diagnostic - - beast_basics\diagnostic - - - beast_basics\diagnostic - - - beast_basics\diagnostic - - - beast_basics\diagnostic - - - beast_basics\diagnostic - - - beast_basics\diagnostic - beast_basics\events - - beast_basics\events - - - beast_basics\functor - beast_basics\functor - - beast_basics\math - beast_basics\math @@ -533,21 +482,6 @@ beast_basics\memory - - beast_basics\memory - - - beast_basics\memory - - - beast_basics\memory - - - beast_basics\memory - - - beast_basics\memory - beast_basics\memory @@ -563,30 +497,12 @@ beast_basics\memory - - beast_basics\memory - beast_basics\memory - - beast_basics\memory - - - beast_basics\memory - - - beast_basics\memory - beast_basics\threads - - beast_basics\threads - - - beast_basics\threads - beast_basics\threads @@ -611,15 +527,90 @@ beast_basics\threads - - beast_basics\threads - beast_basics\threads beast_basics\threads + + beast_core\containers + + + beast_core\memory + + + beast_core\memory + + + beast_core\memory + + + beast_core\memory + + + beast_core\memory + + + beast_core\memory + + + beast_core\containers + + + beast_core\containers + + + beast_core\threads + + + beast_core\memory + + + beast_core\containers + + + beast_core\containers + + + beast_core\diagnostic + + + beast_core\diagnostic + + + beast_core\diagnostic + + + beast_core\diagnostic + + + beast_core\diagnostic + + + beast_core\diagnostic + + + beast_core\memory + + + beast_core\memory + + + beast_core\system + + + beast_core\maths + + + beast_core\time + + + beast_basics\threads + + + beast_basics\events + @@ -877,24 +868,9 @@ beast_basics\diagnostic - - beast_basics\diagnostic - - - beast_basics\diagnostic - - - beast_basics\diagnostic - - - beast_basics\diagnostic - beast_basics\events - - beast_basics\events - beast_basics\math @@ -910,24 +886,9 @@ beast_basics\memory - - beast_basics\native - - - beast_basics\native - - - beast_basics\native - - - beast_basics\native - beast_basics\threads - - beast_basics\threads - beast_basics\threads @@ -952,5 +913,47 @@ beast_basics\threads + + beast_core\native + + + beast_core\native + + + beast_core\native + + + beast_core\native + + + beast_core\diagnostic + + + beast_core\diagnostic + + + beast_core\diagnostic + + + beast_core\diagnostic + + + beast_core\threads + + + beast_core\native + + + beast_core\native + + + beast_core\time + + + beast_basics\events + + + + \ No newline at end of file diff --git a/Subtrees/beast/ReadMe.md b/Subtrees/beast/ReadMe.md index 57c2293e6b..81be44cd33 100644 --- a/Subtrees/beast/ReadMe.md +++ b/Subtrees/beast/ReadMe.md @@ -9,6 +9,23 @@ The hope is that this will replace the use of boost and other cumbersome jalopie ## JUCE -Beast is based on the beast_core module which is provided under the ISC +Parts of Beast are based on the juce_core module which is provided under the ISC license. More information about JUCE is available at + http://www.juce.com + +## License + +Beast is provided under the terms of the ISC license: + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Subtrees/beast/TODO.txt b/Subtrees/beast/TODO.txt new file mode 100644 index 0000000000..a5faea1456 --- /dev/null +++ b/Subtrees/beast/TODO.txt @@ -0,0 +1,60 @@ +-------------------------------------------------------------------------------- +BEAST TODO +-------------------------------------------------------------------------------- + +- Design a WeakPtr / SharedPtr / SharedObject intrusive system + +- Implement beast::Bimap? + +- Use Bimap for storage in the DeadlineTimer::Manager, to support + thousands of timers. + +- Think about adding a shouldStop bool to InterruptibleThread, along + with a shouldStop () function returning bool, and a stop() method. + +- Make OwnedArray add routines return a pointer instead of reference + +- Tidy up CacheLine, MemoryAlignment + +- Remove anything having to do with DLL builds like + BEAST_DLL, BEAST_DLL_BUILD, BEAST_DISABLE_DLL_ALLOCATORS + +- Fix FifoFreeStoreWithTLS reference counting bug + +- Implement a reasonable substitute for boost's thread_local_storage + +- Think about doing away with BEAST_CALLTYPE and BEAST_API + +- Decide if headers should just include BeastConfig.h instead of making the + host program do it. + +- Rename malloc/calloc JUCE members that conflict with the debug CRT from MSVC + +- Make beast::HashMap support assignment via operator[] + +- Reformat every Doxygen comment +- Fix Doxygen metatags +- update Beast Doxyfile + +- Rename include guards to boost style, e.g. BEAST_THROW_H_INCLUDED + +- Decide if we should get rid of AtomicCounter, AtomicFlag, AtomicPointer, AtomicState + +- Clean up CacheLine, StaticObject + +- Clean up ConcurrentObject + +- Rename SharedData to SharedState or something? + +- Figure out what to do with ReadWriteLock, and NamedPipe which uses it? + +- Put BEAST_PUBLIC_FUNCTION in front of all loose functions + +- restructure the repo sources to look like this: + /Source/beast/beast_core/beast_core.h + etc... +- Put the BeastConfig.h at the root of the repo. +- Make sure the template BeastConfig.h is included in the Doxyfile + +- Implement robust key/value database with bulk write + diff --git a/Subtrees/beast/modules/beast_basics/beast_basics.cpp b/Subtrees/beast/modules/beast_basics/beast_basics.cpp index 4a62756512..05d51a7630 100644 --- a/Subtrees/beast/modules/beast_basics/beast_basics.cpp +++ b/Subtrees/beast/modules/beast_basics/beast_basics.cpp @@ -41,39 +41,19 @@ namespace beast { #include "diagnostic/beast_CatchAny.cpp" -#include "diagnostic/beast_Debug.cpp" -#include "diagnostic/beast_Error.cpp" -#include "diagnostic/beast_FPUFlags.cpp" -#include "diagnostic/beast_LeakChecked.cpp" +#include "events/beast_DeadlineTimer.cpp" #include "events/beast_OncePerSecond.cpp" -#include "events/beast_PerformedAtExit.cpp" #include "math/beast_MurmurHash.cpp" #include "threads/beast_InterruptibleThread.cpp" #include "threads/beast_Semaphore.cpp" - -#if BEAST_WINDOWS -#include "native/beast_win32_FPUFlags.cpp" -#include "native/beast_win32_Threads.cpp" - -#else -#include "native/beast_posix_FPUFlags.cpp" -#include "native/beast_posix_Threads.cpp" - -#endif - -#if BEAST_USE_BOOST #include "memory/beast_FifoFreeStoreWithTLS.cpp" -#else #include "memory/beast_FifoFreeStoreWithoutTLS.cpp" -#endif #include "memory/beast_GlobalPagedFreeStore.cpp" #include "memory/beast_PagedFreeStore.cpp" - #include "threads/beast_CallQueue.cpp" -#include "threads/beast_ConcurrentObject.cpp" #include "threads/beast_Listeners.cpp" #include "threads/beast_ManualCallQueue.cpp" #include "threads/beast_ParallelFor.cpp" diff --git a/Subtrees/beast/modules/beast_basics/beast_basics.h b/Subtrees/beast/modules/beast_basics/beast_basics.h index 29049bed4a..2dea753bb9 100644 --- a/Subtrees/beast/modules/beast_basics/beast_basics.h +++ b/Subtrees/beast/modules/beast_basics/beast_basics.h @@ -23,362 +23,219 @@ @ingroup beast_basics */ -#ifndef BEAST_BASICS_BEASTHEADER -#define BEAST_BASICS_BEASTHEADER +#ifndef BEAST_BASICS_H_INCLUDED +#define BEAST_BASICS_H_INCLUDED -//============================================================================== +//------------------------------------------------------------------------------ + +/* If you fail to make sure that all your compile units are building Beast with + the same set of option flags, then there's a risk that different compile + units will treat the classes as having different memory layouts, leading to + very nasty memory corruption errors when they all get linked together. + That's why it's best to always include the BeastConfig.h file before any + beast headers. +*/ +#ifndef BEAST_BEASTCONFIG_H_INCLUDED +# ifdef _MSC_VER +# pragma message ("Have you included your BeastConfig.h file before including the Beast headers?") +# else +# warning "Have you included your BeastConfig.h file before including the Beast headers?" +# endif +#endif + +//------------------------------------------------------------------------------ /** - @mainpage Beast: A multipurpose library using parts of JUCE. - ### Version 1.1 +@mainpage Beast: A C++ library for server development. - Copyright (C) 2008 by Vinnie Falco \ ([e-mail][0]) +### Version 1.0 - Beast is a source code collection of individual modules containing - functionality for a variety of applications, with an emphasis on building - concurrent systems. Beast requires [JUCE][3] (Jules' Utility Class - Extensions), available from [Raw Material Software][4]. JUCE is available - under both the [GNU General Public License][5] and a [commercial license][6]. - Other than JUCE, Beast has no external dependencies. +Copyright 2008, 2013 by Vinnie Falco \ ([e-mail][0]) - Beast is hosted on Github at [https://github.com/vinniefalco/Beast][1] +Beast is a source code collection of individual modules containing +functionality for a variety of applications, with an emphasis on building +concurrent systems. Beast incorporates parts of [JUCE][3] (Jules' Utility +Class Extensions), available from [Raw Material Software][4]. Beast has no +external dependencies - The online documentation is at [http://vinniefalco.github.com/Beast][2] +Beast is hosted on Github at [https://github.com/vinniefalco/Beast][1] - ## Platforms +The online documentation is at [http://vinniefalco.github.com/Beast][2] - All platforms supported by JUCE are also supported by Beast. Currently these - platforms include: +## Platforms - - **Windows**: Applications and VST/RTAS/NPAPI/ActiveX plugins can be built - using MS Visual Studio. The results are all fully compatible with Windows - XP, Vista or Windows 7. +All platforms supported by JUCE are also supported by Beast. Currently these +platforms include: - - **Mac OS X**: Applications and VST/AudioUnit/RTAS/NPAPI plugins with Xcode. +- **Windows**: Applications and VST/RTAS/NPAPI/ActiveX plugins can be built +using MS Visual Studio. The results are all fully compatible with Windows +XP, Vista or Windows 7. - - **GNU/Linux**: Applications and plugins can be built for any kernel 2.6 or - later. +- **Mac OS X**: Applications and VST/AudioUnit/RTAS/NPAPI plugins with Xcode. - - **iOS**: Native iPhone and iPad apps. +- **GNU/Linux**: Applications and plugins can be built for any kernel 2.6 or +later. - - **Android**: Supported. +- **FreeBSD**: Kernel version 8.4 or higher required. - ## Prerequisites +- **iOS**: Native iPhone and iPad apps. - This documentation assumes that the reader has a working knowledge of JUCE. - Some modules built on external libraries assume that the reader understands - the operation of those external libraries. Certain modules assume that the - reader understands additional domain-specific information. Modules with - additional prerequisites are marked in the documentation. +- **Android**: Supported. - ## External Modules +## Prerequisites - Some modules bring in functionality provided by external libraries. For - example, the @ref beast_bzip2 module provides the compression and decompression - algorithms in [bZip2][7]. Usage of these external library modules is optional. - They come with complete source code, as well as options for using either - system or user provided variants of the external libraries: it is not - necessary to download additional source code packages to use these modules. +This documentation assumes that the reader has a working knowledge of JUCE. +Some modules built on external libraries assume that the reader understands +the operation of those external libraries. Certain modules assume that the +reader understands additional domain-specific information. Modules with +additional prerequisites are marked in the documentation. - External code incorporated into Beast is covered by separate licenses. See - the licensing information and notes in the corresponding source files for - copyright information and terms of use. +## External Modules - ## Integration +Some modules bring in functionality provided by external libraries. For +example, the @ref beast_bzip2 module provides the compression and decompression +algorithms in [bZip2][7]. Usage of these external library modules is optional. +They come with complete source code, as well as options for using either +system or user provided variants of the external libraries: it is not +necessary to download additional source code packages to use these modules. - Beast requires recent versions of JUCE. It won't work with versions 1.53 or - earlier. To use the library it is necessary to first download JUCE to a - location where your development environment can find it. Or, you can use your - existing installation of JUCE. +External code incorporated into Beast is covered by separate licenses. See +the licensing information and notes in the corresponding source files for +copyright information and terms of use. - This library uses the same modularized organizational structure as JUCE. To - use a module, first add a path to the list of includes searched by your - development environment or project, which points to the Beast directory. Then, - add the single corresponding .c or .cpp file to your existing project which - already uses JUCE. For example, to use the @ref beast_core module, add the file - beast_core.cpp to your project. Some modules depend on other modules. +## Integration - To use a module, include the appropriate header from within your source code. - For example, to access classes in the @ref beast_concurrent module, use this: +Beast requires recent versions of JUCE. It won't work with versions 1.53 or +earlier. To use the library it is necessary to first download JUCE to a +location where your development environment can find it. Or, you can use your +existing installation of JUCE. - @code +This library uses the same modularized organizational structure as JUCE. To +use a module, first add a path to the list of includes searched by your +development environment or project, which points to the Beast directory. Then, +add the single corresponding .c or .cpp file to your existing project which +already uses JUCE. For example, to use the @ref beast_core module, add the file +beast_core.cpp to your project. Some modules depend on other modules. - #include "modules/beast_concurrent/beast_concurrent.h" +To use a module, include the appropriate header from within your source code. +For example, to access classes in the @ref beast_concurrent module, use this: - @endcode +@code - Then add the corresponding file beast_concurrent.cpp to your build. +#include "modules/beast_concurrent/beast_concurrent.h" - ## AppConfig +@endcode - Some Beast features can be controlled at compilation time through - preprocessor directives. The available choices of compilation options are - described in AppConfig.h, located in the AppConfigTemplate directory. Copy - the provided settings into your existing AppConfig.h (a file used by JUCE - convention). +Then add the corresponding file beast_concurrent.cpp to your build. - ## License +## AppConfig - This library contains portions of other open source products covered by - separate licenses. Please see the corresponding source files for specific - terms. +Some Beast features can be controlled at compilation time through +preprocessor directives. The available choices of compilation options are +described in AppConfig.h, located in the AppConfigTemplate directory. Copy +the provided settings into your existing AppConfig.h (a file used by JUCE +convention). - Beast is provided under the terms of The MIT License (MIT): +## License - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: +This library contains portions of other open source products covered by +separate licenses. Please see the corresponding source files for specific +terms. - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. +Beast is provided under the terms of The ISC License: - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - IN THE SOFTWARE. +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. - Some files contain portions of these external projects, licensed separately: +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - [bZip2][7] is Copyright (C) 1996-2010 Julian R Seward. All rights +Some files contain portions of these external projects, licensed separately: + +- [bZip2][7] is Copyright (C) 1996-2010 Julian R Seward. All rights reserved. See the corresponding file LICENSE for licensing terms. - - Portions of the software are Copyright (C) 1996-2001, 2006 by [The FreeType - Project][8]. All rights reserved. [FreeType][8] is distributed - under both the [GNU General Public License][5], or the - [FreeType License][9]. - - - Portions of this software are Copyright (C) 1994-2012 [Lua.org][10], PUC-Rio. - Lua is distributed under the terms of the [MIT License][11]. - - - [Luabridge][12] is Copyright (C) 2012 by Vinnie Falco and Copyrighted (C) - 2007 by Nathan Reed. [Luabridge][12] is distributed under the terms of the - [MIT License][11]. - - - [Soci][13] is Copyright (C) 2004-2008 Maciej Sobczak, Stephen Hutton, and +- [Soci][13] is Copyright (C) 2004-2008 Maciej Sobczak, Stephen Hutton, and various others noted in the corresponding source files. Soci is distributed under the [Boost Software License, Version 1.0][14]. - - [SQLite][15], placed in the public domain. +- [SQLite][15], placed in the public domain. - - [TagLib][16] is distributed under both the [GNU Lesser General Public License, - Version 2.1][17] and the [Mozilla Public License][18]. +[0]: mailto:vinnie.falco@gmail.com "Vinnie Falco (Email)" +[1]: https://github.com/vinniefalco/Beast "Beast Project" +[2]: http://vinniefalco.github.com/Beast/ "Beast Documentation" +[3]: http://rawmaterialsoftware.com/juce.php "JUCE" +[4]: http://rawmaterialsoftware.com/ "Raw Material Software" +[5]: http://www.gnu.org/licenses/gpl-2.0.html "GNU General Public License, version 2" +[6]: http://rawmaterialsoftware.com/jucelicense.php "JUCE Licenses" +[7]: http://www.bzip.org/ "bZip2: Home" +[8]: http://freetype.org/ "The FreeType Project" +[9]: http://www.freetype.org/FTL.TXT "The FreeType Project License" +[10]: http://www.lua.org/ "The Programming Language Lua" +[11]: http://opensource.org/licenses/ISC "The ISC License" +[12]: https://github.com/vinniefalco/LuaBridge +[13]: http://soci.sourceforge.net/ "SOCI" +[14]: http://www.boost.org/LICENSE_1_0.txt "Boost Software License, Version 1.0" +[15]: http://sqlite.org/ "SQLite Home Page" +[16]: http://developer.kde.org/~wheeler/taglib.html "TagLib" +[17]: http://www.gnu.org/licenses/lgpl-2.1.html "Gnu Lesser General Public License, version 2.1" +[18]: http://www.mozilla.org/MPL/1.1/ "Mozilla Public License" - [0]: mailto:vinnie.falco@gmail.com "Vinnie Falco (Email)" - [1]: https://github.com/vinniefalco/Beast "Beast Project" - [2]: http://vinniefalco.github.com/Beast/ "Beast Documentation" - [3]: http://rawmaterialsoftware.com/juce.php "JUCE" - [4]: http://rawmaterialsoftware.com/ "Raw Material Software" - [5]: http://www.gnu.org/licenses/gpl-2.0.html "GNU General Public License, version 2" - [6]: http://rawmaterialsoftware.com/jucelicense.php "JUCE Licenses" - [7]: http://www.bzip.org/ "bZip2: Home" - [8]: http://freetype.org/ "The FreeType Project" - [9]: http://www.freetype.org/FTL.TXT "The FreeType Project License" - [10]: http://www.lua.org/ "The Programming Language Lua" - [11]: http://www.opensource.org/licenses/mit-license.html "The MIT License" - [12]: https://github.com/vinniefalco/LuaBridge - [13]: http://soci.sourceforge.net/ "SOCI" - [14]: http://www.boost.org/LICENSE_1_0.txt "Boost Software License, Version 1.0" - [15]: http://sqlite.org/ "SQLite Home Page" - [16]: http://developer.kde.org/~wheeler/taglib.html "TagLib" - [17]: http://www.gnu.org/licenses/lgpl-2.1.html "Gnu Lesser General Public License, version 2.1" - [18]: http://www.mozilla.org/MPL/1.1/ "Mozilla Public License" - - @copyright Copyright (C) 2008 by Vinnie Falco \ ([e-mail][0]) - @copyright Provided under the [MIT License][11] +@copyright Copyright 2008-2013 by Vinnie Falco \ ([e-mail][0]) +@copyright Provided under the [ISC LIcense][11] */ -/*============================================================================*/ -/** - @internal +//------------------------------------------------------------------------------ - Implementation classes. +/** Implementation classes. - Thase classes are used internally. + Thase classes are used internally. - @defgroup internal internal + @defgroup internal internal + @internal */ -/*============================================================================*/ -/** - External modules. +//------------------------------------------------------------------------------ - These modules bring in functionality from third party or system libraries. +/** External modules. - @defgroup external external + These modules bring in functionality from third party or system libraries. + + @defgroup external external */ -/*============================================================================*/ -/** - Core classes. +//------------------------------------------------------------------------------ - This module provides core required functionality, and classes useful for - general development. All other modules require this module. +/** Core classes. - @todo Discuss the treatment of exceptions versus Error objects in the library. + This module provides core required functionality, and classes useful for + general development. All other modules require this module. - @todo Discuss the additions to AppConfig.h + @todo Discuss the treatment of exceptions versus Error objects in the + library. - @defgroup beast_core beast_core + @todo Discuss the additions to BeastConfig.h + + @defgroup beast_core beast_core */ -/* See the Juce notes regarding AppConfig.h - - This file must always be included before any Juce headers. - - There are some Beast specific build options that may be placed - into this file. See the AppConfig.h provided with Beast. -*/ - -/* BeastConfig.h must be included before this file */ - -/* Use sensible default configurations if they forgot - to append the necessary macros into their AppConfig.h. -*/ -#ifndef BEAST_USE_BOOST -#define BEAST_USE_BOOST 0 -#endif - -#ifndef BEAST_USE_LEAKCHECKED -#define BEAST_USE_LEAKCHECKED BEAST_CHECK_MEMORY_LEAKS -#endif - /* Get this early so we can use it. */ #include "../beast_core/system/beast_TargetPlatform.h" -#if BEAST_USE_BOOST +//------------------------------------------------------------------------------ + +#if BEAST_BOOST_IS_AVAILABLE #include #endif -#if BEAST_MSVC -# include -# include - -#elif BEAST_IOS -# if BEAST_USE_BOOST -# include -# include -# else -# include // detect std::lib -# if _LIBCPP_VERSION // libc++ -# include -# else // libstdc++ (GNU) -# include -# endif -# endif - -#elif BEAST_MAC -# include // detect std::lib -# if _LIBCPP_VERSION // libc++ -# include -# else // libstdc++ (GNU) -# include -# endif - -#elif BEAST_LINUX -# include - -#else -# error Unnkown platform! - -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _CRTDBG_MAP_ALLOC -#error "MSVC C Runtime Debug Macros not supported" -#endif - -// If the MSVC debug heap headers were included, disable -// the macros during the juce include since they conflict. -#ifdef _CRTDBG_MAP_ALLOC -#include -#include -#include - -#pragma push_macro("calloc") -#pragma push_macro("free") -#pragma push_macro("malloc") -#pragma push_macro("realloc") -#pragma push_macro("_recalloc") -#pragma push_macro("_aligned_free") -#pragma push_macro("_aligned_malloc") -#pragma push_macro("_aligned_offset_malloc") -#pragma push_macro("_aligned_realloc") -#pragma push_macro("_aligned_recalloc") -#pragma push_macro("_aligned_offset_realloc") -#pragma push_macro("_aligned_offset_recalloc") -#pragma push_macro("_aligned_msize") - -#undef calloc -#undef free -#undef malloc -#undef realloc -#undef _recalloc -#undef _aligned_free -#undef _aligned_malloc -#undef _aligned_offset_malloc -#undef _aligned_realloc -#undef _aligned_recalloc -#undef _aligned_offset_realloc -#undef _aligned_offset_recalloc -#undef _aligned_msize -#endif - #include "../beast_core/beast_core.h" -#ifdef _CRTDBG_MAP_ALLOC -#pragma pop_macro("_aligned_msize") -#pragma pop_macro("_aligned_offset_recalloc") -#pragma pop_macro("_aligned_offset_realloc") -#pragma pop_macro("_aligned_recalloc") -#pragma pop_macro("_aligned_realloc") -#pragma pop_macro("_aligned_offset_malloc") -#pragma pop_macro("_aligned_malloc") -#pragma pop_macro("_aligned_free") -#pragma pop_macro("_recalloc") -#pragma pop_macro("realloc") -#pragma pop_macro("malloc") -#pragma pop_macro("free") -#pragma pop_macro("calloc") -#endif - /** The Beast namespace. This namespace contains all Beast symbols. @@ -386,69 +243,28 @@ namespace beast { -// This group must come first since other files need it -#include "memory/beast_Uncopyable.h" -#include "diagnostic/beast_CatchAny.h" -#include "diagnostic/beast_Debug.h" -#include "diagnostic/beast_Error.h" -#include "diagnostic/beast_FPUFlags.h" -#include "diagnostic/beast_LeakChecked.h" -#include "diagnostic/beast_SafeBool.h" -#include "diagnostic/beast_Throw.h" +// Order matters -#include "containers/beast_List.h" -#include "containers/beast_LockFreeStack.h" -#include "containers/beast_LockFreeQueue.h" -#include "containers/beast_SharedTable.h" -#include "containers/beast_SortedLookupTable.h" - -#include "events/beast_OncePerSecond.h" -#include "events/beast_PerformedAtExit.h" - -#include "functor/beast_Bind.h" #include "functor/beast_Function.h" - -#include "math/beast_Interval.h" +#include "diagnostic/beast_CatchAny.h" +#include "events/beast_DeadlineTimer.h" +#include "events/beast_OncePerSecond.h" #include "math/beast_Math.h" #include "math/beast_MurmurHash.h" - -#include "memory/beast_MemoryAlignment.h" -#include "memory/beast_StaticObject.h" -#include "memory/beast_AtomicCounter.h" -#include "memory/beast_AtomicFlag.h" -#include "memory/beast_AtomicPointer.h" -#include "memory/beast_AtomicState.h" #include "memory/beast_AllocatedBy.h" -#include "memory/beast_RefCountedSingleton.h" -#include "memory/beast_FifoFreeStore.h" -#if BEAST_USE_BOOST -#include "memory/beast_FifoFreeStoreWithTLS.h" -#else -#include "memory/beast_FifoFreeStoreWithoutTLS.h" -#endif -#include "memory/beast_GlobalFifoFreeStore.h" -#include "memory/beast_GlobalPagedFreeStore.h" #include "memory/beast_PagedFreeStore.h" - -#if BEAST_MSVC -#pragma warning (push) -#pragma warning (disable: 4100) // unreferenced formal parmaeter -#pragma warning (disable: 4355) // 'this' used in base member -#endif -#include "memory/beast_CacheLine.h" -#if BEAST_MSVC -#pragma warning (pop) -#endif - +#include "memory/beast_GlobalPagedFreeStore.h" +#include "memory/beast_FifoFreeStoreWithTLS.h" +#include "memory/beast_FifoFreeStoreWithoutTLS.h" +#include "memory/beast_FifoFreeStore.h" +#include "memory/beast_GlobalFifoFreeStore.h" #include "threads/beast_Semaphore.h" #include "threads/beast_SerialFor.h" -#include "threads/beast_SpinDelay.h" #include "threads/beast_InterruptibleThread.h" #include "threads/beast_ReadWriteMutex.h" #include "threads/beast_ThreadGroup.h" #include "threads/beast_CallQueue.h" -#include "threads/beast_ConcurrentObject.h" -#include "threads/beast_ConcurrentState.h" +#include "threads/beast_SharedData.h" #include "threads/beast_GlobalThreadGroup.h" #include "threads/beast_Listeners.h" #include "threads/beast_ManualCallQueue.h" diff --git a/Subtrees/beast/modules/beast_basics/native/beast_posix_Threads.cpp b/Subtrees/beast/modules/beast_basics/beast_basics.mm similarity index 97% rename from Subtrees/beast/modules/beast_basics/native/beast_posix_Threads.cpp rename to Subtrees/beast/modules/beast_basics/beast_basics.mm index b6865169f0..dcfebe6055 100644 --- a/Subtrees/beast/modules/beast_basics/native/beast_posix_Threads.cpp +++ b/Subtrees/beast/modules/beast_basics/beast_basics.mm @@ -16,3 +16,5 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== + +#include "beast_basics.cpp" diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_CatchAny.h b/Subtrees/beast/modules/beast_basics/diagnostic/beast_CatchAny.h index 0d8e3ebecc..22df5ca2fc 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_CatchAny.h +++ b/Subtrees/beast/modules/beast_basics/diagnostic/beast_CatchAny.h @@ -20,8 +20,6 @@ #ifndef BEAST_CATCHANY_BEASTHEADER #define BEAST_CATCHANY_BEASTHEADER -#include "../functor/beast_Function.h" - /** Exception catcher. diff --git a/Subtrees/beast/modules/beast_basics/events/beast_DeadlineTimer.cpp b/Subtrees/beast/modules/beast_basics/events/beast_DeadlineTimer.cpp new file mode 100644 index 0000000000..ca5ab93f60 --- /dev/null +++ b/Subtrees/beast/modules/beast_basics/events/beast_DeadlineTimer.cpp @@ -0,0 +1,259 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +class DeadlineTimer::Manager + : public SharedSingleton + , public InterruptibleThread::EntryPoint +{ +private: + typedef CriticalSection LockType; + typedef List Items; + +public: + Manager () + : SharedSingleton (SingletonLifetime::persistAfterCreation) + , m_shouldStop (false) + , m_thread ("DeadlineTimer::Manager") + { + m_thread.start (this); + } + + ~Manager () + { + m_shouldStop = true; + + m_thread.interrupt (); + + bassert (m_items.empty ()); + } + + // Okay to call on an active timer. + // However, an extra notification may still happen due to concurrency. + // + void activate (DeadlineTimer* timer, double secondsRecurring, Time const& when) + { + bassert (secondsRecurring >= 0); + + LockType::ScopedLockType lock (m_mutex); + + if (timer->m_isActive) + { + m_items.erase (m_items.iterator_to (*timer)); + + timer->m_isActive = false; + } + + timer->m_secondsRecurring = secondsRecurring; + timer->m_notificationTime = when; + + insertSorted (*timer); + timer->m_isActive = true; + + m_thread.interrupt (); + } + + // Okay to call this on an inactive timer. + // This can happen naturally based on concurrency. + // + void deactivate (DeadlineTimer* timer) + { + LockType::ScopedLockType lock (m_mutex); + + if (timer->m_isActive) + { + m_items.erase (m_items.iterator_to (*timer)); + + timer->m_isActive = false; + } + + m_thread.interrupt (); + } + + void threadRun () + { + while (! m_shouldStop) + { + Time const currentTime = Time::getCurrentTime (); + double seconds = 0; + + { + LockType::ScopedLockType lock (m_mutex); + + // Notify everyone whose timer has expired + // + if (! m_items.empty ()) + { + for (;;) + { + Items::iterator const iter = m_items.begin (); + + // Has this timer expired? + if (iter->m_notificationTime <= currentTime) + { + // Yes, so call the listener. + // + // Note that this happens while the lock is held. + // + iter->m_listener->onDeadlineTimer (*iter); + + // Remove it from the list. + m_items.erase (iter); + + // Is the timer recurring? + if (iter->m_secondsRecurring > 0) + { + // Yes so set the timer again. + iter->m_notificationTime = + currentTime + RelativeTime (iter->m_secondsRecurring); + + // Keep it active. + insertSorted (*iter); + } + else + { + // Not a recurring timer, deactivate it. + iter->m_isActive = false; + } + } + else + { + break; + } + } + } + + // Figure out how long we need to wait. + // This has to be done while holding the lock. + // + if (! m_items.empty ()) + { + seconds = (m_items.front ().m_notificationTime - currentTime).inSeconds (); + } + else + { + seconds = 0; + } + } + + // Note that we have released the lock here. + // + if (seconds > 0) + { + // Wait until interrupt or next timer. + // + m_thread.wait (static_cast (seconds * 1000 + 0.5)); + } + else if (seconds == 0) + { + // Wait until interrupt + // + m_thread.wait (); + } + else + { + // Do not wait. This can happen if the recurring timer duration + // is extremely short, or if a listener wastes too much time in + // their callback. + } + } + } + + // Caller is responsible for locking + void insertSorted (DeadlineTimer& item) + { + if (! m_items.empty ()) + { + Items::iterator before = m_items.begin (); + + for (;;) + { + if (before->m_notificationTime >= item.m_notificationTime) + { + m_items.insert (before, item); + break; + } + + ++before; + + if (before == m_items.end ()) + { + m_items.push_back (item); + break; + } + } + } + else + { + m_items.push_back (item); + } + } + + static Manager* createInstance () + { + return new Manager; + } + +private: + CriticalSection m_mutex; + bool volatile m_shouldStop; + InterruptibleThread m_thread; + Items m_items; +}; + +//------------------------------------------------------------------------------ + +DeadlineTimer::DeadlineTimer (Listener* listener) + : m_listener (listener) + , m_manager (Manager::getInstance ()) + , m_isActive (false) +{ +} + +DeadlineTimer::~DeadlineTimer () +{ + m_manager->deactivate (this); +} + +void DeadlineTimer::setExpiration (double secondsUntilDeadline) +{ + bassert (secondsUntilDeadline > 0); + + Time const when = Time::getCurrentTime () + RelativeTime (secondsUntilDeadline); + + m_manager->activate (this, 0, when); +} + +void DeadlineTimer::setRecurringExpiration (double secondsUntilDeadline) +{ + bassert (secondsUntilDeadline > 0); + + Time const when = Time::getCurrentTime () + RelativeTime (secondsUntilDeadline); + + m_manager->activate (this, secondsUntilDeadline, when); +} + +void DeadlineTimer::setExpirationTime (Time const& when) +{ + m_manager->activate (this, 0, when); +} + +void DeadlineTimer::reset () +{ + m_manager->deactivate (this); +} diff --git a/Subtrees/beast/modules/beast_basics/events/beast_DeadlineTimer.h b/Subtrees/beast/modules/beast_basics/events/beast_DeadlineTimer.h new file mode 100644 index 0000000000..671ab27d06 --- /dev/null +++ b/Subtrees/beast/modules/beast_basics/events/beast_DeadlineTimer.h @@ -0,0 +1,121 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef BEAST_DEADLINETIMER_H_INCLUDED +#define BEAST_DEADLINETIMER_H_INCLUDED + +/** Provides periodic or one time notifications at a specified time interval. +*/ +class DeadlineTimer : public List ::Node +{ +public: + /** Listener for a deadline timer. + + The listener is called on an auxiliary thread. It is suggested + not to perform any time consuming operations during the call. + */ + // VFALCO TODO Allow construction with a specific ThreadWithCallQueue& + // on which to notify the listener. + class Listener + { + public: + virtual void onDeadlineTimer (DeadlineTimer&) { } + }; + +public: + /** Create a deadline timer with the specified listener attached. + */ + explicit DeadlineTimer (Listener* listener); + + ~DeadlineTimer (); + + /** Set the timer to go off once in the future. + + If the timer is already active, this will reset it. + + @note If the timer is already active, a notification may still + occur due to concurrency. + + @param secondsUntilDeadline The number of seconds until the timer + will send a notification. This must be + greater than zero. + */ + void setExpiration (double secondsUntilDeadline); + + /** Set the timer to go off repeatedly with the specified frequency. + + If the timer is already active, this will reset it. + + @note If the timer is already active, a notification may still + occur due to concurrency. + + @param secondsUntilDeadline The number of seconds until the timer + will send a notification. This must be + greater than zero. + */ + void setRecurringExpiration (double secondsUntilDeadline); + + /** Set the timer to go off at a specific time. + + If the timer is already active, this will reset it. + + @note If the timer is already active, a notification may still + occur due to concurrency. + + @note If the time is in the past, the timer will go off + immediately. + */ + void setExpirationTime (Time const& when); + + /** Reset the timer so that no more notifications are sent. + + It is okay to call this on an inactive timer. + + @note If the timer is already active, a notification may still + occur due to concurrency. + */ + void reset (); + + /** Equality comparison. + + Timers are equal if they have the same address. + */ + inline bool operator== (DeadlineTimer const& other) const + { + return this == &other; + } + + /** Inequality comparison. + */ + inline bool operator!= (DeadlineTimer const& other) const + { + return this != &other; + } + +private: + class Manager; + + Listener* const m_listener; + ReferenceCountedObjectPtr m_manager; + bool m_isActive; + Time m_notificationTime; + double m_secondsRecurring; // non zero if recurring +}; + +#endif diff --git a/Subtrees/beast/modules/beast_basics/events/beast_OncePerSecond.cpp b/Subtrees/beast/modules/beast_basics/events/beast_OncePerSecond.cpp index 8311b397fb..6b436ec51c 100644 --- a/Subtrees/beast/modules/beast_basics/events/beast_OncePerSecond.cpp +++ b/Subtrees/beast/modules/beast_basics/events/beast_OncePerSecond.cpp @@ -18,12 +18,12 @@ //============================================================================== class OncePerSecond::TimerSingleton - : public RefCountedSingleton + : public SharedSingleton , private InterruptibleThread::EntryPoint { private: TimerSingleton () - : RefCountedSingleton ( + : SharedSingleton ( SingletonLifetime::persistAfterCreation) , m_thread ("Once Per Second") { @@ -41,7 +41,7 @@ private: { for (;;) { - const bool interrupted = m_thread.wait (1000); + bool const interrupted = m_thread.wait (1000); if (interrupted) break; diff --git a/Subtrees/beast/modules/beast_basics/events/beast_OncePerSecond.h b/Subtrees/beast/modules/beast_basics/events/beast_OncePerSecond.h index 2c5075aa2e..24b47913db 100644 --- a/Subtrees/beast/modules/beast_basics/events/beast_OncePerSecond.h +++ b/Subtrees/beast/modules/beast_basics/events/beast_OncePerSecond.h @@ -20,8 +20,6 @@ #ifndef BEAST_ONCEPERSECOND_BEASTHEADER #define BEAST_ONCEPERSECOND_BEASTHEADER -#include "../containers/beast_List.h" - /*============================================================================*/ /** Provides a once per second notification. @@ -32,7 +30,7 @@ @ingroup beast_core */ -class OncePerSecond : Uncopyable +class BEAST_API OncePerSecond : Uncopyable { public: OncePerSecond (); diff --git a/Subtrees/beast/modules/beast_basics/functor/beast_Bind.h b/Subtrees/beast/modules/beast_basics/functor/beast_Bind.h deleted file mode 100644 index ed29b3d47c..0000000000 --- a/Subtrees/beast/modules/beast_basics/functor/beast_Bind.h +++ /dev/null @@ -1,96 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_BIND_BEASTHEADER -#define BEAST_BIND_BEASTHEADER - -/* Brings functional support into our namespace, based on environment. -*/ -#if BEAST_MSVC -// Visual Studio has these in std. -using std::ref; -using std::bind; -using std::function; -using std::placeholders::_1; -using std::placeholders::_2; - -#elif BEAST_IOS -#if BEAST_USE_BOOST -/* If boost is activated, use it. This works - around a bug with the iOS implementation of bind. -*/ -using boost::ref -using boost::bind; -using boost::function; -using ::_1; -using ::_2; -#else -#if _LIBCPP_VERSION // libc++ -using std::ref; -using std::bind; -using std::function; -using std::placeholders::_1; -using std::placeholders::_2; -#else // libstdc++ (GNU) -using std::tr1::ref; -using std::tr1::bind; -using std::tr1::function; -using std::tr1::placeholders::_1; -using std::tr1::placeholders::_2; -#endif -#endif - -#elif BEAST_MAC -#if _LIBCPP_VERSION // libc++ -using std::ref; -using std::bind; -using std::function; -using std::placeholders::_1; -using std::placeholders::_2; -#else // libstdc++ (GNU) -using std::tr1::ref; -using std::tr1::bind; -using std::tr1::function; -using std::tr1::placeholders::_1; -using std::tr1::placeholders::_2; -#endif - -#elif BEAST_LINUX -using std::tr1::bind; -using std::tr1::placeholders::_1; -using std::tr1::placeholders::_2; - -#else -#error Unknown platform in beast_Bind.h - -#endif - -/** Max number of arguments to bind, total. -*/ -#if BEAST_MSVC -# ifdef _VARIADIC_MAX -# define BEAST_VARIADIC_MAX _VARIADIC_MAX -# else -# define BEAST_VARIADIC_MAX 9 -# endif -#else -# define BEAST_VARIADIC_MAX 9 -#endif - -#endif diff --git a/Subtrees/beast/modules/beast_basics/math/beast_MurmurHash.h b/Subtrees/beast/modules/beast_basics/math/beast_MurmurHash.h index a406615b13..0f1e8c2385 100644 --- a/Subtrees/beast/modules/beast_basics/math/beast_MurmurHash.h +++ b/Subtrees/beast/modules/beast_basics/math/beast_MurmurHash.h @@ -22,7 +22,7 @@ // Original source code links in .cpp file -// This file depends on some Juce declarations and defines +// This file depends on some Beast declarations and defines namespace Murmur { @@ -31,7 +31,7 @@ extern void MurmurHash3_x86_32 (const void* key, int len, uint32 seed, void* ou extern void MurmurHash3_x86_128 (const void* key, int len, uint32 seed, void* out); extern void MurmurHash3_x64_128 (const void* key, int len, uint32 seed, void* out); -// Uses Juce to choose an appropriate routine +// Uses Beast to choose an appropriate routine // This handy template deduces which size hash is desired template @@ -44,15 +44,15 @@ inline void Hash (const void* key, int len, uint32 seed, HashType* out) break; #if BEAST_64BIT - case 128: MurmurHash3_x64_128 (key, len, seed, out); break; -#else +#else case 128: MurmurHash3_x86_128 (key, len, seed, out); break; + #endif default: diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStore.h b/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStore.h index 452053bfd8..058a986349 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStore.h +++ b/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStore.h @@ -20,19 +20,16 @@ #ifndef BEAST_FIFOFREESTORE_BEASTHEADER #define BEAST_FIFOFREESTORE_BEASTHEADER -#if BEAST_USE_BOOST -#include "beast_FifoFreeStoreWithTLS.h" - -#else -#include "beast_FifoFreeStoreWithoutTLS.h" - -#endif - /** Selected free store based on compilation settings. @ingroup beast_concurrent */ -#if BEAST_USE_BOOST +// VFALCO NOTE Disabled this because it seems that the TLS +// implementation has a leak. Although the other +// one also seems to have a leak. +// +//#if BEAST_BOOST_IS_AVAILABLE +#if 0 typedef FifoFreeStoreWithTLS FifoFreeStoreType; #else typedef FifoFreeStoreWithoutTLS FifoFreeStoreType; diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithTLS.cpp b/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithTLS.cpp index 51c93ae8f1..b868937eb3 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithTLS.cpp +++ b/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithTLS.cpp @@ -32,6 +32,8 @@ // affecting performance. // +#if BEAST_BOOST_IS_AVAILABLE + // This precedes every allocation // struct FifoFreeStoreWithTLS::Header @@ -192,3 +194,5 @@ void FifoFreeStoreWithTLS::deallocate (void* p) if (page->release ()) deletePage (page); } + +#endif diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithTLS.h b/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithTLS.h index 5382c3737f..4ecbe90b70 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithTLS.h +++ b/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithTLS.h @@ -20,7 +20,7 @@ #ifndef BEAST_FIFOFREESTOREWITHTLS_BEASTHEADER #define BEAST_FIFOFREESTOREWITHTLS_BEASTHEADER -#include "beast_GlobalPagedFreeStore.h" +#if BEAST_BOOST_IS_AVAILABLE /*============================================================================*/ /** @@ -39,7 +39,7 @@ @ingroup beast_concurrent */ -class FifoFreeStoreWithTLS +class BEAST_API FifoFreeStoreWithTLS { public: FifoFreeStoreWithTLS (); @@ -65,3 +65,5 @@ private: }; #endif + +#endif diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithoutTLS.h b/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithoutTLS.h index acfe2043f3..4b666a39cb 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithoutTLS.h +++ b/Subtrees/beast/modules/beast_basics/memory/beast_FifoFreeStoreWithoutTLS.h @@ -20,8 +20,6 @@ #ifndef BEAST_FIFOFREESTOREWITHOUTTLS_BEASTHEADER #define BEAST_FIFOFREESTOREWITHOUTTLS_BEASTHEADER -#include "beast_GlobalPagedFreeStore.h" - /*============================================================================*/ /** Lock-free FIFO memory allocator. @@ -40,7 +38,7 @@ @ingroup beast_concurrent */ -class FifoFreeStoreWithoutTLS +class BEAST_API FifoFreeStoreWithoutTLS { public: explicit FifoFreeStoreWithoutTLS (); diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_GlobalFifoFreeStore.h b/Subtrees/beast/modules/beast_basics/memory/beast_GlobalFifoFreeStore.h index 96b9184fe6..987a584cde 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_GlobalFifoFreeStore.h +++ b/Subtrees/beast/modules/beast_basics/memory/beast_GlobalFifoFreeStore.h @@ -20,8 +20,6 @@ #ifndef BEAST_GLOBALFIFOFREESTORE_BEASTHEADER #define BEAST_GLOBALFIFOFREESTORE_BEASTHEADER -#include "beast_FifoFreeStore.h" - /*============================================================================*/ /** A @ref FifoFreeStoreType singleton. @@ -29,7 +27,7 @@ @ingroup beast_concurrent */ template -class GlobalFifoFreeStore : public RefCountedSingleton > +class GlobalFifoFreeStore : public SharedSingleton > { public: inline void* allocate (size_t bytes) @@ -49,7 +47,7 @@ public: private: GlobalFifoFreeStore () - : RefCountedSingleton > + : SharedSingleton > (SingletonLifetime::persistAfterCreation) { } diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_GlobalPagedFreeStore.cpp b/Subtrees/beast/modules/beast_basics/memory/beast_GlobalPagedFreeStore.cpp index 1b3cd6d504..1074728c51 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_GlobalPagedFreeStore.cpp +++ b/Subtrees/beast/modules/beast_basics/memory/beast_GlobalPagedFreeStore.cpp @@ -27,7 +27,7 @@ static const size_t globalPageBytes = 8 * 1024; } GlobalPagedFreeStore::GlobalPagedFreeStore () - : RefCountedSingleton (SingletonLifetime::persistAfterCreation) + : SharedSingleton (SingletonLifetime::persistAfterCreation) , m_allocator (globalPageBytes) { } diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_GlobalPagedFreeStore.h b/Subtrees/beast/modules/beast_basics/memory/beast_GlobalPagedFreeStore.h index ff17214868..17c88af013 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_GlobalPagedFreeStore.h +++ b/Subtrees/beast/modules/beast_basics/memory/beast_GlobalPagedFreeStore.h @@ -20,16 +20,14 @@ #ifndef BEAST_GLOBALPAGEDFREESTORE_BEASTHEADER #define BEAST_GLOBALPAGEDFREESTORE_BEASTHEADER -#include "beast_PagedFreeStore.h" - /*============================================================================*/ /** A PagedFreeStore singleton. @ingroup beast_concurrent */ -class GlobalPagedFreeStore - : public RefCountedSingleton +class BEAST_API GlobalPagedFreeStore + : public SharedSingleton , LeakChecked { private: diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_PagedFreeStore.h b/Subtrees/beast/modules/beast_basics/memory/beast_PagedFreeStore.h index 078a383fd3..14b647cfe5 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_PagedFreeStore.h +++ b/Subtrees/beast/modules/beast_basics/memory/beast_PagedFreeStore.h @@ -29,7 +29,7 @@ @ingroup beast_concurrent */ -class PagedFreeStore : private OncePerSecond +class BEAST_API PagedFreeStore : private OncePerSecond { public: explicit PagedFreeStore (const size_t pageBytes); diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_CallQueue.h b/Subtrees/beast/modules/beast_basics/threads/beast_CallQueue.h index 4a59e121c7..a40cb6a2b1 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_CallQueue.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_CallQueue.h @@ -131,7 +131,7 @@ @ingroup beast_concurrent */ -class CallQueue +class BEAST_API CallQueue { public: //============================================================================ @@ -313,11 +313,11 @@ public: struct SharedState; // contains data shared between threads - ConcurrentState sharedState; + SharedData sharedState; void stateChanged () { - ConcurrentState ::ReadAccess state (sharedState); + SharedData ::ReadAccess state (sharedState); // (read state) } @@ -326,7 +326,7 @@ public: void changeState () { - ConcurrentState ::WriteAccess state (sharedState); + SharedData ::WriteAccess state (sharedState); // (read and write state) diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_ConcurrentObject.h b/Subtrees/beast/modules/beast_basics/threads/beast_ConcurrentObject.h index 4f2f531ec0..de0dc327ba 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_ConcurrentObject.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_ConcurrentObject.h @@ -41,7 +41,7 @@ @ingroup beast_concurrent */ -class ConcurrentObject : Uncopyable +class BEAST_API ConcurrentObject : Uncopyable { public: inline void incReferenceCount () noexcept diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_GlobalThreadGroup.h b/Subtrees/beast/modules/beast_basics/threads/beast_GlobalThreadGroup.h index 6940ebde51..38bf7dd013 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_GlobalThreadGroup.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_GlobalThreadGroup.h @@ -28,14 +28,14 @@ @ingroup beast_concurrent */ -class GlobalThreadGroup : public ThreadGroup, - public RefCountedSingleton +class BEAST_API GlobalThreadGroup : public ThreadGroup, + public SharedSingleton { private: - friend class RefCountedSingleton ; + friend class SharedSingleton ; GlobalThreadGroup () - : RefCountedSingleton ( + : SharedSingleton ( SingletonLifetime::persistAfterCreation) { } diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_InterruptibleThread.h b/Subtrees/beast/modules/beast_basics/threads/beast_InterruptibleThread.h index 2c56e9dc67..f81dfc213d 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_InterruptibleThread.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_InterruptibleThread.h @@ -20,9 +20,6 @@ #ifndef BEAST_INTERRUPTIBLETHREAD_BEASTHEADER #define BEAST_INTERRUPTIBLETHREAD_BEASTHEADER -#include "../diagnostic/beast_SafeBool.h" -#include "../functor/beast_Function.h" - //============================================================================== /** A thread with soft interruption support. @@ -36,7 +33,7 @@ @ingroup beast_core */ -class InterruptibleThread +class BEAST_API InterruptibleThread { public: /** InterruptibleThread entry point. diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_Listeners.h b/Subtrees/beast/modules/beast_basics/threads/beast_Listeners.h index 9236771890..70d798b40a 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_Listeners.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_Listeners.h @@ -137,7 +137,7 @@ void addListener (Listener* listener, CallQueue& callQueue) { // Acquire read access to the shared state. - ConcurrentState ::ReadAccess state (m_state); + SharedData ::ReadAccess state (m_state); // Add the listener. m_listeners.add (listener, callQueue); @@ -171,7 +171,7 @@ // Update shared state. { - ConcurrentState ::WriteAccess state (m_state); + SharedData ::WriteAccess state (m_state); m_state->outputLevel = newOutputLevel; } @@ -188,7 +188,7 @@ float outputLevel; }; - ConcurrentState m_state; + SharedData m_state; ManualCallQueue m_fifo; }; @@ -205,7 +205,7 @@ @class Listeners @ingroup beast_concurrent */ -class ListenersBase +class BEAST_API ListenersBase { public: struct ListenersStructureTag { }; diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_ParallelFor.h b/Subtrees/beast/modules/beast_basics/threads/beast_ParallelFor.h index 7cf6b45d5c..93e8dbf8d8 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_ParallelFor.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_ParallelFor.h @@ -52,7 +52,7 @@ @ingroup beast_concurrent */ -class ParallelFor : Uncopyable +class BEAST_API ParallelFor : Uncopyable { public: /** Create a parallel for loop. diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_ReadWriteMutex.h b/Subtrees/beast/modules/beast_basics/threads/beast_ReadWriteMutex.h index ff1f987e27..aea144a536 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_ReadWriteMutex.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_ReadWriteMutex.h @@ -47,7 +47,8 @@ @ingroup beast_concurrent */ -/*============================================================================*/ +//------------------------------------------------------------------------------ + /** Scoped read lock for ReadWriteMutex. @@ -72,7 +73,8 @@ private: LockType const& m_lock; }; -/*============================================================================*/ +//------------------------------------------------------------------------------ + /** Scoped write lock for ReadWriteMutex. @@ -97,7 +99,9 @@ private: LockType const& m_lock; }; -class ReadWriteMutex +//------------------------------------------------------------------------------ + +class BEAST_API ReadWriteMutex { public: /** Provides the type of scoped read lock to use with a ReadWriteMutex. */ diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_Semaphore.h b/Subtrees/beast/modules/beast_basics/threads/beast_Semaphore.h index 4d8ec2cebe..de78a0a1e4 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_Semaphore.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_Semaphore.h @@ -31,7 +31,7 @@ @ingroup beast_core */ -class Semaphore +class BEAST_API Semaphore { public: /** Create a semaphore with the specified number of resources. diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_SerialFor.h b/Subtrees/beast/modules/beast_basics/threads/beast_SerialFor.h index 3d736fe26b..5be1429af8 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_SerialFor.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_SerialFor.h @@ -31,7 +31,7 @@ @ingroup beast_core */ -class SerialFor : Uncopyable +class BEAST_API SerialFor : Uncopyable { public: /** Create a serial for loop. diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_ConcurrentState.h b/Subtrees/beast/modules/beast_basics/threads/beast_SharedData.h similarity index 81% rename from Subtrees/beast/modules/beast_basics/threads/beast_ConcurrentState.h rename to Subtrees/beast/modules/beast_basics/threads/beast_SharedData.h index 6a5157fe9d..1acf172826 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_ConcurrentState.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_SharedData.h @@ -17,8 +17,8 @@ */ //============================================================================== -#ifndef BEAST_CONCURRENTSTATE_BEASTHEADER -#define BEAST_CONCURRENTSTATE_BEASTHEADER +#ifndef BEAST_SHAREDDATA_H_INCLUDED +#define BEAST_SHAREDDATA_H_INCLUDED /*============================================================================*/ /** @@ -52,7 +52,7 @@ It also makes it easier to search for places in code which use unlocked access. - This code example demonstrates various forms of access to a ConcurrentState: + This code example demonstrates various forms of access to a SharedData: @code @@ -62,7 +62,7 @@ String value2; }; - typedef ConcurrentState SharedState; + typedef SharedData SharedState; SharedState sharedState; @@ -101,7 +101,7 @@ }; // Construct SharedData with one parameter - ConcurrentState sharedState (16); + SharedData sharedState (16); @endcode @@ -112,11 +112,9 @@ read access. Such an attempt will result in undefined behavior. Calling into unknown code while holding a lock can cause deadlock. See @ref CallQueue::queue(). - - @ingroup beast_concurrent */ template -class ConcurrentState : Uncopyable +class SharedData : Uncopyable { public: class ReadAccess; @@ -131,37 +129,37 @@ public: generated. */ /** @{ */ - ConcurrentState () { } + SharedData () { } template - explicit ConcurrentState (T1 t1) + explicit SharedData (T1 t1) : m_obj (t1) { } template - ConcurrentState (T1 t1, T2 t2) + SharedData (T1 t1, T2 t2) : m_obj (t1, t2) { } template - ConcurrentState (T1 t1, T2 t2, T3 t3) + SharedData (T1 t1, T2 t2, T3 t3) : m_obj (t1, t2, t3) { } template - ConcurrentState (T1 t1, T2 t2, T3 t3, T4 t4) + SharedData (T1 t1, T2 t2, T3 t3, T4 t4) : m_obj (t1, t2, t3, t4) { } template - ConcurrentState (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) + SharedData (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) : m_obj (t1, t2, t3, t4, t5) { } template - ConcurrentState (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6) + SharedData (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6) : m_obj (t1, t2, t3, t4, t5, t6) { } template - ConcurrentState (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7) : m_obj (t1, t2, t3, t4, t5, t6, t7) { } + SharedData (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7) : m_obj (t1, t2, t3, t4, t5, t6, t7) { } template - ConcurrentState (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7, T8 t8) + SharedData (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7, T8 t8) : m_obj (t1, t2, t3, t4, t5, t6, t7, t8) { } /** @} */ @@ -174,15 +172,15 @@ private: //------------------------------------------------------------------------------ -/** Unlocked access to a ConcurrentState. +/** Unlocked access to a SharedData. Use sparingly. */ template -class ConcurrentState ::UnlockedAccess : Uncopyable +class SharedData ::UnlockedAccess : Uncopyable { public: - explicit UnlockedAccess (ConcurrentState const& state) + explicit UnlockedAccess (SharedData const& state) : m_state (state) { } @@ -201,19 +199,19 @@ public: } private: - ConcurrentState const& m_state; + SharedData const& m_state; }; //------------------------------------------------------------------------------ -/** Read only access to a ConcurrentState */ +/** Read only access to a SharedData */ template -class ConcurrentState ::ReadAccess : Uncopyable +class SharedData ::ReadAccess : Uncopyable { public: - /** Create a ReadAccess from the specified ConcurrentState */ - explicit ReadAccess (ConcurrentState const volatile& state) - : m_state (const_cast (state)) + /** Create a ReadAccess from the specified SharedData */ + explicit ReadAccess (SharedData const volatile& state) + : m_state (const_cast (state)) , m_lock (m_state.m_mutex) { } @@ -237,18 +235,18 @@ public: } private: - ConcurrentState const& m_state; + SharedData const& m_state; ReadWriteMutexType::ScopedReadLockType m_lock; }; //------------------------------------------------------------------------------ -/** Read/write access to a ConcurrentState */ +/** Read/write access to a SharedData */ template -class ConcurrentState ::WriteAccess : Uncopyable +class SharedData ::WriteAccess : Uncopyable { public: - explicit WriteAccess (ConcurrentState& state) + explicit WriteAccess (SharedData& state) : m_state (state) , m_lock (m_state.m_mutex) { @@ -291,7 +289,7 @@ public: } private: - ConcurrentState& m_state; + SharedData& m_state; ReadWriteMutexType::ScopedWriteLockType m_lock; }; diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_SharedObject.h b/Subtrees/beast/modules/beast_basics/threads/beast_SharedObject.h index 7d9e80761b..65e398b1f9 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_SharedObject.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_SharedObject.h @@ -32,7 +32,7 @@ @ingroup beast_concurrent */ -class SharedObject : Uncopyable +class BEAST_API SharedObject : Uncopyable { public: /** Abstract SharedObject scope. diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_ThreadGroup.h b/Subtrees/beast/modules/beast_basics/threads/beast_ThreadGroup.h index b74a9a08a6..50720e2dc5 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_ThreadGroup.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_ThreadGroup.h @@ -28,7 +28,7 @@ @see ParallelFor */ -class ThreadGroup +class BEAST_API ThreadGroup { public: typedef FifoFreeStoreType AllocatorType; diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_ThreadWithCallQueue.cpp b/Subtrees/beast/modules/beast_basics/threads/beast_ThreadWithCallQueue.cpp index b0141aaeac..c946e8e82b 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_ThreadWithCallQueue.cpp +++ b/Subtrees/beast/modules/beast_basics/threads/beast_ThreadWithCallQueue.cpp @@ -32,6 +32,13 @@ ThreadWithCallQueue::~ThreadWithCallQueue () stop (true); } +ThreadWithCallQueue::EntryPoints* ThreadWithCallQueue::getDefaultEntryPoints () noexcept +{ + static EntryPoints entryPoints; + + return &entryPoints; +} + void ThreadWithCallQueue::start (EntryPoints* const entryPoints) { { diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_ThreadWithCallQueue.h b/Subtrees/beast/modules/beast_basics/threads/beast_ThreadWithCallQueue.h index 6f94b42a97..1db2021f76 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_ThreadWithCallQueue.h +++ b/Subtrees/beast/modules/beast_basics/threads/beast_ThreadWithCallQueue.h @@ -20,29 +20,28 @@ #ifndef BEAST_THREADWITHCALLQUEUE_BEASTHEADER #define BEAST_THREADWITHCALLQUEUE_BEASTHEADER -/*============================================================================*/ -/** - An InterruptibleThread with a CallQueue. +/** An InterruptibleThread with a CallQueue. - This combines an InterruptibleThread with a CallQueue, allowing functors to - be queued for asynchronous execution on the thread. + This combines an InterruptibleThread with a CallQueue, allowing functors to + be queued for asynchronous execution on the thread. - The thread runs an optional user-defined idle function, which must regularly - check for an interruption using the InterruptibleThread interface. When an - interruption is signaled, the idle function returns and the CallQueue is - synchronized. Then, the idle function is resumed. + The thread runs an optional user-defined idle function, which must regularly + check for an interruption using the InterruptibleThread interface. When an + interruption is signaled, the idle function returns and the CallQueue is + synchronized. Then, the idle function is resumed. - When the ThreadWithCallQueue first starts up, an optional user-defined - initialization function is executed on the thread. When the thread exits, - a user-defined exit function may be executed on the thread. + When the ThreadWithCallQueue first starts up, an optional user-defined + initialization function is executed on the thread. When the thread exits, + a user-defined exit function may be executed on the thread. - @see CallQueue + @see CallQueue - @ingroup beast_concurrent + @ingroup beast_concurrent */ -class ThreadWithCallQueue +class BEAST_API ThreadWithCallQueue : public CallQueue , private InterruptibleThread::EntryPoint + , LeakChecked { public: /** Entry points for a ThreadWithCallQueue. @@ -71,6 +70,12 @@ public: */ explicit ThreadWithCallQueue (String name); + /** Retrieve the default entry points. + + The default entry points do nothing. + */ + static EntryPoints* getDefaultEntryPoints () noexcept; + /** Destroy a ThreadWithCallQueue. If the thread is still running it is stopped. The destructor blocks @@ -78,9 +83,16 @@ public: */ ~ThreadWithCallQueue (); - /** Start the thread. + /** Start the thread, with optional entry points. + + If `entryPoints` is specified then the thread runs using those + entry points. If ommitted, the default entry simply do nothing. + This is useful for creating a thread whose sole activities are + performed through the call queue. + + @param entryPoints An optional pointer to @ref EntryPoints. */ - void start (EntryPoints* const entryPoints); + void start (EntryPoints* const entryPoints = getDefaultEntryPoints ()); /* Stop the thread. @@ -102,18 +114,17 @@ public: void stop (bool const wait); - /** - Determine if the thread needs interruption. + /** Determine if the thread needs interruption. - Should be called periodically by the idle function. If interruptionPoint - returns true or throws, it must not be called again until the idle function - returns and is re-entered. + Should be called periodically by the idle function. If interruptionPoint + returns true or throws, it must not be called again until the idle function + returns and is re-entered. - @invariant No previous calls to interruptionPoint() made after the idle - function entry point returned `true`. + @invariant No previous calls to interruptionPoint() made after the idle + function entry point returned `true`. - @return `false` if the idle function may continue, or `true` if the - idle function must return as soon as possible. + @return `false` if the idle function may continue, or `true` if the + idle function must return as soon as possible. */ bool interruptionPoint (); diff --git a/Subtrees/beast/modules/beast_core/beast_core.cpp b/Subtrees/beast/modules/beast_core/beast_core.cpp index 0514b14c54..15bb00a58c 100644 --- a/Subtrees/beast/modules/beast_core/beast_core.cpp +++ b/Subtrees/beast/modules/beast_core/beast_core.cpp @@ -102,8 +102,39 @@ #include #endif +//------------------------------------------------------------------------------ + +// If the MSVC debug heap headers were included, disable +// the macros during the juce include since they conflict. +#ifdef _CRTDBG_MAP_ALLOC +#pragma push_macro("calloc") +#pragma push_macro("free") +#pragma push_macro("malloc") +#pragma push_macro("realloc") +#pragma push_macro("_recalloc") +#pragma push_macro("_aligned_free") +#pragma push_macro("_aligned_malloc") +#pragma push_macro("_aligned_offset_malloc") +#pragma push_macro("_aligned_realloc") +#pragma push_macro("_aligned_recalloc") +#pragma push_macro("_aligned_offset_realloc") +#pragma push_macro("_aligned_offset_recalloc") +#pragma push_macro("_aligned_msize") +#undef calloc +#undef free +#undef malloc +#undef realloc +#undef _recalloc +#undef _aligned_free +#undef _aligned_malloc +#undef _aligned_offset_malloc +#undef _aligned_realloc +#undef _aligned_recalloc +#undef _aligned_offset_realloc +#undef _aligned_offset_recalloc +#undef _aligned_msize +#endif -//============================================================================== namespace beast { @@ -112,26 +143,39 @@ namespace beast #include "containers/beast_NamedValueSet.cpp" #include "containers/beast_PropertySet.cpp" #include "containers/beast_Variant.cpp" + +#include "diagnostic/beast_Debug.cpp" +#include "diagnostic/beast_Error.cpp" +#include "diagnostic/beast_FPUFlags.cpp" +#include "diagnostic/beast_LeakChecked.cpp" + #include "files/beast_DirectoryIterator.cpp" #include "files/beast_File.cpp" #include "files/beast_FileInputStream.cpp" #include "files/beast_FileOutputStream.cpp" #include "files/beast_FileSearchPath.cpp" #include "files/beast_TemporaryFile.cpp" + #include "json/beast_JSON.cpp" + #include "logging/beast_FileLogger.cpp" #include "logging/beast_Logger.cpp" + #include "maths/beast_BigInteger.cpp" #include "maths/beast_Expression.cpp" #include "maths/beast_Random.cpp" + #include "memory/beast_MemoryBlock.cpp" + #include "misc/beast_Result.cpp" #include "misc/beast_Uuid.cpp" + #include "network/beast_MACAddress.cpp" #include "network/beast_NamedPipe.cpp" #include "network/beast_Socket.cpp" #include "network/beast_URL.cpp" #include "network/beast_IPAddress.cpp" + #include "streams/beast_BufferedInputStream.cpp" #include "streams/beast_FileInputSource.cpp" #include "streams/beast_InputStream.cpp" @@ -139,8 +183,11 @@ namespace beast #include "streams/beast_MemoryOutputStream.cpp" #include "streams/beast_OutputStream.cpp" #include "streams/beast_SubregionStream.cpp" + #include "system/beast_SystemStats.cpp" + #include "text/beast_CharacterFunctions.cpp" + #include "text/beast_Identifier.cpp" #include "text/beast_LocalisedStrings.cpp" #include "text/beast_String.cpp" @@ -148,26 +195,38 @@ namespace beast #include "text/beast_StringPairArray.cpp" #include "text/beast_StringPool.cpp" #include "text/beast_TextDiff.cpp" + #include "threads/beast_ChildProcess.cpp" #include "threads/beast_ReadWriteLock.cpp" +#include "threads/beast_SpinDelay.cpp" #include "threads/beast_Thread.cpp" #include "threads/beast_ThreadPool.cpp" #include "threads/beast_TimeSliceThread.cpp" + #include "time/beast_PerformanceCounter.cpp" +#include "time/beast_PerformedAtExit.cpp" #include "time/beast_RelativeTime.cpp" #include "time/beast_Time.cpp" + #include "unit_tests/beast_UnitTest.cpp" + #include "xml/beast_XmlDocument.cpp" #include "xml/beast_XmlElement.cpp" + #include "zip/beast_GZIPDecompressorInputStream.cpp" #include "zip/beast_GZIPCompressorOutputStream.cpp" #include "zip/beast_ZipFile.cpp" -//============================================================================== #if BEAST_MAC || BEAST_IOS #include "native/beast_osx_ObjCHelpers.h" #endif +#if BEAST_WINDOWS +#include "native/beast_win32_FPUFlags.cpp" +#else +#include "native/beast_posix_FPUFlags.cpp" +#endif + #if BEAST_ANDROID #include "native/beast_android_JNIHelpers.h" #endif @@ -177,7 +236,6 @@ namespace beast #include "native/beast_posix_NamedPipe.cpp" #endif -//============================================================================== #if BEAST_MAC || BEAST_IOS #include "native/beast_mac_Files.mm" #include "native/beast_mac_Network.mm" @@ -185,7 +243,6 @@ namespace beast #include "native/beast_mac_SystemStats.mm" #include "native/beast_mac_Threads.mm" -//============================================================================== #elif BEAST_WINDOWS #include "native/beast_win32_ComSmartPtr.h" #include "native/beast_win32_Files.cpp" @@ -194,14 +251,18 @@ namespace beast #include "native/beast_win32_SystemStats.cpp" #include "native/beast_win32_Threads.cpp" -//============================================================================== #elif BEAST_LINUX #include "native/beast_linux_Files.cpp" #include "native/beast_linux_Network.cpp" #include "native/beast_linux_SystemStats.cpp" #include "native/beast_linux_Threads.cpp" -//============================================================================== +#elif BEAST_BSD +#include "native/beast_bsd_Files.cpp" +#include "native/beast_bsd_Network.cpp" +#include "native/beast_bsd_SystemStats.cpp" +#include "native/beast_bsd_Threads.cpp" + #elif BEAST_ANDROID #include "native/beast_android_Files.cpp" #include "native/beast_android_Misc.cpp" @@ -214,3 +275,37 @@ namespace beast #include "threads/beast_HighResolutionTimer.cpp" } + +#ifdef _CRTDBG_MAP_ALLOC +#pragma pop_macro("calloc") +#pragma pop_macro("free") +#pragma pop_macro("malloc") +#pragma pop_macro("realloc") +#pragma pop_macro("_recalloc") +#pragma pop_macro("_aligned_free") +#pragma pop_macro("_aligned_malloc") +#pragma pop_macro("_aligned_offset_malloc") +#pragma pop_macro("_aligned_realloc") +#pragma pop_macro("_aligned_recalloc") +#pragma pop_macro("_aligned_offset_realloc") +#pragma pop_macro("_aligned_offset_recalloc") +#pragma pop_macro("_aligned_msize") +#endif + +//------------------------------------------------------------------------------ + +#if BEAST_BOOST_IS_AVAILABLE +namespace boost { +namespace placeholders { +boost::arg<1> _1; +boost::arg<2> _2; +boost::arg<3> _3; +boost::arg<4> _4; +boost::arg<5> _5; +boost::arg<6> _6; +boost::arg<7> _7; +boost::arg<8> _8; +boost::arg<9> _9; +} +} +#endif diff --git a/Subtrees/beast/modules/beast_core/beast_core.h b/Subtrees/beast/modules/beast_core/beast_core.h index a37730cd61..8ce82f6c26 100644 --- a/Subtrees/beast/modules/beast_core/beast_core.h +++ b/Subtrees/beast/modules/beast_core/beast_core.h @@ -21,89 +21,52 @@ */ //============================================================================== -#ifndef BEAST_CORE_BEASTHEADER -#define BEAST_CORE_BEASTHEADER +#ifndef BEAST_CORE_H_INCLUDED +#define BEAST_CORE_H_INCLUDED -#ifndef BEAST_BEASTCONFIG_HEADER - /* If you fail to make sure that all your compile units are building Beast with the same set of - option flags, then there's a risk that different compile units will treat the classes as having - different memory layouts, leading to very nasty memory corruption errors when they all get - linked together. That's why it's best to always include the BeastConfig.h file before any beast headers. - */ - #ifdef _MSC_VER -#pragma message ("Have you included your BeastConfig.h file before including the Beast headers?") - #else - #warning "Have you included your BeastConfig.h file before including the Beast headers?" - #endif +//------------------------------------------------------------------------------ + +/* If you fail to make sure that all your compile units are building Beast with + the same set of option flags, then there's a risk that different compile + units will treat the classes as having different memory layouts, leading to + very nasty memory corruption errors when they all get linked together. + That's why it's best to always include the BeastConfig.h file before any + beast headers. +*/ +#ifndef BEAST_BEASTCONFIG_H_INCLUDED +# ifdef _MSC_VER +# pragma message ("Have you included your BeastConfig.h file before including the Beast headers?") +# else +# warning "Have you included your BeastConfig.h file before including the Beast headers?" +# endif #endif -//============================================================================== +//------------------------------------------------------------------------------ + #include "system/beast_TargetPlatform.h" -//============================================================================= -/** Config: BEAST_FORCE_DEBUG +// +// Apply sensible defaults for the configuration settings +// - Normally, BEAST_DEBUG is set to 1 or 0 based on compiler and project settings, - but if you define this value, you can override this to force it to be true or false. -*/ -#ifndef BEAST_FORCE_DEBUG - //#define BEAST_FORCE_DEBUG 0 -#endif - -//============================================================================= -/** Config: BEAST_LOG_ASSERTIONS - - If this flag is enabled, the the bassert and bassertfalse macros will always use Logger::writeToLog() - to write a message when an assertion happens. - - Enabling it will also leave this turned on in release builds. When it's disabled, - however, the bassert and bassertfalse macros will not be compiled in a - release build. - - @see bassert, bassertfalse, Logger -*/ #ifndef BEAST_LOG_ASSERTIONS - #if BEAST_ANDROID - #define BEAST_LOG_ASSERTIONS 1 - #else - #define BEAST_LOG_ASSERTIONS 0 - #endif +# if BEAST_ANDROID +# define BEAST_LOG_ASSERTIONS 1 +# else +# define BEAST_LOG_ASSERTIONS 0 +# endif #endif -//============================================================================= -/** Config: BEAST_CHECK_MEMORY_LEAKS - - Enables a memory-leak check for certain objects when the app terminates. See the LeakedObjectDetector - class and the BEAST_LEAK_DETECTOR macro for more details about enabling leak checking for specific classes. -*/ #if BEAST_DEBUG && ! defined (BEAST_CHECK_MEMORY_LEAKS) - #define BEAST_CHECK_MEMORY_LEAKS 1 +#define BEAST_CHECK_MEMORY_LEAKS 1 #endif -//============================================================================= -/** Config: BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES - - In a Visual C++ build, this can be used to stop the required system libs being - automatically added to the link stage. -*/ -#ifndef BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES - #define BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES 0 -#endif - -/* Config: BEAST_INCLUDE_ZLIB_CODE - This can be used to disable Beast's embedded 3rd-party zlib code. - You might need to tweak this if you're linking to an external zlib library in your app, - but for normal apps, this option should be left alone. - - If you disable this, you might also want to set a value for BEAST_ZLIB_INCLUDE_PATH, to - specify the path where your zlib headers live. -*/ #ifndef BEAST_INCLUDE_ZLIB_CODE - #define BEAST_INCLUDE_ZLIB_CODE 1 +#define BEAST_INCLUDE_ZLIB_CODE 1 #endif #ifndef BEAST_ZLIB_INCLUDE_PATH - #define BEAST_ZLIB_INCLUDE_PATH +#define BEAST_ZLIB_INCLUDE_PATH #endif /* Config: BEAST_CATCH_UNHANDLED_EXCEPTIONS @@ -111,330 +74,274 @@ to your BEASTApplication::unhandledException() callback. */ #ifndef BEAST_CATCH_UNHANDLED_EXCEPTIONS - //#define BEAST_CATCH_UNHANDLED_EXCEPTIONS 1 +//#define BEAST_CATCH_UNHANDLED_EXCEPTIONS 1 #endif -//============================================================================= -//============================================================================= -#if BEAST_MSVC - #pragma warning (disable: 4251) // (DLL build warning, must be disabled before pushing the warning state) - #pragma warning (push) - #pragma warning (disable: 4786) // (long class name warning) - #ifdef __INTEL_COMPILER - #pragma warning (disable: 1125) - #endif +#ifndef BEAST_BOOST_IS_AVAILABLE +#define BEAST_BOOST_IS_AVAILABLE 0 #endif +//------------------------------------------------------------------------------ +// +// This is a hack to fix boost's goofy placeholders +// + +#if BEAST_BOOST_IS_AVAILABLE +#ifdef BOOST_BIND_PLACEHOLDERS_HPP_INCLUDED +#error must not be included before this file +#endif +// Prevent from being included +#define BOOST_BIND_PLACEHOLDERS_HPP_INCLUDED +#include +#include +// This based on +namespace boost { +namespace placeholders { +extern boost::arg<1> _1; +extern boost::arg<2> _2; +extern boost::arg<3> _3; +extern boost::arg<4> _4; +extern boost::arg<5> _5; +extern boost::arg<6> _6; +extern boost::arg<7> _7; +extern boost::arg<8> _8; +extern boost::arg<9> _9; +} +using namespace placeholders; +} +#endif + +//------------------------------------------------------------------------------ +// +// Choose a source of bind, placeholders, and function +// + +#if !BEAST_BIND_USES_STD && !BEAST_BIND_USES_TR1 && !BEAST_BIND_USES_BOOST +# if BEAST_MSVC +# define BEAST_BIND_USES_STD 1 +# elif BEAST_IOS || BEAST_MAC +# include // detect version of std::lib +# if BEAST_IOS && BEAST_BOOST_IS_AVAILABLE // Work-around for iOS bugs with bind. +# define BEAST_BIND_USES_BOOST 1 +# elif _LIBCPP_VERSION // libc++ +# define BEAST_BIND_USES_STD 1 +# else // libstdc++ (GNU) +# define BEAST_BIND_USES_TR1 1 +# endif +# elif BEAST_LINUX || BEAST_BSD +# define BEAST_BIND_USES_TR1 1 +# else +# define BEAST_BIND_USES_STD 1 +# endif +#endif + +#if BEAST_BIND_USES_STD +# include +#elif BEAST_BIND_USES_TR1 +# include +#elif BEAST_BIND_USES_BOOST +# include +# include +#endif + +//------------------------------------------------------------------------------ + #include "system/beast_StandardHeader.h" +#if BEAST_MSVC +# pragma warning (disable: 4251) // (DLL build warning, must be disabled before pushing the warning state) +# pragma warning (push) +# pragma warning (disable: 4786) // (long class name warning) +# ifdef __INTEL_COMPILER +# pragma warning (disable: 1125) +# endif +#endif + +// If the MSVC debug heap headers were included, disable +// the macros during the juce include since they conflict. +#ifdef _CRTDBG_MAP_ALLOC +#pragma push_macro("calloc") +#pragma push_macro("free") +#pragma push_macro("malloc") +#pragma push_macro("realloc") +#pragma push_macro("_recalloc") +#pragma push_macro("_aligned_free") +#pragma push_macro("_aligned_malloc") +#pragma push_macro("_aligned_offset_malloc") +#pragma push_macro("_aligned_realloc") +#pragma push_macro("_aligned_recalloc") +#pragma push_macro("_aligned_offset_realloc") +#pragma push_macro("_aligned_offset_recalloc") +#pragma push_macro("_aligned_msize") +#undef calloc +#undef free +#undef malloc +#undef realloc +#undef _recalloc +#undef _aligned_free +#undef _aligned_malloc +#undef _aligned_offset_malloc +#undef _aligned_realloc +#undef _aligned_recalloc +#undef _aligned_offset_realloc +#undef _aligned_offset_recalloc +#undef _aligned_msize +#endif + namespace beast { -// START_AUTOINCLUDE containers, files, json, logging, maths, memory, misc, network, -// streams, system, text, threads, time, unit_tests, xml, zip -#ifndef BEAST_ABSTRACTFIFO_BEASTHEADER - #include "containers/beast_AbstractFifo.h" -#endif -#ifndef BEAST_ARRAY_BEASTHEADER - #include "containers/beast_Array.h" -#endif -#ifndef BEAST_ARRAYALLOCATIONBASE_BEASTHEADER - #include "containers/beast_ArrayAllocationBase.h" -#endif -#ifndef BEAST_DYNAMICOBJECT_BEASTHEADER - #include "containers/beast_DynamicObject.h" -#endif -#ifndef BEAST_ELEMENTCOMPARATOR_BEASTHEADER - #include "containers/beast_ElementComparator.h" -#endif -#ifndef BEAST_HASHMAP_BEASTHEADER - #include "containers/beast_HashMap.h" -#endif -#ifndef BEAST_LINKEDLISTPOINTER_BEASTHEADER - #include "containers/beast_LinkedListPointer.h" -#endif -#ifndef BEAST_NAMEDVALUESET_BEASTHEADER - #include "containers/beast_NamedValueSet.h" -#endif -#ifndef BEAST_OWNEDARRAY_BEASTHEADER - #include "containers/beast_OwnedArray.h" -#endif -#ifndef BEAST_PROPERTYSET_BEASTHEADER - #include "containers/beast_PropertySet.h" -#endif -#ifndef BEAST_REFERENCECOUNTEDARRAY_BEASTHEADER - #include "containers/beast_ReferenceCountedArray.h" -#endif -#ifndef BEAST_SCOPEDVALUESETTER_BEASTHEADER - #include "containers/beast_ScopedValueSetter.h" -#endif -#ifndef BEAST_SORTEDSET_BEASTHEADER - #include "containers/beast_SortedSet.h" -#endif -#ifndef BEAST_SPARSESET_BEASTHEADER - #include "containers/beast_SparseSet.h" -#endif -#ifndef BEAST_VARIANT_BEASTHEADER - #include "containers/beast_Variant.h" -#endif -#ifndef BEAST_DIRECTORYITERATOR_BEASTHEADER - #include "files/beast_DirectoryIterator.h" -#endif -#ifndef BEAST_FILE_BEASTHEADER - #include "files/beast_File.h" -#endif -#ifndef BEAST_FILEINPUTSTREAM_BEASTHEADER - #include "files/beast_FileInputStream.h" -#endif -#ifndef BEAST_FILEOUTPUTSTREAM_BEASTHEADER - #include "files/beast_FileOutputStream.h" -#endif -#ifndef BEAST_FILESEARCHPATH_BEASTHEADER - #include "files/beast_FileSearchPath.h" -#endif -#ifndef BEAST_MEMORYMAPPEDFILE_BEASTHEADER - #include "files/beast_MemoryMappedFile.h" -#endif -#ifndef BEAST_TEMPORARYFILE_BEASTHEADER - #include "files/beast_TemporaryFile.h" -#endif -#ifndef BEAST_JSON_BEASTHEADER - #include "json/beast_JSON.h" -#endif -#ifndef BEAST_FILELOGGER_BEASTHEADER - #include "logging/beast_FileLogger.h" -#endif -#ifndef BEAST_LOGGER_BEASTHEADER - #include "logging/beast_Logger.h" -#endif -#ifndef BEAST_BIGINTEGER_BEASTHEADER - #include "maths/beast_BigInteger.h" -#endif -#ifndef BEAST_EXPRESSION_BEASTHEADER - #include "maths/beast_Expression.h" -#endif -#ifndef BEAST_MATHSFUNCTIONS_BEASTHEADER - #include "maths/beast_MathsFunctions.h" -#endif -#ifndef BEAST_RANDOM_BEASTHEADER - #include "maths/beast_Random.h" -#endif -#ifndef BEAST_RANGE_BEASTHEADER - #include "maths/beast_Range.h" -#endif -#ifndef BEAST_ATOMIC_BEASTHEADER - #include "memory/beast_Atomic.h" -#endif -#ifndef BEAST_BYTEORDER_BEASTHEADER - #include "memory/beast_ByteOrder.h" -#endif -#ifndef BEAST_HEAPBLOCK_BEASTHEADER - #include "memory/beast_HeapBlock.h" -#endif -#ifndef BEAST_LEAKEDOBJECTDETECTOR_BEASTHEADER - #include "memory/beast_LeakedObjectDetector.h" -#endif -#ifndef BEAST_MEMORY_BEASTHEADER - #include "memory/beast_Memory.h" -#endif -#ifndef BEAST_MEMORYBLOCK_BEASTHEADER - #include "memory/beast_MemoryBlock.h" -#endif -#ifndef BEAST_OPTIONALSCOPEDPOINTER_BEASTHEADER - #include "memory/beast_OptionalScopedPointer.h" -#endif -#ifndef BEAST_REFERENCECOUNTEDOBJECT_BEASTHEADER - #include "memory/beast_ReferenceCountedObject.h" -#endif -#ifndef BEAST_SCOPEDPOINTER_BEASTHEADER - #include "memory/beast_ScopedPointer.h" -#endif -#ifndef BEAST_SINGLETON_BEASTHEADER - #include "memory/beast_Singleton.h" -#endif -#ifndef BEAST_WEAKREFERENCE_BEASTHEADER - #include "memory/beast_WeakReference.h" -#endif -#ifndef BEAST_RESULT_BEASTHEADER - #include "misc/beast_Result.h" -#endif -#ifndef BEAST_UUID_BEASTHEADER - #include "misc/beast_Uuid.h" -#endif -#ifndef BEAST_WINDOWSREGISTRY_BEASTHEADER - #include "misc/beast_WindowsRegistry.h" -#endif -#ifndef BEAST_IPADDRESS_BEASTHEADER - #include "network/beast_IPAddress.h" -#endif -#ifndef BEAST_MACADDRESS_BEASTHEADER - #include "network/beast_MACAddress.h" -#endif -#ifndef BEAST_NAMEDPIPE_BEASTHEADER - #include "network/beast_NamedPipe.h" -#endif -#ifndef BEAST_SOCKET_BEASTHEADER - #include "network/beast_Socket.h" -#endif -#ifndef BEAST_URL_BEASTHEADER - #include "network/beast_URL.h" -#endif -#ifndef BEAST_BUFFEREDINPUTSTREAM_BEASTHEADER - #include "streams/beast_BufferedInputStream.h" -#endif -#ifndef BEAST_FILEINPUTSOURCE_BEASTHEADER - #include "streams/beast_FileInputSource.h" -#endif -#ifndef BEAST_INPUTSOURCE_BEASTHEADER - #include "streams/beast_InputSource.h" -#endif -#ifndef BEAST_INPUTSTREAM_BEASTHEADER - #include "streams/beast_InputStream.h" -#endif -#ifndef BEAST_MEMORYINPUTSTREAM_BEASTHEADER - #include "streams/beast_MemoryInputStream.h" -#endif -#ifndef BEAST_MEMORYOUTPUTSTREAM_BEASTHEADER - #include "streams/beast_MemoryOutputStream.h" -#endif -#ifndef BEAST_OUTPUTSTREAM_BEASTHEADER - #include "streams/beast_OutputStream.h" -#endif -#ifndef BEAST_SUBREGIONSTREAM_BEASTHEADER - #include "streams/beast_SubregionStream.h" -#endif -#ifndef BEAST_PLATFORMDEFS_BEASTHEADER - #include "system/beast_PlatformDefs.h" -#endif -#ifndef BEAST_STANDARDHEADER_BEASTHEADER - #include "system/beast_StandardHeader.h" -#endif -#ifndef BEAST_SYSTEMSTATS_BEASTHEADER - #include "system/beast_SystemStats.h" -#endif -#ifndef BEAST_TARGETPLATFORM_BEASTHEADER - #include "system/beast_TargetPlatform.h" -#endif -#ifndef BEAST_CHARACTERFUNCTIONS_BEASTHEADER - #include "text/beast_CharacterFunctions.h" -#endif -#ifndef BEAST_CHARPOINTER_ASCII_BEASTHEADER - #include "text/beast_CharPointer_ASCII.h" -#endif -#ifndef BEAST_CHARPOINTER_UTF16_BEASTHEADER - #include "text/beast_CharPointer_UTF16.h" -#endif -#ifndef BEAST_CHARPOINTER_UTF32_BEASTHEADER - #include "text/beast_CharPointer_UTF32.h" -#endif -#ifndef BEAST_CHARPOINTER_UTF8_BEASTHEADER - #include "text/beast_CharPointer_UTF8.h" -#endif -#ifndef BEAST_IDENTIFIER_BEASTHEADER - #include "text/beast_Identifier.h" -#endif -#ifndef BEAST_LOCALISEDSTRINGS_BEASTHEADER - #include "text/beast_LocalisedStrings.h" -#endif -#ifndef BEAST_NEWLINE_BEASTHEADER - #include "text/beast_NewLine.h" -#endif -#ifndef BEAST_STRING_BEASTHEADER - #include "text/beast_String.h" -#endif -#ifndef BEAST_STRINGARRAY_BEASTHEADER - #include "text/beast_StringArray.h" -#endif -#ifndef BEAST_STRINGPAIRARRAY_BEASTHEADER - #include "text/beast_StringPairArray.h" -#endif -#ifndef BEAST_STRINGPOOL_BEASTHEADER - #include "text/beast_StringPool.h" -#endif -#ifndef BEAST_TEXTDIFF_BEASTHEADER - #include "text/beast_TextDiff.h" -#endif -#ifndef BEAST_CHILDPROCESS_BEASTHEADER - #include "threads/beast_ChildProcess.h" -#endif -#ifndef BEAST_CRITICALSECTION_BEASTHEADER - #include "threads/beast_CriticalSection.h" -#endif -#ifndef BEAST_DYNAMICLIBRARY_BEASTHEADER - #include "threads/beast_DynamicLibrary.h" -#endif -#ifndef BEAST_HIGHRESOLUTIONTIMER_BEASTHEADER - #include "threads/beast_HighResolutionTimer.h" -#endif -#ifndef BEAST_INTERPROCESSLOCK_BEASTHEADER - #include "threads/beast_InterProcessLock.h" -#endif -#ifndef BEAST_PROCESS_BEASTHEADER - #include "threads/beast_Process.h" -#endif -#ifndef BEAST_READWRITELOCK_BEASTHEADER - #include "threads/beast_ReadWriteLock.h" -#endif -#ifndef BEAST_SCOPEDLOCK_BEASTHEADER - #include "threads/beast_ScopedLock.h" -#endif -#ifndef BEAST_SCOPEDREADLOCK_BEASTHEADER - #include "threads/beast_ScopedReadLock.h" -#endif -#ifndef BEAST_SCOPEDWRITELOCK_BEASTHEADER - #include "threads/beast_ScopedWriteLock.h" -#endif -#ifndef BEAST_SPINLOCK_BEASTHEADER - #include "threads/beast_SpinLock.h" -#endif -#ifndef BEAST_THREAD_BEASTHEADER - #include "threads/beast_Thread.h" -#endif -#ifndef BEAST_THREADLOCALVALUE_BEASTHEADER - #include "threads/beast_ThreadLocalValue.h" -#endif -#ifndef BEAST_THREADPOOL_BEASTHEADER - #include "threads/beast_ThreadPool.h" -#endif -#ifndef BEAST_TIMESLICETHREAD_BEASTHEADER - #include "threads/beast_TimeSliceThread.h" -#endif -#ifndef BEAST_WAITABLEEVENT_BEASTHEADER - #include "threads/beast_WaitableEvent.h" -#endif -#ifndef BEAST_PERFORMANCECOUNTER_BEASTHEADER - #include "time/beast_PerformanceCounter.h" -#endif -#ifndef BEAST_RELATIVETIME_BEASTHEADER - #include "time/beast_RelativeTime.h" -#endif -#ifndef BEAST_TIME_BEASTHEADER - #include "time/beast_Time.h" -#endif -#ifndef BEAST_UNITTEST_BEASTHEADER - #include "unit_tests/beast_UnitTest.h" -#endif -#ifndef BEAST_XMLDOCUMENT_BEASTHEADER - #include "xml/beast_XmlDocument.h" -#endif -#ifndef BEAST_XMLELEMENT_BEASTHEADER - #include "xml/beast_XmlElement.h" -#endif -#ifndef BEAST_GZIPCOMPRESSOROUTPUTSTREAM_BEASTHEADER - #include "zip/beast_GZIPCompressorOutputStream.h" -#endif -#ifndef BEAST_GZIPDECOMPRESSORINPUTSTREAM_BEASTHEADER - #include "zip/beast_GZIPDecompressorInputStream.h" -#endif -#ifndef BEAST_ZIPFILE_BEASTHEADER - #include "zip/beast_ZipFile.h" -#endif -// END_AUTOINCLUDE +// Order matters, since headers don't have their own #include lines. +// Add new includes to the bottom. + +#include "memory/beast_Uncopyable.h" +#include "maths/beast_MathsFunctions.h" +#include "memory/beast_Atomic.h" +#include "memory/beast_AtomicCounter.h" +#include "memory/beast_AtomicFlag.h" +#include "memory/beast_AtomicPointer.h" +#include "memory/beast_AtomicState.h" +#include "containers/beast_LockFreeStack.h" +#include "threads/beast_SpinDelay.h" +#include "memory/beast_StaticObject.h" +#include "time/beast_PerformedAtExit.h" +#include "diagnostic/beast_LeakChecked.h" +#include "memory/beast_Memory.h" +#include "memory/beast_ByteOrder.h" +#include "logging/beast_Logger.h" +#include "threads/beast_Thread.h" +#include "diagnostic/beast_Debug.h" +#include "diagnostic/beast_SafeBool.h" +#include "diagnostic/beast_Error.h" +#include "diagnostic/beast_FPUFlags.h" +#include "diagnostic/beast_Throw.h" +#include "containers/beast_AbstractFifo.h" +#include "containers/beast_Array.h" +#include "containers/beast_ArrayAllocationBase.h" +#include "containers/beast_DynamicObject.h" +#include "containers/beast_ElementComparator.h" +#include "containers/beast_HashMap.h" +#include "containers/beast_List.h" +#include "containers/beast_LinkedListPointer.h" +#include "containers/beast_LockFreeQueue.h" +#include "containers/beast_NamedValueSet.h" +#include "containers/beast_OwnedArray.h" +#include "containers/beast_PropertySet.h" +#include "containers/beast_ReferenceCountedArray.h" +#include "containers/beast_ScopedValueSetter.h" +#include "containers/beast_SharedTable.h" +#include "containers/beast_SortedLookupTable.h" +#include "containers/beast_SortedSet.h" +#include "containers/beast_SparseSet.h" +#include "containers/beast_Variant.h" +#include "files/beast_DirectoryIterator.h" +#include "files/beast_File.h" +#include "files/beast_FileInputStream.h" +#include "files/beast_FileOutputStream.h" +#include "files/beast_FileSearchPath.h" +#include "files/beast_MemoryMappedFile.h" +#include "files/beast_TemporaryFile.h" +#include "json/beast_JSON.h" +#include "logging/beast_FileLogger.h" +#include "logging/beast_Logger.h" +#include "maths/beast_BigInteger.h" +#include "maths/beast_Expression.h" +#include "maths/beast_Interval.h" +#include "maths/beast_MathsFunctions.h" +#include "maths/beast_Random.h" +#include "maths/beast_Range.h" +#include "memory/beast_ByteOrder.h" +#include "memory/beast_HeapBlock.h" +#include "memory/beast_Memory.h" +#include "memory/beast_MemoryBlock.h" +#include "memory/beast_OptionalScopedPointer.h" +#include "memory/beast_ReferenceCountedObject.h" +#include "memory/beast_ScopedPointer.h" +#include "threads/beast_SpinLock.h" +#include "memory/beast_SharedSingleton.h" +#include "memory/beast_WeakReference.h" +#include "memory/beast_MemoryAlignment.h" +#include "memory/beast_CacheLine.h" +#include "misc/beast_Result.h" +#include "misc/beast_Uuid.h" +#include "misc/beast_WindowsRegistry.h" +#include "network/beast_IPAddress.h" +#include "network/beast_MACAddress.h" +#include "network/beast_NamedPipe.h" +#include "network/beast_Socket.h" +#include "network/beast_URL.h" +#include "streams/beast_BufferedInputStream.h" +#include "streams/beast_FileInputSource.h" +#include "streams/beast_InputSource.h" +#include "streams/beast_InputStream.h" +#include "streams/beast_MemoryInputStream.h" +#include "streams/beast_MemoryOutputStream.h" +#include "streams/beast_OutputStream.h" +#include "streams/beast_SubregionStream.h" +#include "system/beast_Functional.h" +#include "system/beast_PlatformDefs.h" +#include "system/beast_StandardHeader.h" +#include "system/beast_SystemStats.h" +#include "system/beast_TargetPlatform.h" +#include "text/beast_CharacterFunctions.h" +#include "text/beast_CharPointer_ASCII.h" +#include "text/beast_CharPointer_UTF16.h" +#include "text/beast_CharPointer_UTF32.h" +#include "text/beast_CharPointer_UTF8.h" +#include "text/beast_Identifier.h" +#include "text/beast_LocalisedStrings.h" +#include "text/beast_NewLine.h" +#include "text/beast_String.h" +#include "text/beast_StringArray.h" +#include "text/beast_StringPairArray.h" +#include "text/beast_StringPool.h" +#include "text/beast_TextDiff.h" +#include "threads/beast_ChildProcess.h" +#include "threads/beast_CriticalSection.h" +#include "threads/beast_DynamicLibrary.h" +#include "threads/beast_HighResolutionTimer.h" +#include "threads/beast_InterProcessLock.h" +#include "threads/beast_Process.h" +#include "threads/beast_ReadWriteLock.h" +#include "threads/beast_ScopedLock.h" +#include "threads/beast_ScopedReadLock.h" +#include "threads/beast_ScopedWriteLock.h" +#include "threads/beast_ThreadLocalValue.h" +#include "threads/beast_ThreadPool.h" +#include "threads/beast_TimeSliceThread.h" +#include "threads/beast_WaitableEvent.h" +#include "time/beast_PerformanceCounter.h" +#include "time/beast_RelativeTime.h" +#include "time/beast_Time.h" +#include "unit_tests/beast_UnitTest.h" +#include "xml/beast_XmlDocument.h" +#include "xml/beast_XmlElement.h" +#include "zip/beast_GZIPCompressorOutputStream.h" +#include "zip/beast_GZIPDecompressorInputStream.h" +#include "zip/beast_ZipFile.h" } -#if BEAST_MSVC - #pragma warning (pop) +#ifdef _CRTDBG_MAP_ALLOC +#pragma pop_macro("_aligned_msize") +#pragma pop_macro("_aligned_offset_recalloc") +#pragma pop_macro("_aligned_offset_realloc") +#pragma pop_macro("_aligned_recalloc") +#pragma pop_macro("_aligned_realloc") +#pragma pop_macro("_aligned_offset_malloc") +#pragma pop_macro("_aligned_malloc") +#pragma pop_macro("_aligned_free") +#pragma pop_macro("_recalloc") +#pragma pop_macro("realloc") +#pragma pop_macro("malloc") +#pragma pop_macro("free") +#pragma pop_macro("calloc") #endif -#endif // BEAST_CORE_BEASTHEADER +#if BEAST_MSVC +#pragma warning (pop) +#endif + +//------------------------------------------------------------------------------ + +#endif diff --git a/Subtrees/beast/modules/beast_core/containers/beast_AbstractFifo.h b/Subtrees/beast/modules/beast_core/containers/beast_AbstractFifo.h index 4c5e6856c8..f0fe0e571e 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_AbstractFifo.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_AbstractFifo.h @@ -24,21 +24,19 @@ #ifndef BEAST_ABSTRACTFIFO_BEASTHEADER #define BEAST_ABSTRACTFIFO_BEASTHEADER -#include "../memory/beast_Atomic.h" - - //============================================================================== /** Encapsulates the logic required to implement a lock-free FIFO. - This class handles the logic needed when building a single-reader, single-writer FIFO. + This class handles the logic needed when building a single-reader, + single-writer FIFO. - It doesn't actually hold any data itself, but your FIFO class can use one of these to manage - its position and status when reading or writing to it. + It doesn't actually hold any data itself, but your FIFO class can use one of + these to manage its position and status when reading or writing to it. - To use it, you can call prepareToWrite() to determine the position within your own buffer that - an incoming block of data should be stored, and prepareToRead() to find out when the next - outgoing block should be read from. + To use it, you can call prepareToWrite() to determine the position within + your own buffer that an incoming block of data should be stored, and + prepareToRead() to find out when the next outgoing block should be read from. e.g. @code @@ -83,7 +81,7 @@ }; @endcode */ -class BEAST_API AbstractFifo +class BEAST_API AbstractFifo : LeakChecked , Uncopyable { public: //============================================================================== @@ -204,14 +202,11 @@ public: */ void finishedRead (int numRead) noexcept; - private: //============================================================================== int bufferSize; Atomic validStart, validEnd; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (AbstractFifo) }; +#endif -#endif // BEAST_ABSTRACTFIFO_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/containers/beast_Array.h b/Subtrees/beast/modules/beast_core/containers/beast_Array.h index 1cec6d7fdd..c3f5a0c7e1 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_Array.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_Array.h @@ -230,8 +230,14 @@ public: ElementType operator[] (const int index) const { const ScopedLockType lock (getLock()); - return isPositiveAndBelow (index, numUsed) ? data.elements [index] - : ElementType(); + + if (isPositiveAndBelow (index, numUsed)) + { + bassert (data.elements != nullptr); + return data.elements [index]; + } + + return ElementType(); } /** Returns one of the elements in the array, without checking the index passed in. @@ -246,7 +252,7 @@ public: inline ElementType getUnchecked (const int index) const { const ScopedLockType lock (getLock()); - bassert (isPositiveAndBelow (index, numUsed)); + bassert (isPositiveAndBelow (index, numUsed) && data.elements != nullptr); return data.elements [index]; } @@ -262,7 +268,7 @@ public: inline ElementType& getReference (const int index) const noexcept { const ScopedLockType lock (getLock()); - bassert (isPositiveAndBelow (index, numUsed)); + bassert (isPositiveAndBelow (index, numUsed) && data.elements != nullptr); return data.elements [index]; } @@ -383,6 +389,7 @@ public: { const ScopedLockType lock (getLock()); data.ensureAllocatedSize (numUsed + 1); + bassert (data.elements != nullptr); if (isPositiveAndBelow (indexToInsertAt, numUsed)) { @@ -716,6 +723,7 @@ public: if (isPositiveAndBelow (indexToRemove, numUsed)) { + bassert (data.elements != nullptr); ElementType removed (data.elements[indexToRemove]); removeInternal (indexToRemove); return removed; @@ -1043,5 +1051,4 @@ private: } }; - #endif // BEAST_ARRAY_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/containers/beast_ArrayAllocationBase.h b/Subtrees/beast/modules/beast_core/containers/beast_ArrayAllocationBase.h index 479fb9091f..4fb0d5767f 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_ArrayAllocationBase.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_ArrayAllocationBase.h @@ -40,7 +40,8 @@ @see Array, OwnedArray, ReferenceCountedArray */ template -class ArrayAllocationBase : public TypeOfCriticalSectionToUse +class ArrayAllocationBase + : public TypeOfCriticalSectionToUse { public: //============================================================================== @@ -103,6 +104,8 @@ public: { if (minNumElements > numAllocated) setAllocatedSize ((minNumElements + minNumElements / 2 + 8) & ~7); + + bassert (numAllocated <= 0 || elements != nullptr); } /** Minimises the amount of storage allocated so that it's no more than @@ -124,10 +127,6 @@ public: //============================================================================== HeapBlock elements; int numAllocated; - -private: - BEAST_DECLARE_NON_COPYABLE (ArrayAllocationBase) }; - #endif // BEAST_ARRAYALLOCATIONBASE_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/containers/beast_DynamicObject.h b/Subtrees/beast/modules/beast_core/containers/beast_DynamicObject.h index 08ce6d21b7..66901d99bd 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_DynamicObject.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_DynamicObject.h @@ -39,7 +39,9 @@ by subclassing hasMethod() and invokeMethod(), you can give your object methods. */ -class BEAST_API DynamicObject : public ReferenceCountedObject +class BEAST_API DynamicObject + : public ReferenceCountedObject + , LeakChecked { public: //============================================================================== @@ -112,8 +114,6 @@ public: private: //============================================================================== NamedValueSet properties; - - BEAST_LEAK_DETECTOR (DynamicObject) }; diff --git a/Subtrees/beast/modules/beast_core/containers/beast_HashMap.h b/Subtrees/beast/modules/beast_core/containers/beast_HashMap.h index d488020bcc..cee0dfd6b2 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_HashMap.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_HashMap.h @@ -95,6 +95,11 @@ template class HashMap + : Uncopyable + , LeakChecked > { private: typedef PARAMETER_TYPE (KeyType) KeyTypeParameter; @@ -333,7 +338,7 @@ public: private: //============================================================================== - class HashEntry + class HashEntry : Uncopyable { public: HashEntry (KeyTypeParameter k, ValueTypeParameter val, HashEntry* const next) @@ -343,8 +348,6 @@ private: const KeyType key; ValueType value; HashEntry* nextEntry; - - BEAST_DECLARE_NON_COPYABLE (HashEntry) }; public: @@ -371,7 +374,7 @@ public: @see HashMap */ - class Iterator + class Iterator : LeakChecked , Uncopyable { public: //============================================================================== @@ -420,8 +423,6 @@ public: const HashMap& hashMap; HashEntry* entry; int index; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (Iterator) }; private: @@ -439,8 +440,6 @@ private: bassert (isPositiveAndBelow (hash, getNumSlots())); // your hash function is generating out-of-range numbers! return hash; } - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (HashMap) }; diff --git a/Subtrees/beast/modules/beast_core/containers/beast_LinkedListPointer.h b/Subtrees/beast/modules/beast_core/containers/beast_LinkedListPointer.h index 4ac97c5849..da6785669e 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_LinkedListPointer.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_LinkedListPointer.h @@ -53,7 +53,7 @@ @endcode */ template -class LinkedListPointer +class LinkedListPointer : Uncopyable { public: //============================================================================== @@ -330,7 +330,7 @@ public: list, and then repeatedly call Appender::append() to add items to the end of the list in O(1) time. */ - class Appender + class Appender : Uncopyable { public: /** Creates an appender which will add items to the given list. @@ -351,15 +351,11 @@ public: private: LinkedListPointer* endOfList; - - BEAST_DECLARE_NON_COPYABLE (Appender) }; private: //============================================================================== ObjectType* item; - - BEAST_DECLARE_NON_COPYABLE (LinkedListPointer) }; diff --git a/Subtrees/beast/modules/beast_basics/containers/beast_List.h b/Subtrees/beast/modules/beast_core/containers/beast_List.h similarity index 59% rename from Subtrees/beast/modules/beast_basics/containers/beast_List.h rename to Subtrees/beast/modules/beast_core/containers/beast_List.h index 00e90912f4..38c9493281 100644 --- a/Subtrees/beast/modules/beast_basics/containers/beast_List.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_List.h @@ -17,289 +17,291 @@ */ //============================================================================== -#ifndef BEAST_LIST_BEASTHEADER -#define BEAST_LIST_BEASTHEADER +#ifndef BEAST_LIST_H_INCLUDED +#define BEAST_LIST_H_INCLUDED -struct ListDefaultTag; +/** Intrusive Containers -/*============================================================================*/ -/** - Intrusive Containers + # Introduction - # Introduction + Intrusive containers are special containers that offer better performance + and exception safety guarantees than non-intrusive containers (like the + STL containers). They are useful building blocks for high performance + concurrent systems or other purposes where allocations are restricted + (such as the AudioIODeviceCallback object), because intrusive list + operations do not allocate or free memory. - Intrusive containers are special containers that offer better performance - and exception safety guarantees than non-intrusive containers (like the - STL containers). They are useful building blocks for high performance - concurrent systems or other purposes where allocations are restricted - (such as the AudioIODeviceCallback object), because intrusive list - operations do not allocate or free memory. + While intrusive containers were and are widely used in C, they became more + and more forgotten in C++ due to the presence of the standard containers + which don't support intrusive techniques. VFLib not only reintroduces this + technique to C++ for lists, it also encapsulates the implementation in a + mostly compliant STL interface. Hence anyone familiar with standard + containers can easily use them. - While intrusive containers were and are widely used in C, they became more - and more forgotten in C++ due to the presence of the standard containers - which don't support intrusive techniques. VFLib not only reintroduces this - technique to C++ for lists, it also encapsulates the implementation in a - mostly compliant STL interface. Hence anyone familiar with standard - containers can easily use them. + # Interface - # Interface + The interface for intrusive elements in this library is unified for all + containers. Unlike STL containers, objects placed into intrusive containers + are not copied. Instead, a pointer to the object is stored. All + responsibility for object lifetime is the responsibility of the caller; + the intrusive container just keeps track of what is in it. - The interface for intrusive elements in this library is unified for all - containers. Unlike STL containers, objects placed into intrusive containers - are not copied. Instead, a pointer to the object is stored. All - responsibility for object lifetime is the responsibility of the caller; - the intrusive container just keeps track of what is in it. + Summary of intrusive container differences: - Summary of intrusive container differences: + - Holds pointers to existing objects instead of copies. - - Holds pointers to existing objects instead of copies. + - Does not allocate or free any objects. - - Does not allocate or free any objects. + - Requires a element's class declaration to be modified. - - Requires a element's class declaration to be modified. + - Methods never throw exceptions when called with valid arguments. - - Methods never throw exceptions when called with valid arguments. + # Usage - # Usage + Like STL containers, intrusive containers are all template based, where the + template argument specifies the type of object that the container will hold. + These declarations specify a doubly linked list where each element points + to a user defined class: - Like STL containers, intrusive containers are all template based, where the - template argument specifies the type of object that the container will hold. - These declarations specify a doubly linked list where each element points - to a user defined class: + @code - @code + struct Object; // Forward declaration - class Object; // Forward declaration + List list; // Doubly-linked list of Object - List list; // Doubly-linked list of Object + @endcode - @endcode + Because intrusive containers allocate no memory, allowing objects to be + placed inside requires a modification to their class declaration. Each + intrusive container declares a nested class `Node` which elements must be + derived from, using the Curiously Recurring Template Pattern (CRTP). We + will continue to fully declare the Object type from the previous example + to support emplacement into an intrusive container: - Because intrusive containers allocate no memory, allowing objects to be - placed inside requires a modification to their class declaration. Each - intrusive container declares a nested class `Node` which elements must be - derived from, using the Curiously Recurring Template Pattern (CRTP). We - will continue to fully declare the Object type from the previous example - to support emplacement into an intrusive container: + @code - @code + struct Object : public List ::Node // Required for List + { + void performAction (); + }; - class Object : public List ::Node // Required for List - { - public: - void performAction (); - }; + @endcode - @endcode + Usage of a typedef eliminates redundant specification of the template + arguments but requires a forward declaration. The following code is + equivalent. - Usage of a typedef eliminates redundant specification of the template - arguments but requires a forward declaration. The following code is - equivalent. + @code - @code + struct Object; // Forward declaration - class Object; // Forward declaration + // Specify template parameters just once + typedef List ListType; - // Specify template parameters just once - typedef List ListType; + struct Object : public ListType::Node + { + void performAction (); + }; - class Object : public ListType::Node - { - void performAction (); - }; + ListType::Node list; - ListType::Node list; + @endcode - @endcode + With these declarations we may proceed to create our objects, add them to + the list, and perform operations: - With these declarations we may proceed to create our objects, add them to - the list, and perform operations: + @code - @code + // Create a few objects and put them in the list + for (i = 0; i < 5; ++i) + list.push_back (*new Object); - // Create a few objects and put them in the list - for (i = 0; i < 5; ++i) - list.push_back (*new Object); + // Call a method on each list + for (ListType::iterator iter = list.begin(); iter != list.end (); ++iter) + iter->performAction (); - // Call a method on each list - for (ListType::iterator iter = list.begin(); iter != list.end (); ++iter) - iter->performAction (); + @endcode - @endcode + Unlike regular STL containers, an object derived from an intrusive container + node cannot exist in more than one instance of that list at a time. This is + because the bookkeeping information for maintaining the list is kept in + the object rather than the list. - Unlike regular STL containers, an object derived from an intrusive container - node cannot exist in more than one instance of that list at a time. This is - because the bookkeeping information for maintaining the list is kept in - the object rather than the list. + To support objects existing in multiple containers, templates variations + are instantiated by distinguishing them with an empty structure, called a + tag. The object is derived from multiple instances of Node, where each + instance specifies a unique tag. The tag is passed as the second template + argument. When the second argument is unspecified, the default tag is used. - To support objects existing in multiple containers, templates variations - are instantiated by distinguishing them with an empty structure, called a - tag. The object is derived from multiple instances of Node, where each - instance specifies a unique tag. The tag is passed as the second template - argument. When the second argument is unspecified, the default tag is used. + This declaration example shows the usage of tags to allow an object to exist + simultaneously in two separate lists: - This declaration example shows the usage of tags to allow an object to exist - simultaneously in two separate lists: + @code - @code + struct GlobalListTag { }; // list of all objects + struct ActiveListTag { }; // subset of all objects that are active - struct GlobalListTag { }; // list of all objects - struct ActiveListTag { }; // subset of all objects that are active - - class Object : public List - , public List - { - public: + class Object : public List + , public List + { + public: Object () : m_isActive (false) { - // Add ourselves to the global list - s_globalList.push_front (*this); + // Add ourselves to the global list + s_globalList.push_front (*this); } ~Object () { - deactivate (); + deactivate (); } void becomeActive () { - // Add ourselves to the active list - if (!m_isActive) - { - s_activeList.push_front (*this); - m_isActive = true; - } + // Add ourselves to the active list + if (!m_isActive) + { + s_activeList.push_front (*this); + m_isActive = true; + } } void deactivate () { - if (m_isActive) - { - // Doesn't delete the object - s_activeList.erase (s_activeList.iterator_to (this)); + if (m_isActive) + { + // Doesn't delete the object + s_activeList.erase (s_activeList.iterator_to (this)); - m_isActive = false; - } + m_isActive = false; + } } - private: - bool m_isActive; + private: + bool m_isActive; - static List s_globalList; - static List s_activeList; - } + static List s_globalList; + static List s_activeList; + } - @endcode + @endcode - @defgroup intrusive intrusive - @ingroup beast_core + @defgroup intrusive intrusive + @ingroup beast_core */ -/*============================================================================*/ +//------------------------------------------------------------------------------ + +/** Default tag for List. + + @ingroup beast_core intrusive +*/ +struct ListDefaultTag; + /** - Intrusive doubly linked list. - - This intrusive List is a container similar in operation to std::list in the - Standard Template Library (STL). Like all @ref intrusive containers, List - requires you to first derive your class from List<>::Node: - - @code - - struct Object : List ::Node - { - Object (int value) : m_value (value) + Intrusive doubly linked list. + + This intrusive List is a container similar in operation to std::list in the + Standard Template Library (STL). Like all @ref intrusive containers, List + requires you to first derive your class from List<>::Node: + + @code + + struct Object : List ::Node { - } - - int m_value; - }; - - @endcode - - Now we define the list, and add a couple of items. - - @code - - List list; - - list.push_back (* (new Object (1))); - list.push_back (* (new Object (2))); - - @endcode - - For compatibility with the standard containers, push_back() expects a - reference to the object. Unlike the standard container, however, push_back() - places the actual object in the list and not a copy-constructed duplicate. - - Iterating over the list follows the same idiom as the STL: - - @code - - for (List ::iterator iter = list.begin(); iter != list.end; ++iter) - std::cout << iter->m_value; - - @endcode - - You can even use BOOST_FOREACH, or range based for loops: - - @code - - BOOST_FOREACH (Object& object, list) // boost only - std::cout << object.m_value; - - for (Object& object : list) // C++11 only - std::cout << object.m_value; - - @endcode - - Because List is mostly STL compliant, it can be passed into STL algorithms: - e.g. `std::for_each()` or `std::find_first_of()`. - - In general, objects placed into a List should be dynamically allocated - although this cannot be enforced at compile time. Since the caller provides - the storage for the object, the caller is also responsible for deleting the - object. An object still exists after being removed from a List, until the - caller deletes it. This means an element can be moved from one List to - another with practically no overhead. - - Unlike the standard containers, an object may only exist in one list at a - time, unless special preparations are made. The Tag template parameter is - used to distinguish between different list types for the same object, - allowing the object to exist in more than one list simultaneously. - - For example, consider an actor system where a global list of actors is - maintained, so that they can each be periodically receive processing - time. We wish to also maintain a list of the subset of actors that require - a domain-dependent update. To achieve this, we declare two tags, the - associated list types, and the list element thusly: - - @code - - struct Actor; // Forward declaration required - - struct ProcessTag { }; - struct UpdateTag { }; - - typedef List ProcessList; - typedef List UpdateList; - - // Derive from both node types so we can be in each list at once. - // - struct Actor : ProcessList::Node, UpdateList::Node - { - bool process (); // returns true if we need an update - void update (); - }; - - @endcode - - @tparam Element The base type of element which the list will store - pointers to. - - @tparam Tag An optional unique type name used to distinguish lists and nodes, - when the object can exist in multiple lists simultaneously. - - @ingroup beast_core intrusive + explicit Object (int value) : m_value (value) + { + } + + int m_value; + }; + + @endcode + + Now we define the list, and add a couple of items. + + @code + + List list; + + list.push_back (* (new Object (1))); + list.push_back (* (new Object (2))); + + @endcode + + For compatibility with the standard containers, push_back() expects a + reference to the object. Unlike the standard container, however, push_back() + places the actual object in the list and not a copy-constructed duplicate. + + Iterating over the list follows the same idiom as the STL: + + @code + + for (List ::iterator iter = list.begin(); iter != list.end; ++iter) + std::cout << iter->m_value; + + @endcode + + You can even use BOOST_FOREACH, or range based for loops: + + @code + + BOOST_FOREACH (Object& object, list) // boost only + std::cout << object.m_value; + + for (Object& object : list) // C++11 only + std::cout << object.m_value; + + @endcode + + Because List is mostly STL compliant, it can be passed into STL algorithms: + e.g. `std::for_each()` or `std::find_first_of()`. + + In general, objects placed into a List should be dynamically allocated + although this cannot be enforced at compile time. Since the caller provides + the storage for the object, the caller is also responsible for deleting the + object. An object still exists after being removed from a List, until the + caller deletes it. This means an element can be moved from one List to + another with practically no overhead. + + Unlike the standard containers, an object may only exist in one list at a + time, unless special preparations are made. The Tag template parameter is + used to distinguish between different list types for the same object, + allowing the object to exist in more than one list simultaneously. + + For example, consider an actor system where a global list of actors is + maintained, so that they can each be periodically receive processing + time. We wish to also maintain a list of the subset of actors that require + a domain-dependent update. To achieve this, we declare two tags, the + associated list types, and the list element thusly: + + @code + + struct Actor; // Forward declaration required + + struct ProcessTag { }; + struct UpdateTag { }; + + typedef List ProcessList; + typedef List UpdateList; + + // Derive from both node types so we can be in each list at once. + // + struct Actor : ProcessList::Node, UpdateList::Node + { + bool process (); // returns true if we need an update + void update (); + }; + + @endcode + + @tparam Element The base type of element which the list will store + pointers to. + + @tparam Tag An optional unique type name used to distinguish lists and nodes, + when the object can exist in multiple lists simultaneously. + + @ingroup beast_core intrusive */ template class List : Uncopyable @@ -786,11 +788,4 @@ private: Node m_tail; }; -/** - Default tag for List. - - @ingroup beast_core intrusive -*/ -struct ListDefaultTag { }; - #endif diff --git a/Subtrees/beast/modules/beast_basics/containers/beast_LockFreeQueue.h b/Subtrees/beast/modules/beast_core/containers/beast_LockFreeQueue.h similarity index 83% rename from Subtrees/beast/modules/beast_basics/containers/beast_LockFreeQueue.h rename to Subtrees/beast/modules/beast_core/containers/beast_LockFreeQueue.h index 28d51c6c7d..e64649b1e1 100644 --- a/Subtrees/beast/modules/beast_basics/containers/beast_LockFreeQueue.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_LockFreeQueue.h @@ -20,33 +20,31 @@ #ifndef BEAST_LOCKFREEQUEUE_BEASTHEADER #define BEAST_LOCKFREEQUEUE_BEASTHEADER -#include "../memory/beast_CacheLine.h" -#include "../memory/beast_AtomicPointer.h" -#include "../threads/beast_SpinDelay.h" +/** Default tag for LockFreeQueue -struct LockFreeQueueDefaultTag; + @ingroup beast_core intrusive +*/ +struct LockFreeQueueDefaultTag { }; -/*============================================================================*/ -/** - Multiple Producer, Single Consumer (MPSC) intrusive FIFO. +/** Multiple Producer, Single Consumer (MPSC) intrusive FIFO. - This container uses the same intrusive interface as List. It is wait-free - for producers and lock-free for consumers. The caller is responsible for - preventing the ABA problem (http://en.wikipedia.org/wiki/ABA_problem) + This container uses the same intrusive interface as List. It is wait-free + for producers and lock-free for consumers. The caller is responsible for + preventing the ABA problem (http://en.wikipedia.org/wiki/ABA_problem) - Invariants: + Invariants: - - Any thread may call push_back() at any time (Multiple Producer). + - Any thread may call push_back() at any time (Multiple Producer). - - Only one thread may call try_pop_front() at a time (Single Consumer) + - Only one thread may call try_pop_front() at a time (Single Consumer) - - The queue is signaled if there are one or more elements. + - The queue is signaled if there are one or more elements. - @param Tag A type name used to distinguish lists and nodes, for - putting objects in multiple lists. If this parameter is - omitted, the default tag is used. + @param Tag A type name used to distinguish lists and nodes, for + putting objects in multiple lists. If this parameter is + omitted, the default tag is used. - @ingroup beast_core intrusive + @ingroup beast_core intrusive */ template class LockFreeQueue @@ -216,11 +214,4 @@ private: Node m_null; }; -/*============================================================================*/ -/** Default tag for LockFreeQueue - - @ingroup beast_core intrusive -*/ -struct LockFreeQueueDefaultTag { }; - #endif diff --git a/Subtrees/beast/modules/beast_basics/containers/beast_LockFreeStack.h b/Subtrees/beast/modules/beast_core/containers/beast_LockFreeStack.h similarity index 99% rename from Subtrees/beast/modules/beast_basics/containers/beast_LockFreeStack.h rename to Subtrees/beast/modules/beast_core/containers/beast_LockFreeStack.h index 606163a29e..d32a47e38e 100644 --- a/Subtrees/beast/modules/beast_basics/containers/beast_LockFreeStack.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_LockFreeStack.h @@ -20,8 +20,6 @@ #ifndef BEAST_LOCKFREESTACK_BEASTHEADER #define BEAST_LOCKFREESTACK_BEASTHEADER -#include "../memory/beast_AtomicPointer.h" - struct LockFreeStackDefaultTag; /*============================================================================*/ @@ -58,6 +56,7 @@ public: private: friend class LockFreeStack; + // VFALCO TODO Use regular Atomic<> AtomicPointer m_next; }; diff --git a/Subtrees/beast/modules/beast_core/containers/beast_NamedValueSet.h b/Subtrees/beast/modules/beast_core/containers/beast_NamedValueSet.h index 1c2d08d93b..b451832f83 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_NamedValueSet.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_NamedValueSet.h @@ -132,7 +132,7 @@ public: private: //============================================================================== - class NamedValue + class NamedValue : LeakChecked { public: NamedValue() noexcept; @@ -151,7 +151,6 @@ private: var value; private: - BEAST_LEAK_DETECTOR (NamedValue) }; friend class LinkedListPointer; diff --git a/Subtrees/beast/modules/beast_core/containers/beast_OwnedArray.h b/Subtrees/beast/modules/beast_core/containers/beast_OwnedArray.h index e4a13429ff..04a62fc03e 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_OwnedArray.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_OwnedArray.h @@ -24,11 +24,6 @@ #ifndef BEAST_OWNEDARRAY_BEASTHEADER #define BEAST_OWNEDARRAY_BEASTHEADER -#include "beast_ArrayAllocationBase.h" -#include "beast_ElementComparator.h" -#include "../threads/beast_CriticalSection.h" - - //============================================================================== /** An array designed for holding objects. @@ -52,6 +47,8 @@ template class OwnedArray + : LeakChecked > + , Uncopyable { public: //============================================================================== @@ -124,8 +121,13 @@ public: inline ObjectClass* operator[] (const int index) const noexcept { const ScopedLockType lock (getLock()); - return isPositiveAndBelow (index, numUsed) ? data.elements [index] - : static_cast (nullptr); + if (isPositiveAndBelow (index, numUsed)) + { + bassert (data.elements != nullptr); + return data.elements [index]; + } + + return nullptr; } /** Returns a pointer to the object at this index in the array, without checking whether the index is in-range. @@ -136,7 +138,7 @@ public: inline ObjectClass* getUnchecked (const int index) const noexcept { const ScopedLockType lock (getLock()); - bassert (isPositiveAndBelow (index, numUsed)); + bassert (isPositiveAndBelow (index, numUsed) && data.elements != nullptr); return data.elements [index]; } @@ -239,11 +241,13 @@ public: @param newObject the new object to add to the array @see set, insert, addIfNotAlreadyThere, addSorted */ - void add (const ObjectClass* const newObject) noexcept + ObjectClass* add (ObjectClass* const newObject) noexcept { const ScopedLockType lock (getLock()); data.ensureAllocatedSize (numUsed + 1); + bassert (data.elements != nullptr); data.elements [numUsed++] = const_cast (newObject); + return const_cast (newObject); } /** Inserts a new object into the array at the given index. @@ -264,7 +268,7 @@ public: @see add, addSorted, addIfNotAlreadyThere, set */ void insert (int indexToInsertAt, - const ObjectClass* const newObject) noexcept + ObjectClass* const newObject) noexcept { if (indexToInsertAt >= 0) { @@ -274,6 +278,7 @@ public: indexToInsertAt = numUsed; data.ensureAllocatedSize (numUsed + 1); + bassert (data.elements != nullptr); ObjectClass** const e = data.elements + indexToInsertAt; const int numToMove = numUsed - indexToInsertAt; @@ -337,7 +342,7 @@ public: @param newObject the new object to add to the array */ - void addIfNotAlreadyThere (const ObjectClass* const newObject) noexcept + void addIfNotAlreadyThere (ObjectClass* const newObject) noexcept { const ScopedLockType lock (getLock()); @@ -426,6 +431,7 @@ public: numElementsToAdd = arrayToAddFrom.size() - startIndex; data.ensureAllocatedSize (numUsed + numElementsToAdd); + bassert (numElementsToAdd <= 0 || data.elements != nullptr); while (--numElementsToAdd >= 0) { @@ -466,6 +472,7 @@ public: numElementsToAdd = arrayToAddFrom.size() - startIndex; data.ensureAllocatedSize (numUsed + numElementsToAdd); + bassert (numElementsToAdd <= 0 || data.elements != nullptr); while (--numElementsToAdd >= 0) { @@ -857,9 +864,7 @@ private: while (numUsed > 0) delete data.elements [--numUsed]; } - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (OwnedArray) }; -#endif // BEAST_OWNEDARRAY_BEASTHEADER +#endif diff --git a/Subtrees/beast/modules/beast_core/containers/beast_PropertySet.h b/Subtrees/beast/modules/beast_core/containers/beast_PropertySet.h index f6513acc35..c3b5e61c8f 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_PropertySet.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_PropertySet.h @@ -39,7 +39,7 @@ See the PropertiesFile class for a subclass of this, which automatically broadcasts change messages and saves/loads the list from a file. */ -class BEAST_API PropertySet +class BEAST_API PropertySet : LeakChecked { public: //============================================================================== @@ -206,8 +206,6 @@ private: PropertySet* fallbackProperties; CriticalSection lock; bool ignoreCaseOfKeys; - - BEAST_LEAK_DETECTOR (PropertySet) }; diff --git a/Subtrees/beast/modules/beast_core/containers/beast_ReferenceCountedArray.h b/Subtrees/beast/modules/beast_core/containers/beast_ReferenceCountedArray.h index 9010b78fa8..6df23a2c14 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_ReferenceCountedArray.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_ReferenceCountedArray.h @@ -288,14 +288,17 @@ public: @param newObject the new object to add to the array @see set, insert, addIfNotAlreadyThere, addSorted, addArray */ - void add (ObjectClass* const newObject) noexcept + ObjectClass* add (ObjectClass* const newObject) noexcept { const ScopedLockType lock (getLock()); data.ensureAllocatedSize (numUsed + 1); + bassert (data.elements != nullptr); data.elements [numUsed++] = newObject; if (newObject != nullptr) newObject->incReferenceCount(); + + return newObject; } /** Inserts a new object into the array at the given index. @@ -311,8 +314,8 @@ public: @param newObject the new object to add to the array @see add, addSorted, addIfNotAlreadyThere, set */ - void insert (int indexToInsertAt, - ObjectClass* const newObject) noexcept + ObjectClass* insert (int indexToInsertAt, + ObjectClass* const newObject) noexcept { if (indexToInsertAt >= 0) { @@ -322,6 +325,7 @@ public: indexToInsertAt = numUsed; data.ensureAllocatedSize (numUsed + 1); + bassert (data.elements != nullptr); ObjectClass** const e = data.elements + indexToInsertAt; const int numToMove = numUsed - indexToInsertAt; @@ -335,10 +339,12 @@ public: newObject->incReferenceCount(); ++numUsed; + + return newObject; } else { - add (newObject); + return add (newObject); } } @@ -388,6 +394,7 @@ public: else { data.ensureAllocatedSize (numUsed + 1); + bassert (data.elements != nullptr); data.elements [numUsed++] = newObject; } } @@ -844,7 +851,6 @@ public: /** Returns the type of scoped lock to use for locking this array */ typedef typename TypeOfCriticalSectionToUse::ScopedLockType ScopedLockType; - private: //============================================================================== ArrayAllocationBase data; @@ -852,4 +858,4 @@ private: }; -#endif // BEAST_REFERENCECOUNTEDARRAY_BEASTHEADER +#endif \ No newline at end of file diff --git a/Subtrees/beast/modules/beast_core/containers/beast_ScopedValueSetter.h b/Subtrees/beast/modules/beast_core/containers/beast_ScopedValueSetter.h index 945ee13a5d..3709d6e721 100644 --- a/Subtrees/beast/modules/beast_core/containers/beast_ScopedValueSetter.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_ScopedValueSetter.h @@ -52,7 +52,7 @@ */ template -class ScopedValueSetter +class ScopedValueSetter : Uncopyable { public: /** Creates a ScopedValueSetter that will immediately change the specified value to the @@ -87,8 +87,6 @@ private: //============================================================================== ValueType& value; const ValueType originalValue; - - BEAST_DECLARE_NON_COPYABLE (ScopedValueSetter) }; diff --git a/Subtrees/beast/modules/beast_basics/containers/beast_SharedTable.h b/Subtrees/beast/modules/beast_core/containers/beast_SharedTable.h similarity index 98% rename from Subtrees/beast/modules/beast_basics/containers/beast_SharedTable.h rename to Subtrees/beast/modules/beast_core/containers/beast_SharedTable.h index 9e76c155fc..931569599a 100644 --- a/Subtrees/beast/modules/beast_basics/containers/beast_SharedTable.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_SharedTable.h @@ -26,7 +26,7 @@ @tparam ElementType The type of element. - @ingroup beast_gui + @ingroup beast_basics */ template class SharedTable @@ -72,8 +72,7 @@ public: #if BEAST_COMPILER_SUPPORTS_MOVE_SEMANTICS SharedTable (SharedTable&& other) noexcept -: - m_data (static_cast < typename Data::Ptr&& > (other.m_data)) + : m_data (static_cast < typename Data::Ptr&& > (other.m_data)) { } diff --git a/Subtrees/beast/modules/beast_basics/containers/beast_SortedLookupTable.h b/Subtrees/beast/modules/beast_core/containers/beast_SortedLookupTable.h similarity index 86% rename from Subtrees/beast/modules/beast_basics/containers/beast_SortedLookupTable.h rename to Subtrees/beast/modules/beast_core/containers/beast_SortedLookupTable.h index cdfd3e4539..af5ce63902 100644 --- a/Subtrees/beast/modules/beast_basics/containers/beast_SortedLookupTable.h +++ b/Subtrees/beast/modules/beast_core/containers/beast_SortedLookupTable.h @@ -20,30 +20,28 @@ #ifndef BEAST_SORTEDLOOKUPTABLE_BEASTHEADER #define BEAST_SORTEDLOOKUPTABLE_BEASTHEADER -//============================================================================== -/** - Sorted map for fast lookups. +/** Sorted map for fast lookups. - This container is optimized for a data set with fixed elements. + This container is optimized for a data set with fixed elements. - SchemaType obeys this concept: + SchemaType obeys this concept: - @code + @code - struct SchemaType - { - typename KeyType; - typename ValueType; + struct SchemaType + { + typename KeyType; + typename ValueType; - // Retrieve the key for a specified value. - KeyType getKey (Value const& value); - }; + // Retrieve the key for a specified value. + KeyType getKey (Value const& value); + }; - @endcode + @endcode - To use the table, reserve space with reserveSpaceForValues() if the number - of elements is known ahead of time. Then, call insert() for all the your - elements. Call prepareForLookups() once then call lookupValueByKey () + To use the table, reserve space with reserveSpaceForValues() if the number + of elements is known ahead of time. Then, call insert() for all the your + elements. Call prepareForLookups() once then call lookupValueByKey () */ template class SortedLookupTable @@ -51,11 +49,8 @@ class SortedLookupTable private: typedef typename SchemaType::KeyType KeyType; typedef typename SchemaType::ValueType ValueType; - typedef std::vector values_t; - values_t m_values; - private: struct SortCompare { @@ -151,6 +146,9 @@ public: return found; } + +private: + values_t m_values; }; #endif diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Debug.cpp b/Subtrees/beast/modules/beast_core/diagnostic/beast_Debug.cpp similarity index 91% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_Debug.cpp rename to Subtrees/beast/modules/beast_core/diagnostic/beast_Debug.cpp index 9cf37f37b1..e52ea11a67 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Debug.cpp +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_Debug.cpp @@ -20,35 +20,37 @@ namespace Debug { -//------------------------------------------------------------------------------ - -bool isDebuggerAttached () -{ - return beast_isRunningUnderDebugger (); -} - -//------------------------------------------------------------------------------ - -#if BEAST_DEBUG && defined (beast_breakDebugger) void breakPoint () { - if (isDebuggerAttached ()) +#if BEAST_DEBUG + if (beast_isRunningUnderDebugger ()) beast_breakDebugger; -} #else -void breakPoint () -{ - bassertfalse -} + bassertfalse; #endif +} -//---------------------------------------------------------------------------- +//------------------------------------------------------------------------------ #if BEAST_MSVC && defined (_DEBUG) -void setHeapAlwaysCheck (bool bAlwaysCheck) +#if BEAST_CHECK_MEMORY_LEAKS +struct DebugFlagsInitialiser +{ + DebugFlagsInitialiser() + { + // Activate leak checks on exit in the MSVC Debug CRT (C Runtime) + // + _CrtSetDbgFlag (_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); + } +}; + +static DebugFlagsInitialiser debugFlagsInitialiser; +#endif + +void setAlwaysCheckHeap (bool bAlwaysCheck) { int flags = _CrtSetDbgFlag (_CRTDBG_REPORT_FLAG); @@ -78,14 +80,21 @@ void setHeapReportLeaks (bool bReportLeaks) _CrtSetDbgFlag (flags); } +void reportLeaks () +{ + _CrtDumpMemoryLeaks (); +} + void checkHeap () { _CrtCheckMemory (); } +//------------------------------------------------------------------------------ + #else -void setHeapAlwaysCheck (bool) +void setAlwaysCheckHeap (bool) { } @@ -97,6 +106,10 @@ void setHeapReportLeaks (bool) { } +void reportLeaks () +{ +} + void checkHeap () { } diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Debug.h b/Subtrees/beast/modules/beast_core/diagnostic/beast_Debug.h similarity index 64% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_Debug.h rename to Subtrees/beast/modules/beast_core/diagnostic/beast_Debug.h index 205bcb0bab..c6013d46c5 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Debug.h +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_Debug.h @@ -17,22 +17,22 @@ */ //============================================================================== -#ifndef BEAST_DEBUG_BEASTHEADER -#define BEAST_DEBUG_BEASTHEADER +#ifndef BEAST_DEBUG_H_INCLUDED +#define BEAST_DEBUG_H_INCLUDED // Auxiliary outines for debugging namespace Debug { -// Returns true if a debugger is attached, for any build. -extern bool isDebuggerAttached (); +/** Break to debugger if a debugger is attached to a debug build. -// Breaks to the debugger if a debugger is attached. + Does nothing if no debugger is attached, or the build is not a debug build. +*/ extern void breakPoint (); -// VF: IS THIS REALLY THE RIGHT PLACE FOR THESE?? - +// VFALCO NOTE IS THIS REALLY THE RIGHT PLACE FOR THESE?? +// // Return only the filename portion of sourceFileName // This hides the programmer's directory structure from end-users. const String getFileNameFromPath (const char* sourceFileName); @@ -46,9 +46,34 @@ String stringToCommandLine (const String& s); // that can contain newlines and double quotes. String commandLineToString (const String& commandLine); -extern void setHeapAlwaysCheck (bool bAlwaysCheck); +// +// These control the MSVC C Runtime Debug heap. +// +// The calls currently do nothing on other platforms. +// + +/** Calls checkHeap() at every allocation and deallocation. +*/ +extern void setAlwaysCheckHeap (bool bAlwaysCheck); + +/** Keep freed memory blocks in the heap's linked list, assign them the + _FREE_BLOCK type, and fill them with the byte value 0xDD. +*/ extern void setHeapDelayedFree (bool bDelayedFree); + +/** Perform automatic leak checking at program exit through a call to + dumpMemoryLeaks() and generate an error report if the application + failed to free all the memory it allocated. +*/ extern void setHeapReportLeaks (bool bReportLeaks); + +/** Report all memory blocks which have not been freed. +*/ +extern void reportLeaks (); + +/** Confirms the integrity of the memory blocks allocated in the + debug heap (debug version only. +*/ extern void checkHeap (); } diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Error.cpp b/Subtrees/beast/modules/beast_core/diagnostic/beast_Error.cpp similarity index 92% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_Error.cpp rename to Subtrees/beast/modules/beast_core/diagnostic/beast_Error.cpp index 1b6952cccc..055d89a09a 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Error.cpp +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_Error.cpp @@ -25,7 +25,7 @@ Error::Error () { } -Error::Error (const Error& other) +Error::Error (Error const& other) : m_code (other.m_code) , m_reasonText (other.m_reasonText) , m_sourceFileName (other.m_sourceFileName) @@ -42,7 +42,7 @@ Error::~Error () noexcept bassert (!m_needsToBeChecked); } -Error& Error::operator= (const Error& other) +Error& Error::operator= (Error const& other) { m_code = other.m_code; m_reasonText = other.m_reasonText; @@ -73,12 +73,12 @@ bool Error::asBoolean () const return code () != success; } -const String Error::getReasonText () const +String const Error::getReasonText () const { return m_reasonText; } -const String Error::getSourceFilename () const +String const Error::getSourceFilename () const { return m_sourceFileName; } @@ -88,9 +88,9 @@ int Error::getLineNumber () const return m_lineNumber; } -Error& Error::fail (const char* sourceFileName, +Error& Error::fail (char const* sourceFileName, int lineNumber, - const String reasonText, + String const reasonText, Code errorCode) { bassert (m_code == success); @@ -105,7 +105,7 @@ Error& Error::fail (const char* sourceFileName, return *this; } -Error& Error::fail (const char* sourceFileName, +Error& Error::fail (char const* sourceFileName, int lineNumber, Code errorCode) { @@ -131,9 +131,9 @@ void Error::willBeReported () const m_needsToBeChecked = false; } -const char* Error::what () const noexcept +char const* Error::what () const noexcept { - if (!m_szWhat) + if (! m_szWhat) { // The application could not be initialized because sqlite was denied access permission // The application unexpectedly quit because the exception 'sqlite was denied access permission at file ' was thrown @@ -152,7 +152,7 @@ const char* Error::what () const noexcept return m_szWhat; } -const String Error::getReasonTextForCode (Code code) +String const Error::getReasonTextForCode (Code code) { String s; diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Error.h b/Subtrees/beast/modules/beast_core/diagnostic/beast_Error.h similarity index 76% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_Error.h rename to Subtrees/beast/modules/beast_core/diagnostic/beast_Error.h index 1d67ffa314..c5528dccbc 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Error.h +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_Error.h @@ -20,31 +20,28 @@ #ifndef BEAST_ERROR_BEASTHEADER #define BEAST_ERROR_BEASTHEADER -#include "beast_SafeBool.h" +/** A concise error report. -/** - A concise error report. + This lightweight but flexible class records lets you record the file and + line where a recoverable error occurred, along with some optional human + readable text. - This lightweight but flexible class records lets you record the file and - line where a recoverable error occurred, along with some optional human - readable text. + A recoverable error can be passed along and turned into a non recoverable + error by throwing the object: it's derivation from std::exception is + fully compliant with the C++ exception interface. - A recoverable error can be passed along and turned into a non recoverable - error by throwing the object: it's derivation from std::exception is - fully compliant with the C++ exception interface. - - @ingroup beast_core + @ingroup beast_core */ -class Error +class BEAST_API Error : public std::exception , public SafeBool { public: /** Numeric code. - This enumeration is useful when the caller needs to take different - actions depending on the failure. For example, trying again later if - a file is locked. + This enumeration is useful when the caller needs to take different + actions depending on the failure. For example, trying again later if + a file is locked. */ enum Code { @@ -74,8 +71,8 @@ public: }; Error (); - Error (const Error& other); - Error& operator= (const Error& other); + Error (Error const& other); + Error& operator= (Error const& other); virtual ~Error () noexcept; @@ -84,16 +81,16 @@ public: bool asBoolean () const; - const String getReasonText () const; - const String getSourceFilename () const; + String const getReasonText () const; + String const getSourceFilename () const; int getLineNumber () const; - Error& fail (const char* sourceFileName, + Error& fail (char const* sourceFileName, int lineNumber, - const String reasonText, + String const reasonText, Code errorCode = general); - Error& fail (const char* sourceFileName, + Error& fail (char const* sourceFileName, int lineNumber, Code errorCode = general); @@ -108,9 +105,9 @@ public: // for std::exception. This lets you throw an Error that should // terminate the application. The what() message will be less // descriptive so ideally you should catch the Error object instead. - const char* what () const noexcept; + char const* what () const noexcept; - static const String getReasonTextForCode (Code code); + static String const getReasonTextForCode (Code code); private: Code m_code; @@ -118,9 +115,8 @@ private: String m_sourceFileName; int m_lineNumber; mutable bool m_needsToBeChecked; - mutable String m_what; // created on demand - mutable const char* m_szWhat; + mutable char const* m_szWhat; }; #endif diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_FPUFlags.cpp b/Subtrees/beast/modules/beast_core/diagnostic/beast_FPUFlags.cpp similarity index 96% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_FPUFlags.cpp rename to Subtrees/beast/modules/beast_core/diagnostic/beast_FPUFlags.cpp index 8e83e14864..39eb589cfc 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_FPUFlags.cpp +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_FPUFlags.cpp @@ -17,7 +17,7 @@ */ //============================================================================== -void FPUFlags::clearUnsetFlagsFrom (const FPUFlags& flags) +void FPUFlags::clearUnsetFlagsFrom (FPUFlags const& flags) { if (!flags.getMaskNaNs ().is_set ()) m_maskNaNs.clear (); diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_FPUFlags.h b/Subtrees/beast/modules/beast_core/diagnostic/beast_FPUFlags.h similarity index 99% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_FPUFlags.h rename to Subtrees/beast/modules/beast_core/diagnostic/beast_FPUFlags.h index 71b9abde8a..b1c8d51d7f 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_FPUFlags.h +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_FPUFlags.h @@ -28,7 +28,7 @@ @ingroup beast_core */ -class FPUFlags +class BEAST_API FPUFlags { public: /** An individual FPU flag */ diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_LeakChecked.cpp b/Subtrees/beast/modules/beast_core/diagnostic/beast_LeakChecked.cpp similarity index 51% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_LeakChecked.cpp rename to Subtrees/beast/modules/beast_core/diagnostic/beast_LeakChecked.cpp index 0cf686ec78..a0694dd647 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_LeakChecked.cpp +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_LeakChecked.cpp @@ -17,10 +17,9 @@ */ //============================================================================== -#if BEAST_USE_LEAKCHECKED +namespace Implemented +{ -/*============================================================================*/ -// Type-independent portion of Counter class LeakCheckedBase::CounterBase::Singleton { public: @@ -29,16 +28,16 @@ public: m_list.push_front (counter); } - void detectAllLeaks () + void checkForLeaks () { for (;;) { - CounterBase* counter = m_list.pop_front (); + CounterBase* const counter = m_list.pop_front (); if (!counter) break; - counter->detectLeaks (); + counter->checkForLeaks (); } } @@ -50,6 +49,8 @@ public: } private: + friend class LeakCheckedBase; + LockFreeStack m_list; }; @@ -60,12 +61,7 @@ LeakCheckedBase::CounterBase::CounterBase () Singleton::getInstance ().push_back (this); } -void LeakCheckedBase::CounterBase::detectAllLeaks () -{ - Singleton::getInstance ().detectAllLeaks (); -} - -void LeakCheckedBase::CounterBase::detectLeaks () +void LeakCheckedBase::CounterBase::checkForLeaks () { // If there's a runtime error from this line, it means there's // an order of destruction problem between different translation units! @@ -76,16 +72,51 @@ void LeakCheckedBase::CounterBase::detectLeaks () if (count > 0) { - bassertfalse; - DBG ("[LEAK] " << count << " of " << getClassName ()); + /** If you hit this, then you've leaked one or more objects of the + specified class; the name should have been printed by the line + below. + + If you're leaking, it's probably because you're using old-fashioned, + non-RAII techniques for your object management. Tut, tut. Always, + always use ScopedPointers, OwnedArrays, ReferenceCountedObjects, + etc, and avoid the 'delete' operator at all costs! + */ + DBG ("Leaked objects: " << count << " of " << getClassName ()); + + //bassertfalse; } } //------------------------------------------------------------------------------ -void LeakCheckedBase::detectAllLeaks () +void LeakCheckedBase::reportDanglingPointer (char const* objectName) { - CounterBase::detectAllLeaks (); + /* If you hit this, then you've managed to delete more instances + of this class than you've created. That indicates that you're + deleting some dangling pointers. + + Note that although this assertion will have been triggered + during a destructor, it might not be this particular deletion + that's at fault - the incorrect one may have happened at an + earlier point in the program, and simply not been detected + until now. + + Most errors like this are caused by using old-fashioned, + non-RAII techniques for your object management. Tut, tut. + Always, always use ScopedPointers, OwnedArrays, + ReferenceCountedObjects, etc, and avoid the 'delete' operator + at all costs! + */ + DBG ("Dangling pointer deletion: " << objectName); + + bassertfalse; } -#endif +//------------------------------------------------------------------------------ + +void LeakCheckedBase::checkForLeaks () +{ + CounterBase::Singleton::getInstance ().checkForLeaks (); +} + +} diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_LeakChecked.h b/Subtrees/beast/modules/beast_core/diagnostic/beast_LeakChecked.h similarity index 70% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_LeakChecked.h rename to Subtrees/beast/modules/beast_core/diagnostic/beast_LeakChecked.h index 256812d9e6..4d84722422 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_LeakChecked.h +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_LeakChecked.h @@ -17,25 +17,14 @@ */ //============================================================================== -#ifndef BEAST_LEAKCHECKED_BEASTHEADER -#define BEAST_LEAKCHECKED_BEASTHEADER +#ifndef BEAST_LEAKCHECKED_H_INCLUDED +#define BEAST_LEAKCHECKED_H_INCLUDED -#include "beast_Error.h" -#include "beast_Throw.h" -#include "../memory/beast_StaticObject.h" -#include "../containers/beast_LockFreeStack.h" - -// -// Derived classes are automatically leak-checked on exit -// - -#if BEAST_USE_LEAKCHECKED - -class LeakCheckedBase +namespace Implemented { -public: - static void detectAllLeaks (); +class BEAST_API LeakCheckedBase +{ protected: class CounterBase : public LockFreeStack ::Node { @@ -56,18 +45,26 @@ protected: virtual char const* getClassName () const = 0; - static void detectAllLeaks (); - private: - void detectLeaks (); + void checkForLeaks (); virtual void checkPureVirtual () const = 0; - protected: + private: + friend class LeakCheckedBase; + class Singleton; Atomic m_count; }; + +protected: + static void reportDanglingPointer (char const* objectName); + +private: + friend class PerformedAtExit::ExitHook; + + static void checkForLeaks (); }; //------------------------------------------------------------------------------ @@ -83,32 +80,25 @@ class LeakChecked : private LeakCheckedBase protected: LeakChecked () noexcept { - if (getLeakCheckedCounter ().increment () == 0) - { - DBG ("[LOGIC] " << getLeakCheckedName ()); - Throw (Error ().fail (__FILE__, __LINE__)); - } + getCounter ().increment (); } - LeakChecked (const LeakChecked&) noexcept + LeakChecked (LeakChecked const&) noexcept { - if (getLeakCheckedCounter ().increment () == 0) - { - DBG ("[LOGIC] " << getLeakCheckedName ()); - Throw (Error ().fail (__FILE__, __LINE__)); - } + getCounter ().increment (); } ~LeakChecked () { - if (getLeakCheckedCounter ().decrement () < 0) + if (getCounter ().decrement () < 0) { - DBG ("[LOGIC] " << getLeakCheckedName ()); - Throw (Error ().fail (__FILE__, __LINE__)); + reportDanglingPointer (getLeakCheckedName ()); } } private: + // Singleton that maintains the count of this object + // class Counter : public CounterBase { public: @@ -135,30 +125,39 @@ private: return typeid (Object).name (); } - static Counter& getLeakCheckedCounter () noexcept + // Retrieve the singleton for this object + // + static Counter& getCounter () noexcept { static Counter* volatile s_instance; static Static::Initializer s_initializer; - if (s_initializer.begin ()) + if (s_initializer.beginConstruction ()) { static char s_storage [sizeof (Counter)]; s_instance = new (s_storage) Counter; - s_initializer.end (); + + s_initializer.endConstruction (); } return *s_instance; } }; -#else +} -class LeakCheckedBase +//------------------------------------------------------------------------------ + +namespace Dummy +{ + +class BEAST_API LeakCheckedBase { private: friend class PerformedAtExit; - static void detectAllLeaks () { } +public: + static void checkForLeaks () { } }; template @@ -166,6 +165,18 @@ struct LeakChecked : LeakCheckedBase { }; +} + +//------------------------------------------------------------------------------ + +// Lift the corresponding implementation +// +#if BEAST_CHECK_MEMORY_LEAKS +using Implemented::LeakChecked; +using Implemented::LeakCheckedBase; +#else +using Dummy::LeakChecked; +using Dummy::LeakCheckedBase; #endif #endif diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_SafeBool.h b/Subtrees/beast/modules/beast_core/diagnostic/beast_SafeBool.h similarity index 98% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_SafeBool.h rename to Subtrees/beast/modules/beast_core/diagnostic/beast_SafeBool.h index f7fb22b6e7..be592299bb 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_SafeBool.h +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_SafeBool.h @@ -40,7 +40,7 @@ @ingroup beast_core */ -class SafeBoolBase +class BEAST_API SafeBoolBase { private: void disallowed () const { } diff --git a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Throw.h b/Subtrees/beast/modules/beast_core/diagnostic/beast_Throw.h similarity index 85% rename from Subtrees/beast/modules/beast_basics/diagnostic/beast_Throw.h rename to Subtrees/beast/modules/beast_core/diagnostic/beast_Throw.h index d1116df77f..a488193fef 100644 --- a/Subtrees/beast/modules/beast_basics/diagnostic/beast_Throw.h +++ b/Subtrees/beast/modules/beast_core/diagnostic/beast_Throw.h @@ -17,16 +17,14 @@ */ //============================================================================== -#ifndef BEAST_THROW_BEASTHEADER -#define BEAST_THROW_BEASTHEADER +#ifndef BEAST_THROW_H_INCLUDED +#define BEAST_THROW_H_INCLUDED -#include "beast_Debug.h" - -// -// Throw an exception, with the opportunity to get a -// breakpoint with the call stack before the throw. -// +/** Throw an exception, with a debugger hook. + This provides an opportunity to utilize the debugger before + the stack is unwound. +*/ template inline void Throw (Exception const& e) { diff --git a/Subtrees/beast/modules/beast_core/files/beast_DirectoryIterator.h b/Subtrees/beast/modules/beast_core/files/beast_DirectoryIterator.h index 3ba068a899..144dca788c 100644 --- a/Subtrees/beast/modules/beast_core/files/beast_DirectoryIterator.h +++ b/Subtrees/beast/modules/beast_core/files/beast_DirectoryIterator.h @@ -41,7 +41,7 @@ It can also guess how far it's got using a wildly inaccurate algorithm. */ -class BEAST_API DirectoryIterator +class BEAST_API DirectoryIterator : LeakChecked , Uncopyable { public: //============================================================================== @@ -116,7 +116,7 @@ public: private: //============================================================================== - class NativeIterator + class NativeIterator : LeakChecked , Uncopyable { public: NativeIterator (const File& directory, const String& wildCard); @@ -132,8 +132,6 @@ private: friend class DirectoryIterator; friend class ScopedPointer; ScopedPointer pimpl; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (NativeIterator) }; friend class ScopedPointer; @@ -147,8 +145,6 @@ private: bool hasBeenAdvanced; ScopedPointer subIterator; File currentFile; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (DirectoryIterator) }; #endif // BEAST_DIRECTORYITERATOR_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/files/beast_File.cpp b/Subtrees/beast/modules/beast_core/files/beast_File.cpp index ffa83d7260..6c0706ed02 100644 --- a/Subtrees/beast/modules/beast_core/files/beast_File.cpp +++ b/Subtrees/beast/modules/beast_core/files/beast_File.cpp @@ -21,11 +21,46 @@ */ //============================================================================== +// We need to make a shared singleton or else there are +// issues with the leak detector and order of detruction. +// +class NonexistentHolder : public SharedSingleton +{ +public: + NonexistentHolder () + : SharedSingleton (SingletonLifetime::persistAfterCreation) + { + } + + static NonexistentHolder* createInstance () + { + return new NonexistentHolder; + } + + File const file; +}; + +File const& File::nonexistent () +{ + return NonexistentHolder::getInstance ()->file; +} + +//------------------------------------------------------------------------------ + File::File (const String& fullPathName) : fullPath (parseAbsolutePath (fullPathName)) { } +File::File (const File& other) + : fullPath (other.fullPath) +{ +} + +File::~File() noexcept +{ +} + File File::createFileWithoutCheckingPath (const String& path) noexcept { File f; @@ -33,11 +68,6 @@ File File::createFileWithoutCheckingPath (const String& path) noexcept return f; } -File::File (const File& other) - : fullPath (other.fullPath) -{ -} - File& File::operator= (const String& newPath) { fullPath = parseAbsolutePath (newPath); @@ -63,9 +93,6 @@ File& File::operator= (File&& other) noexcept } #endif -const File File::nonexistent; - - //============================================================================== String File::parseAbsolutePath (const String& p) { @@ -317,7 +344,7 @@ String File::getFileNameWithoutExtension() const bool File::isAChildOf (const File& potentialParent) const { - if (potentialParent == File::nonexistent) + if (potentialParent == File::nonexistent ()) return false; const String ourPath (getPathUpToLastSlash()); @@ -629,7 +656,7 @@ bool File::hasFileExtension (const String& possibleSuffix) const File File::withFileExtension (const String& newExtension) const { if (fullPath.isEmpty()) - return File::nonexistent; + return File::nonexistent (); String filePart (getFileName()); @@ -910,7 +937,7 @@ public: const File home (File::getSpecialLocation (File::userHomeDirectory)); const File temp (File::getSpecialLocation (File::tempDirectory)); - expect (! File::nonexistent.exists()); + expect (! File::nonexistent ().exists()); expect (home.isDirectory()); expect (home.exists()); expect (! home.existsAsFile()); diff --git a/Subtrees/beast/modules/beast_core/files/beast_File.h b/Subtrees/beast/modules/beast_core/files/beast_File.h index 462aeb58a9..9566a5ece8 100644 --- a/Subtrees/beast/modules/beast_core/files/beast_File.h +++ b/Subtrees/beast/modules/beast_core/files/beast_File.h @@ -46,14 +46,14 @@ class FileOutputStream; @see FileInputStream, FileOutputStream */ -class BEAST_API File +class BEAST_API File : LeakChecked { public: //============================================================================== /** Creates an (invalid) file object. The file is initially set to an empty path, so getFullPath() will return - an empty string, and comparing the file to File::nonexistent will return + an empty string, and comparing the file to File::nonexistent() will return true. You can use its operator= method to point it at a proper file. @@ -76,7 +76,7 @@ public: File (const File& other); /** Destructor. */ - ~File() noexcept {} + ~File() noexcept; /** Sets the file based on an absolute pathname. @@ -100,7 +100,7 @@ public: //============================================================================== /** This static constant is used for referring to an 'invalid' file. */ - static const File nonexistent; + static File const& nonexistent (); //============================================================================== /** Checks whether the file actually exists. @@ -246,20 +246,21 @@ public: int64 hashCode64() const; //============================================================================== - /** Returns a file based on a relative path. + /** Returns a file that represents a relative (or absolute) sub-path of the current one. This will find a child file or directory of the current object. e.g. File ("/moose/fish").getChildFile ("foo.txt") will produce "/moose/fish/foo.txt". + File ("/moose/fish").getChildFile ("haddock/foo.txt") will produce "/moose/fish/haddock/foo.txt". File ("/moose/fish").getChildFile ("../foo.txt") will produce "/moose/foo.txt". If the string is actually an absolute path, it will be treated as such, e.g. - File ("/moose/fish").getChildFile ("/foo.txt") will produce "/foo.txt" + File ("/moose/fish").getChildFile ("/foo.txt") will produce "/foo.txt" @see getSiblingFile, getParentDirectory, getRelativePathFrom, isAChildOf */ - File getChildFile (String relativePath) const; + File getChildFile (String relativeOrAbsolutePath) const; /** Returns a file which is in the same directory as this one. @@ -948,8 +949,6 @@ private: bool setFileTimesInternal (int64 m, int64 a, int64 c) const; void getFileTimesInternal (int64& m, int64& a, int64& c) const; bool setFileReadOnlyInternal (bool) const; - - BEAST_LEAK_DETECTOR (File) }; #endif // BEAST_FILE_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/files/beast_FileInputStream.h b/Subtrees/beast/modules/beast_core/files/beast_FileInputStream.h index 72ee64f126..1e1d94fecf 100644 --- a/Subtrees/beast/modules/beast_core/files/beast_FileInputStream.h +++ b/Subtrees/beast/modules/beast_core/files/beast_FileInputStream.h @@ -34,7 +34,9 @@ @see InputStream, FileOutputStream, File::createInputStream */ -class BEAST_API FileInputStream : public InputStream +class BEAST_API FileInputStream + : public InputStream + , LeakChecked { public: //============================================================================== @@ -87,8 +89,6 @@ private: void openHandle(); void closeHandle(); size_t readInternal (void* buffer, size_t numBytes); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (FileInputStream) }; #endif // BEAST_FILEINPUTSTREAM_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/files/beast_FileOutputStream.h b/Subtrees/beast/modules/beast_core/files/beast_FileOutputStream.h index a51d806c51..5f358ecd63 100644 --- a/Subtrees/beast/modules/beast_core/files/beast_FileOutputStream.h +++ b/Subtrees/beast/modules/beast_core/files/beast_FileOutputStream.h @@ -34,7 +34,9 @@ @see OutputStream, FileInputStream, File::createOutputStream */ -class BEAST_API FileOutputStream : public OutputStream +class BEAST_API FileOutputStream + : public OutputStream + , LeakChecked { public: //============================================================================== @@ -107,8 +109,6 @@ private: bool flushBuffer(); int64 setPositionInternal (int64); ssize_t writeInternal (const void*, size_t); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (FileOutputStream) }; #endif // BEAST_FILEOUTPUTSTREAM_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/files/beast_FileSearchPath.h b/Subtrees/beast/modules/beast_core/files/beast_FileSearchPath.h index 334b0a94de..bb6906af36 100644 --- a/Subtrees/beast/modules/beast_core/files/beast_FileSearchPath.h +++ b/Subtrees/beast/modules/beast_core/files/beast_FileSearchPath.h @@ -34,7 +34,7 @@ @see File */ -class BEAST_API FileSearchPath +class BEAST_API FileSearchPath : LeakChecked { public: //============================================================================== @@ -157,8 +157,6 @@ private: StringArray directories; void init (const String& path); - - BEAST_LEAK_DETECTOR (FileSearchPath) }; #endif // BEAST_FILESEARCHPATH_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/files/beast_MemoryMappedFile.h b/Subtrees/beast/modules/beast_core/files/beast_MemoryMappedFile.h index 0c20619b8a..56b6579148 100644 --- a/Subtrees/beast/modules/beast_core/files/beast_MemoryMappedFile.h +++ b/Subtrees/beast/modules/beast_core/files/beast_MemoryMappedFile.h @@ -30,7 +30,7 @@ /** Maps a file into virtual memory for easy reading and/or writing. */ -class BEAST_API MemoryMappedFile +class BEAST_API MemoryMappedFile : LeakChecked , Uncopyable { public: /** The read/write flags used when opening a memory mapped file. */ @@ -103,8 +103,6 @@ private: #endif void openInternal (const File&, AccessMode); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (MemoryMappedFile) }; diff --git a/Subtrees/beast/modules/beast_core/files/beast_TemporaryFile.cpp b/Subtrees/beast/modules/beast_core/files/beast_TemporaryFile.cpp index 561d2c16cf..e5603da72c 100644 --- a/Subtrees/beast/modules/beast_core/files/beast_TemporaryFile.cpp +++ b/Subtrees/beast/modules/beast_core/files/beast_TemporaryFile.cpp @@ -45,7 +45,7 @@ TemporaryFile::TemporaryFile (const File& target, const int optionFlags) targetFile (target) { // If you use this constructor, you need to give it a valid target file! - bassert (targetFile != File::nonexistent); + bassert (targetFile != File::nonexistent ()); } TemporaryFile::TemporaryFile (const File& target, const File& temporary) @@ -74,7 +74,7 @@ bool TemporaryFile::overwriteTargetFileWithTemporary() const { // This method only works if you created this object with the constructor // that takes a target file! - bassert (targetFile != File::nonexistent); + bassert (targetFile != File::nonexistent ()); if (temporaryFile.exists()) { diff --git a/Subtrees/beast/modules/beast_core/files/beast_TemporaryFile.h b/Subtrees/beast/modules/beast_core/files/beast_TemporaryFile.h index bdc9a67284..b46371b482 100644 --- a/Subtrees/beast/modules/beast_core/files/beast_TemporaryFile.h +++ b/Subtrees/beast/modules/beast_core/files/beast_TemporaryFile.h @@ -65,7 +65,7 @@ @see File, FileOutputStream */ -class BEAST_API TemporaryFile +class BEAST_API TemporaryFile : LeakChecked , Uncopyable { public: //============================================================================== @@ -159,8 +159,6 @@ public: private: //============================================================================== const File temporaryFile, targetFile; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (TemporaryFile) }; #endif // BEAST_TEMPORARYFILE_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/logging/beast_FileLogger.h b/Subtrees/beast/modules/beast_core/logging/beast_FileLogger.h index be3ae39963..d6cb1c6020 100644 --- a/Subtrees/beast/modules/beast_core/logging/beast_FileLogger.h +++ b/Subtrees/beast/modules/beast_core/logging/beast_FileLogger.h @@ -35,7 +35,10 @@ @see Logger */ -class BEAST_API FileLogger : public Logger +class BEAST_API FileLogger + : public Logger + , LeakChecked + , Uncopyable { public: //============================================================================== @@ -126,8 +129,6 @@ private: CriticalSection logLock; void trimFileSize (int64 maxFileSizeBytes) const; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (FileLogger) }; diff --git a/Subtrees/beast/modules/beast_core/maths/beast_BigInteger.h b/Subtrees/beast/modules/beast_core/maths/beast_BigInteger.h index c5737d8179..90e47728fc 100644 --- a/Subtrees/beast/modules/beast_core/maths/beast_BigInteger.h +++ b/Subtrees/beast/modules/beast_core/maths/beast_BigInteger.h @@ -39,7 +39,7 @@ class MemoryBlock; Negative values are possible, but the value isn't stored as 2s-complement, so be careful if you use negative values and look at the values of individual bits. */ -class BEAST_API BigInteger +class BEAST_API BigInteger : LeakChecked { public: //============================================================================== @@ -312,8 +312,6 @@ private: void ensureSize (size_t numVals); void shiftLeft (int bits, int startBit); void shiftRight (int bits, int startBit); - - BEAST_LEAK_DETECTOR (BigInteger) }; /** Writes a BigInteger to an OutputStream as a UTF8 decimal string. */ diff --git a/Subtrees/beast/modules/beast_core/maths/beast_Expression.cpp b/Subtrees/beast/modules/beast_core/maths/beast_Expression.cpp index d0b1c29403..5c3dd36915 100644 --- a/Subtrees/beast/modules/beast_core/maths/beast_Expression.cpp +++ b/Subtrees/beast/modules/beast_core/maths/beast_Expression.cpp @@ -21,7 +21,8 @@ */ //============================================================================== -class Expression::Term : public SingleThreadedReferenceCountedObject +class Expression::Term + : public SingleThreadedReferenceCountedObject { public: Term() {} @@ -69,9 +70,6 @@ public: for (int i = getNumInputs(); --i >= 0;) getInput(i)->visitAllSymbols (visitor, scope, recursionDepth); } - -private: - BEAST_DECLARE_NON_COPYABLE (Term) }; @@ -295,7 +293,8 @@ struct Expression::Helpers }; //============================================================================== - class DotOperator : public BinaryTerm + class DotOperator + : public BinaryTerm { public: DotOperator (SymbolTerm* const l, Term* const r) : BinaryTerm (l, r) {} @@ -345,7 +344,9 @@ struct Expression::Helpers private: //============================================================================== - class EvaluationVisitor : public Scope::Visitor + class EvaluationVisitor + : public Scope::Visitor + , Uncopyable { public: EvaluationVisitor (const TermPtr& t, const int recursion) @@ -356,12 +357,11 @@ struct Expression::Helpers const TermPtr input; TermPtr output; const int recursionCount; - - private: - BEAST_DECLARE_NON_COPYABLE (EvaluationVisitor) }; - class SymbolVisitingVisitor : public Scope::Visitor + class SymbolVisitingVisitor + : public Scope::Visitor + , Uncopyable { public: SymbolVisitingVisitor (const TermPtr& t, SymbolVisitor& v, const int recursion) @@ -373,11 +373,11 @@ struct Expression::Helpers const TermPtr input; SymbolVisitor& visitor; const int recursionCount; - - BEAST_DECLARE_NON_COPYABLE (SymbolVisitingVisitor) }; - class SymbolRenamingVisitor : public Scope::Visitor + class SymbolRenamingVisitor + : public Scope::Visitor + , Uncopyable { public: SymbolRenamingVisitor (const TermPtr& t, const Expression::Symbol& symbol_, const String& newName_, const int recursionCount_) @@ -390,13 +390,9 @@ struct Expression::Helpers const Symbol& symbol; const String newName; const int recursionCount; - - BEAST_DECLARE_NON_COPYABLE (SymbolRenamingVisitor) }; SymbolTerm* getSymbol() const { return static_cast (left.get()); } - - BEAST_DECLARE_NON_COPYABLE (DotOperator) }; //============================================================================== @@ -446,7 +442,7 @@ struct Expression::Helpers }; //============================================================================== - class Add : public BinaryTerm + class Add : public BinaryTerm { public: Add (Term* const l, Term* const r) : BinaryTerm (l, r) {} @@ -465,13 +461,10 @@ struct Expression::Helpers return new Subtract (newDest, (input == left ? right : left)->clone()); } - - private: - BEAST_DECLARE_NON_COPYABLE (Add) }; //============================================================================== - class Subtract : public BinaryTerm + class Subtract : public BinaryTerm { public: Subtract (Term* const l, Term* const r) : BinaryTerm (l, r) {} @@ -493,13 +486,10 @@ struct Expression::Helpers return new Subtract (left->clone(), newDest); } - - private: - BEAST_DECLARE_NON_COPYABLE (Subtract) }; //============================================================================== - class Multiply : public BinaryTerm + class Multiply : public BinaryTerm { public: Multiply (Term* const l, Term* const r) : BinaryTerm (l, r) {} @@ -518,13 +508,10 @@ struct Expression::Helpers return new Divide (newDest, (input == left ? right : left)->clone()); } - - private: - BEAST_DECLARE_NON_COPYABLE (Multiply) }; //============================================================================== - class Divide : public BinaryTerm + class Divide : public BinaryTerm { public: Divide (Term* const l, Term* const r) : BinaryTerm (l, r) {} @@ -546,9 +533,6 @@ struct Expression::Helpers return new Divide (left->clone(), newDest); } - - private: - BEAST_DECLARE_NON_COPYABLE (Divide) }; //============================================================================== @@ -621,7 +605,9 @@ struct Expression::Helpers } //============================================================================== - class SymbolCheckVisitor : public Term::SymbolVisitor + class SymbolCheckVisitor + : public Term::SymbolVisitor + , Uncopyable { public: SymbolCheckVisitor (const Symbol& symbol_) : wasFound (false), symbol (symbol_) {} @@ -631,12 +617,12 @@ struct Expression::Helpers private: const Symbol& symbol; - - BEAST_DECLARE_NON_COPYABLE (SymbolCheckVisitor) }; //============================================================================== - class SymbolListVisitor : public Term::SymbolVisitor + class SymbolListVisitor + : public Term::SymbolVisitor + , Uncopyable { public: SymbolListVisitor (Array& list_) : list (list_) {} @@ -644,12 +630,10 @@ struct Expression::Helpers private: Array& list; - - BEAST_DECLARE_NON_COPYABLE (SymbolListVisitor) }; //============================================================================== - class Parser + class Parser : Uncopyable { public: //============================================================================== @@ -908,8 +892,6 @@ struct Expression::Helpers return e; } - - BEAST_DECLARE_NON_COPYABLE (Parser) }; }; diff --git a/Subtrees/beast/modules/beast_basics/math/beast_Interval.h b/Subtrees/beast/modules/beast_core/maths/beast_Interval.h similarity index 100% rename from Subtrees/beast/modules/beast_basics/math/beast_Interval.h rename to Subtrees/beast/modules/beast_core/maths/beast_Interval.h diff --git a/Subtrees/beast/modules/beast_core/maths/beast_MathsFunctions.h b/Subtrees/beast/modules/beast_core/maths/beast_MathsFunctions.h index 29168654d2..f4b7e288b4 100644 --- a/Subtrees/beast/modules/beast_core/maths/beast_MathsFunctions.h +++ b/Subtrees/beast/modules/beast_core/maths/beast_MathsFunctions.h @@ -199,7 +199,7 @@ void findMinAndMax (const Type* values, int numValues, Type& lowest, Type& highe @param valueToConstrain the value to try to return @returns the closest value to valueToConstrain which lies between lowerLimit and upperLimit (inclusive) - @see jlimit0To, bmin, bmax + @see blimit0To, bmin, bmax */ template inline Type blimit (const Type lowerLimit, diff --git a/Subtrees/beast/modules/beast_core/maths/beast_Random.h b/Subtrees/beast/modules/beast_core/maths/beast_Random.h index cf9ce61613..f35c0eed92 100644 --- a/Subtrees/beast/modules/beast_core/maths/beast_Random.h +++ b/Subtrees/beast/modules/beast_core/maths/beast_Random.h @@ -33,7 +33,7 @@ You can create a Random object and use it to generate a sequence of random numbers. */ -class BEAST_API Random +class BEAST_API Random : LeakChecked { public: //============================================================================== @@ -127,8 +127,6 @@ public: private: //============================================================================== int64 seed; - - BEAST_LEAK_DETECTOR (Random) }; diff --git a/Subtrees/beast/modules/beast_core/memory/beast_Atomic.h b/Subtrees/beast/modules/beast_core/memory/beast_Atomic.h index 83d43994b2..74ba13c135 100644 --- a/Subtrees/beast/modules/beast_core/memory/beast_Atomic.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_Atomic.h @@ -148,10 +148,14 @@ public: volatile Type value; private: - static inline Type castFrom32Bit (int32 value) noexcept { return *(Type*) &value; } - static inline Type castFrom64Bit (int64 value) noexcept { return *(Type*) &value; } - static inline int32 castTo32Bit (Type value) noexcept { return *(int32*) &value; } - static inline int64 castTo64Bit (Type value) noexcept { return *(int64*) &value; } + template + static inline Dest castTo (Source value) noexcept { union { Dest d; Source s; } u; u.s = value; return u.d; } + + static inline Type castFrom32Bit (int32 value) noexcept { return castTo (value); } + static inline Type castFrom64Bit (int64 value) noexcept { return castTo (value); } + static inline int32 castTo32Bit (Type value) noexcept { return castTo (value); } + static inline int64 castTo64Bit (Type value) noexcept { return castTo (value); } + Type operator++ (int); // better to just use pre-increment with atomics.. Type operator-- (int); diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_AtomicCounter.h b/Subtrees/beast/modules/beast_core/memory/beast_AtomicCounter.h similarity index 98% rename from Subtrees/beast/modules/beast_basics/memory/beast_AtomicCounter.h rename to Subtrees/beast/modules/beast_core/memory/beast_AtomicCounter.h index 5d8a5f7301..978dd5a855 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_AtomicCounter.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_AtomicCounter.h @@ -30,7 +30,7 @@ @ingroup beast_core */ -class AtomicCounter +class BEAST_API AtomicCounter { public: /** Create a new counter. diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_AtomicFlag.h b/Subtrees/beast/modules/beast_core/memory/beast_AtomicFlag.h similarity index 99% rename from Subtrees/beast/modules/beast_basics/memory/beast_AtomicFlag.h rename to Subtrees/beast/modules/beast_core/memory/beast_AtomicFlag.h index cb1ba7ea3f..e7d605bf63 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_AtomicFlag.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_AtomicFlag.h @@ -30,7 +30,7 @@ @ingroup beast_core */ -class AtomicFlag +class BEAST_API AtomicFlag { public: /** Create an AtomicFlag in the reset state. */ diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_AtomicPointer.h b/Subtrees/beast/modules/beast_core/memory/beast_AtomicPointer.h similarity index 100% rename from Subtrees/beast/modules/beast_basics/memory/beast_AtomicPointer.h rename to Subtrees/beast/modules/beast_core/memory/beast_AtomicPointer.h diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_AtomicState.h b/Subtrees/beast/modules/beast_core/memory/beast_AtomicState.h similarity index 99% rename from Subtrees/beast/modules/beast_basics/memory/beast_AtomicState.h rename to Subtrees/beast/modules/beast_core/memory/beast_AtomicState.h index 7c2cb85bb1..a3224c935f 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_AtomicState.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_AtomicState.h @@ -29,7 +29,7 @@ @ingroup beast_core */ -class AtomicState +class BEAST_API AtomicState { public: /** Create a new state with an optional starting value. diff --git a/Subtrees/beast/modules/beast_core/memory/beast_ByteOrder.h b/Subtrees/beast/modules/beast_core/memory/beast_ByteOrder.h index 0d41fe2266..b02eedf9d6 100644 --- a/Subtrees/beast/modules/beast_core/memory/beast_ByteOrder.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_ByteOrder.h @@ -29,7 +29,7 @@ /** Contains static methods for converting the byte order between different endiannesses. */ -class BEAST_API ByteOrder +class BEAST_API ByteOrder : Uncopyable { public: //============================================================================== @@ -93,8 +93,6 @@ public: private: ByteOrder(); - - BEAST_DECLARE_NON_COPYABLE (ByteOrder) }; diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_CacheLine.h b/Subtrees/beast/modules/beast_core/memory/beast_CacheLine.h similarity index 76% rename from Subtrees/beast/modules/beast_basics/memory/beast_CacheLine.h rename to Subtrees/beast/modules/beast_core/memory/beast_CacheLine.h index 3cdd28997a..ef7b476bbd 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_CacheLine.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_CacheLine.h @@ -20,12 +20,10 @@ #ifndef BEAST_CACHELINE_BEASTHEADER #define BEAST_CACHELINE_BEASTHEADER -#include "beast_MemoryAlignment.h" - // Allows turning off of all padding, // e.g. for memory-constrained systems or testing. // -#define GLOBAL_PADDING_ENABLED 0 +#define GLOBAL_PADDING_ENABLED 1 namespace CacheLine { @@ -34,77 +32,76 @@ namespace CacheLine // Pads an object so that it starts on a cache line boundary. // +/** Pad an object to start on a cache line boundary. + + Up to 8 constructor parameters are passed through. +*/ template class Aligned { public: - ~Aligned () - { - ptr ()->~T (); - } - Aligned () { new (ptr ()) T; } template - explicit Aligned (const T1& t1) + Aligned (T1 t1) { new (ptr ()) T (t1); } template - Aligned (const T1& t1, const T2& t2) + Aligned (T1 t1, T2 t2) { new (ptr ()) T (t1, t2); } template - Aligned (const T1& t1, const T2& t2, const T3& t3) + Aligned (T1 t1, T2 t2, T3 t3) { new (ptr ()) T (t1, t2, t3); } template - Aligned (const T1& t1, const T2& t2, const T3& t3, const T4& t4) + Aligned (T1 t1, T2 t2, T3 t3, T4 t4) { new (ptr ()) T (t1, t2, t3, t4); } template - Aligned (const T1& t1, const T2& t2, const T3& t3, - const T4& t4, const T5& t5) + Aligned (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) { new (ptr ()) T (t1, t2, t3, t4, t5); } template - Aligned (const T1& t1, const T2& t2, const T3& t3, - const T4& t4, const T5& t5, const T6& t6) + Aligned (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6) { new (ptr ()) T (t1, t2, t3, t4, t5, t6); } - template < class T1, class T2, class T3, class T4, - class T5, class T6, class T7 > - Aligned (const T1& t1, const T2& t2, const T3& t3, const T4& t4, - const T5& t5, const T6& t6, const T7& t7) + template + Aligned (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7) { new (ptr ()) T (t1, t2, t3, t4, t5, t6, t7); } - template < class T1, class T2, class T3, class T4, - class T5, class T6, class T7, class T8 > - Aligned (const T1& t1, const T2& t2, const T3& t3, const T4& t4, - const T5& t5, const T6& t6, const T7& t7, const T8& t8) + template + Aligned (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7, T8 t8) { new (ptr ()) T (t1, t2, t3, t4, t5, t6, t7, t8); } - void operator= (T const& other) + ~Aligned () + { + ptr ()->~T (); + } + + T& operator= (T const& other) { *ptr () = other; + return *ptr (); } inline T& operator* () noexcept { return *ptr (); } @@ -112,22 +109,10 @@ public: inline operator T& () noexcept { return *ptr (); } inline operator T* () noexcept { return ptr (); } - inline const T& operator* () const noexcept - { - return *ptr (); - } - inline const T* operator-> () const noexcept - { - return ptr (); - } - inline operator const T& () const noexcept - { - return *ptr (); - } - inline operator const T* () const noexcept - { - return ptr (); - } + inline const T& operator* () const noexcept { return *ptr (); } + inline const T* operator-> () const noexcept { return ptr (); } + inline operator T const& () const noexcept { return *ptr (); } + inline operator T const* () const noexcept { return ptr (); } private: inline T* ptr () noexcept @@ -140,85 +125,85 @@ private: */ } - char m_storage [ (sizeof (T) + Memory::cacheLineAlignMask) - & ~Memory::cacheLineAlignMask]; + char m_storage [ (sizeof (T) + Memory::cacheLineAlignMask) & ~Memory::cacheLineAlignMask]; }; -// Holds an object padded it to completely fill a CPU cache line. -// The caller must ensure that this object starts at the beginning -// of a cache line. -// +/** End-pads an object to completely fill straddling CPU cache lines. + + The caller must ensure that this object starts at the beginning + of a cache line. +*/ template class Padded { public: Padded () - { } + { + } template - explicit Padded (const T1& t1) - : m_t (t1) { } + explicit Padded (T1 t1) + : m_t (t1) + { + } template - Padded (const T1& t1, const T2& t2) - : m_t (t1, t2) { } + Padded (T1 t1, T2 t2) + : m_t (t1, t2) + { + } template - Padded (const T1& t1, const T2& t2, const T3& t3) - : m_t (t1, t2, t3) { } + Padded (T1 t1, T2 t2, T3 t3) + : m_t (t1, t2, t3) + { + } template - Padded (const T1& t1, const T2& t2, const T3& t3, const T4& t4) - : m_t (t1, t2, t3, t4) { } + Padded (T1 t1, T2 t2, T3 t3, T4 t4) + : m_t (t1, t2, t3, t4) + { + } template - Padded (const T1& t1, const T2& t2, const T3& t3, - const T4& t4, const T5& t5) - : m_t (t1, t2, t3, t4, t5) { } + Padded (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) + : m_t (t1, t2, t3, t4, t5) + { + } template - Padded (const T1& t1, const T2& t2, const T3& t3, - const T4& t4, const T5& t5, const T6& t6) - : m_t (t1, t2, t3, t4, t5, t6) { } + Padded (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6) + : m_t (t1, t2, t3, t4, t5, t6) + { + } - template < class T1, class T2, class T3, class T4, - class T5, class T6, class T7 > - Padded (const T1& t1, const T2& t2, const T3& t3, const T4& t4, - const T5& t5, const T6& t6, const T7& t7) - : m_t (t1, t2, t3, t4, t5, t6, t7) { } + template + Padded (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7) + : m_t (t1, t2, t3, t4, t5, t6, t7) + { + } - template < class T1, class T2, class T3, class T4, - class T5, class T6, class T7, class T8 > - Padded (const T1& t1, const T2& t2, const T3& t3, const T4& t4, - const T5& t5, const T6& t6, const T7& t7, const T8& t8) - : m_t (t1, t2, t3, t4, t5, t6, t7, t8) { } + template + Padded (T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7, T8 t8) + : m_t (t1, t2, t3, t4, t5, t6, t7, t8) + { + } - void operator= (const T& other) + T& operator= (T const& other) { m_t = other; + return m_t; } - T& operator* () noexcept { return m_t; } + T& operator* () noexcept { return m_t; } T* operator-> () noexcept { return &m_t; } - operator T& () noexcept { return m_t; } - operator T* () noexcept { return &m_t; } + operator T& () noexcept { return m_t; } + operator T* () noexcept { return &m_t; } - const T& operator* () const noexcept - { - return m_t; - } - const T* operator-> () const noexcept - { - return &m_t; - } - operator const T& () const noexcept - { - return m_t; - } - operator const T* () const noexcept - { - return &m_t; - } + const T& operator* () const noexcept { return m_t; } + const T* operator-> () const noexcept { return &m_t; } + operator T const& () const noexcept { return m_t; } + operator T const* () const noexcept { return &m_t; } private: T m_t; diff --git a/Subtrees/beast/modules/beast_core/memory/beast_HeapBlock.h b/Subtrees/beast/modules/beast_core/memory/beast_HeapBlock.h index 4badf561c2..074d3ecad2 100644 --- a/Subtrees/beast/modules/beast_core/memory/beast_HeapBlock.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_HeapBlock.h @@ -82,7 +82,7 @@ namespace HeapBlockHelper @see Array, OwnedArray, MemoryBlock */ template -class HeapBlock +class HeapBlock : Uncopyable { public: //============================================================================== @@ -294,7 +294,6 @@ private: } #if ! (defined (BEAST_DLL) || defined (BEAST_DLL_BUILD)) - BEAST_DECLARE_NON_COPYABLE (HeapBlock) BEAST_PREVENT_HEAP_ALLOCATION // Creating a 'new HeapBlock' would be missing the point! #endif }; diff --git a/Subtrees/beast/modules/beast_core/memory/beast_LeakedObjectDetector.h b/Subtrees/beast/modules/beast_core/memory/beast_LeakedObjectDetector.h deleted file mode 100644 index eaa01b15df..0000000000 --- a/Subtrees/beast/modules/beast_core/memory/beast_LeakedObjectDetector.h +++ /dev/null @@ -1,144 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Portions of this file are from JUCE. - Copyright (c) 2013 - Raw Material Software Ltd. - Please visit http://www.juce.com - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_LEAKEDOBJECTDETECTOR_BEASTHEADER -#define BEAST_LEAKEDOBJECTDETECTOR_BEASTHEADER - -#include "../text/beast_String.h" -#include "beast_Atomic.h" - - -//============================================================================== -/** - Embedding an instance of this class inside another class can be used as a low-overhead - way of detecting leaked instances. - - This class keeps an internal static count of the number of instances that are - active, so that when the app is shutdown and the static destructors are called, - it can check whether there are any left-over instances that may have been leaked. - - To use it, use the BEAST_LEAK_DETECTOR macro as a simple way to put one in your - class declaration. Have a look through the beast codebase for examples, it's used - in most of the classes. -*/ -template -class LeakedObjectDetector -{ -public: - //============================================================================== - LeakedObjectDetector() noexcept { ++(getCounter().numObjects); } - LeakedObjectDetector (const LeakedObjectDetector&) noexcept { ++(getCounter().numObjects); } - - ~LeakedObjectDetector() - { - if (--(getCounter().numObjects) < 0) - { - DBG ("*** Dangling pointer deletion! Class: " << getLeakedObjectClassName()); - - /** If you hit this, then you've managed to delete more instances of this class than you've - created.. That indicates that you're deleting some dangling pointers. - - Note that although this assertion will have been triggered during a destructor, it might - not be this particular deletion that's at fault - the incorrect one may have happened - at an earlier point in the program, and simply not been detected until now. - - Most errors like this are caused by using old-fashioned, non-RAII techniques for - your object management. Tut, tut. Always, always use ScopedPointers, OwnedArrays, - ReferenceCountedObjects, etc, and avoid the 'delete' operator at all costs! - */ - bassertfalse; - } - } - -private: - //============================================================================== - class LeakCounter - { - public: - LeakCounter() noexcept {} - - ~LeakCounter() - { - if (numObjects.value > 0) - { - DBG ("*** Leaked objects detected: " << numObjects.value << " instance(s) of class " << getLeakedObjectClassName()); - - /** If you hit this, then you've leaked one or more objects of the type specified by - the 'OwnerClass' template parameter - the name should have been printed by the line above. - - If you're leaking, it's probably because you're using old-fashioned, non-RAII techniques for - your object management. Tut, tut. Always, always use ScopedPointers, OwnedArrays, - ReferenceCountedObjects, etc, and avoid the 'delete' operator at all costs! - */ - bassertfalse; - } - } - - Atomic numObjects; - }; - - static const char* getLeakedObjectClassName() - { - return OwnerClass::getLeakedObjectClassName(); - } - - static LeakCounter& getCounter() noexcept - { - static LeakCounter counter; - return counter; - } -}; - -//============================================================================== -#if DOXYGEN || ! defined (BEAST_LEAK_DETECTOR) - #if (DOXYGEN || BEAST_CHECK_MEMORY_LEAKS) - /** This macro lets you embed a leak-detecting object inside a class. - - To use it, simply declare a BEAST_LEAK_DETECTOR(YourClassName) inside a private section - of the class declaration. E.g. - - @code - class MyClass - { - public: - MyClass(); - void blahBlah(); - - private: - BEAST_LEAK_DETECTOR (MyClass) - }; - @endcode - - @see BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR, LeakedObjectDetector - */ - #define BEAST_LEAK_DETECTOR(OwnerClass) \ - friend class beast::LeakedObjectDetector; \ - static const char* getLeakedObjectClassName() noexcept { return #OwnerClass; } \ - beast::LeakedObjectDetector BEAST_JOIN_MACRO (leakDetector, __LINE__); - #else - #define BEAST_LEAK_DETECTOR(OwnerClass) - #endif -#endif - - -#endif // BEAST_LEAKEDOBJECTDETECTOR_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/memory/beast_Memory.h b/Subtrees/beast/modules/beast_core/memory/beast_Memory.h index 68def8bd8e..092113cd7a 100644 --- a/Subtrees/beast/modules/beast_core/memory/beast_Memory.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_Memory.h @@ -65,7 +65,7 @@ inline Type* createCopyIfNotNull (const Type* pointer) { return pointer != n /** A handy C++ wrapper that creates and deletes an NSAutoreleasePool object using RAII. You should use the BEAST_AUTORELEASEPOOL macro to create a local auto-release pool on the stack. */ - class BEAST_API ScopedAutoReleasePool + class BEAST_API ScopedAutoReleasePool : Uncopyable { public: ScopedAutoReleasePool(); @@ -73,8 +73,6 @@ inline Type* createCopyIfNotNull (const Type* pointer) { return pointer != n private: void* pool; - - BEAST_DECLARE_NON_COPYABLE (ScopedAutoReleasePool) }; /** A macro that can be used to easily declare a local ScopedAutoReleasePool @@ -92,30 +90,5 @@ inline Type* createCopyIfNotNull (const Type* pointer) { return pointer != n #define BEAST_AUTORELEASEPOOL #endif -//============================================================================== -/* In a Windows DLL build, we'll expose some malloc/free functions that live inside the DLL, and use these for - allocating all the objects - that way all beast objects in the DLL and in the host will live in the same heap, - avoiding problems when an object is created in one module and passed across to another where it is deleted. - By piggy-backing on the BEAST_LEAK_DETECTOR macro, these allocators can be injected into most beast classes. -*/ -#if BEAST_MSVC && (defined (BEAST_DLL) || defined (BEAST_DLL_BUILD)) && ! (BEAST_DISABLE_DLL_ALLOCATORS || DOXYGEN) - extern BEAST_API void* beastDLL_malloc (size_t); - extern BEAST_API void beastDLL_free (void*); - - #define BEAST_LEAK_DETECTOR(OwnerClass) public:\ - static void* operator new (size_t sz) { return beast::beastDLL_malloc (sz); } \ - static void* operator new (size_t, void* p) { return p; } \ - static void operator delete (void* p) { beast::beastDLL_free (p); } \ - static void operator delete (void*, void*) {} #endif -//============================================================================== -/** (Deprecated) This was a Windows-specific way of checking for object leaks - now please - use the BEAST_LEAK_DETECTOR instead. -*/ -#ifndef beast_UseDebuggingNewOperator - #define beast_UseDebuggingNewOperator -#endif - - -#endif // BEAST_MEMORY_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_MemoryAlignment.h b/Subtrees/beast/modules/beast_core/memory/beast_MemoryAlignment.h similarity index 90% rename from Subtrees/beast/modules/beast_basics/memory/beast_MemoryAlignment.h rename to Subtrees/beast/modules/beast_core/memory/beast_MemoryAlignment.h index f11f57b769..0b2e50a5ba 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_MemoryAlignment.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_MemoryAlignment.h @@ -23,7 +23,12 @@ namespace Memory { +//------------------------------------------------------------------------------ + // Constants +// +// These need to be set based on the target CPU +// const int cacheLineAlignBits = 6; // 64 bytes const int cacheLineAlignBytes = 1 << cacheLineAlignBits; @@ -33,6 +38,8 @@ const int allocAlignBits = 3; // 8 bytes const int allocAlignBytes = 1 << allocAlignBits; const int allocAlignMask = allocAlignBytes - 1; +//------------------------------------------------------------------------------ + // Returns the number of bytes needed to advance p to the correct alignment template inline size_t bytesNeededForAlignment (P const* const p) diff --git a/Subtrees/beast/modules/beast_core/memory/beast_MemoryBlock.h b/Subtrees/beast/modules/beast_core/memory/beast_MemoryBlock.h index 54107a4b0f..103f885ac0 100644 --- a/Subtrees/beast/modules/beast_core/memory/beast_MemoryBlock.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_MemoryBlock.h @@ -33,7 +33,7 @@ A class to hold a resizable block of raw data. */ -class BEAST_API MemoryBlock +class BEAST_API MemoryBlock : LeakChecked { public: //============================================================================== @@ -247,8 +247,6 @@ private: HeapBlock data; size_t size; static const char* const encodingTable; - - BEAST_LEAK_DETECTOR (MemoryBlock) }; diff --git a/Subtrees/beast/modules/beast_core/memory/beast_ReferenceCountedObject.h b/Subtrees/beast/modules/beast_core/memory/beast_ReferenceCountedObject.h index fe0e02a05c..f0458ea8c8 100644 --- a/Subtrees/beast/modules/beast_core/memory/beast_ReferenceCountedObject.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_ReferenceCountedObject.h @@ -59,7 +59,7 @@ @see ReferenceCountedObjectPtr, ReferenceCountedArray, SingleThreadedReferenceCountedObject */ -class BEAST_API ReferenceCountedObject +class BEAST_API ReferenceCountedObject : Uncopyable { public: //============================================================================== @@ -114,8 +114,6 @@ protected: private: //============================================================================== Atomic refCount; - - BEAST_DECLARE_NON_COPYABLE (ReferenceCountedObject) }; @@ -130,7 +128,7 @@ private: @see ReferenceCountedObject, ReferenceCountedObjectPtr, ReferenceCountedArray */ -class BEAST_API SingleThreadedReferenceCountedObject +class BEAST_API SingleThreadedReferenceCountedObject : public Uncopyable { public: //============================================================================== @@ -175,8 +173,6 @@ protected: private: //============================================================================== int refCount; - - BEAST_DECLARE_NON_COPYABLE (SingleThreadedReferenceCountedObject) }; diff --git a/Subtrees/beast/modules/beast_core/memory/beast_ScopedPointer.h b/Subtrees/beast/modules/beast_core/memory/beast_ScopedPointer.h index f07b7f2929..e0bb516d29 100644 --- a/Subtrees/beast/modules/beast_core/memory/beast_ScopedPointer.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_ScopedPointer.h @@ -62,7 +62,7 @@ you'd need to return a raw pointer (or use a std::auto_ptr instead). */ template -class ScopedPointer +class ScopedPointer : Uncopyable { public: //============================================================================== @@ -215,7 +215,6 @@ private: It's probably best to use the latter form when writing your object declarations anyway, as this is a better representation of the code that you actually want the compiler to produce. */ - BEAST_DECLARE_NON_COPYABLE (ScopedPointer) #endif }; diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_RefCountedSingleton.h b/Subtrees/beast/modules/beast_core/memory/beast_SharedSingleton.h similarity index 65% rename from Subtrees/beast/modules/beast_basics/memory/beast_RefCountedSingleton.h rename to Subtrees/beast/modules/beast_core/memory/beast_SharedSingleton.h index 54ce76dcbb..fbf1fe7f2e 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_RefCountedSingleton.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_SharedSingleton.h @@ -20,36 +20,35 @@ #ifndef BEAST_REFERENCECOUNTEDSINGLETON_BEASTHEADER #define BEAST_REFERENCECOUNTEDSINGLETON_BEASTHEADER -#include "../events/beast_PerformedAtExit.h" -#include "../memory/beast_StaticObject.h" +/** Thread-safe singleton which comes into existence on first use. Use this + instead of creating objects with static storage duration. These singletons + are automatically reference counted, so if you hold a pointer to it in every + object that depends on it, the order of destruction of objects is assured + to be correct. -/** - Thread-safe singleton which comes into existence on first use. Use this - instead of creating objects with static storage duration. These singletons - are automatically reference counted, so if you hold a pointer to it in every - object that depends on it, the order of destruction of objects is assured - to be correct. + class Object must provide the function `Object* Object::createInstance()` - class Object must provide the function `Object* Object::createInstance()` - - @class RefCountedSingleton - @ingroup beast_core + @class SharedSingleton + @ingroup beast_core */ /** @{ */ -class SingletonLifetime +class BEAST_API SingletonLifetime { - // "base classes dependent on a template parameter - // aren't part of lookup." - ville public: - /** - Construction options for RefCountedSingleton + // It would be nice if we didn't have to qualify the enumeration but + // Argument Dependent Lookup is inapplicable here because: + // + // "Base classes dependent on a template parameter aren't part of lookup." + // - ville + // - @ingroup beast_core + /** Construction options for SharedSingleton + + @ingroup beast_core */ enum Lifetime { - /** Singleton is created on first use and destroyed when - the last reference is removed. + /** Created on first use, destroyed when the last reference is removed. */ createOnDemand, @@ -60,12 +59,20 @@ public: /** The singleton is created on first use and persists until program exit. */ - persistAfterCreation + persistAfterCreation, + + /** The singleton is created when needed and never destroyed. + + This is useful for applications which do not have a clean exit. + */ + neverDestroyed }; }; +//------------------------------------------------------------------------------ + template -class RefCountedSingleton +class SharedSingleton : public SingletonLifetime , private PerformedAtExit { @@ -76,12 +83,13 @@ protected: @param lifetime The lifetime management option. */ - explicit RefCountedSingleton (Lifetime const lifetime) + explicit SharedSingleton (Lifetime const lifetime) : m_lifetime (lifetime) { bassert (s_instance == nullptr); - if (m_lifetime == persistAfterCreation) + if (m_lifetime == persistAfterCreation || + m_lifetime == neverDestroyed) { incReferenceCount (); } @@ -93,7 +101,7 @@ protected: *s_created = true; } - virtual ~RefCountedSingleton () + virtual ~SharedSingleton () { bassert (s_instance == nullptr); } @@ -154,6 +162,9 @@ private: { bool destroy; + // Handle the condition where one thread is releasing the last + // reference just as another thread is trying to acquire it. + // { LockType::ScopedLockType lock (*s_mutex); @@ -170,6 +181,8 @@ private: if (destroy) { + bassert (m_lifetime != neverDestroyed); + delete this; } } @@ -180,20 +193,20 @@ private: private: static Object* s_instance; - static Static::Storage > s_mutex; - static Static::Storage > s_created; + static Static::Storage > s_mutex; + static Static::Storage > s_created; }; /** @{ */ template -Object* RefCountedSingleton ::s_instance; +Object* SharedSingleton ::s_instance; template -Static::Storage ::LockType, RefCountedSingleton > -RefCountedSingleton ::s_mutex; +Static::Storage ::LockType, SharedSingleton > +SharedSingleton ::s_mutex; template -Static::Storage > -RefCountedSingleton ::s_created; +Static::Storage > +SharedSingleton ::s_created; #endif diff --git a/Subtrees/beast/modules/beast_core/memory/beast_Singleton.h b/Subtrees/beast/modules/beast_core/memory/beast_Singleton.h deleted file mode 100644 index bb039ea32c..0000000000 --- a/Subtrees/beast/modules/beast_core/memory/beast_Singleton.h +++ /dev/null @@ -1,287 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Portions of this file are from JUCE. - Copyright (c) 2013 - Raw Material Software Ltd. - Please visit http://www.juce.com - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_SINGLETON_BEASTHEADER -#define BEAST_SINGLETON_BEASTHEADER - - -//============================================================================== -/** - Macro to declare member variables and methods for a singleton class. - - To use this, add the line beast_DeclareSingleton (MyClass, doNotRecreateAfterDeletion) - to the class's definition. - - Then put a macro beast_ImplementSingleton (MyClass) along with the class's - implementation code. - - It's also a very good idea to also add the call clearSingletonInstance() in your class's - destructor, in case it is deleted by other means than deleteInstance() - - Clients can then call the static method MyClass::getInstance() to get a pointer - to the singleton, or MyClass::getInstanceWithoutCreating() which will return 0 if - no instance currently exists. - - e.g. @code - - class MySingleton - { - public: - MySingleton() - { - } - - ~MySingleton() - { - // this ensures that no dangling pointers are left when the - // singleton is deleted. - clearSingletonInstance(); - } - - beast_DeclareSingleton (MySingleton, false) - }; - - beast_ImplementSingleton (MySingleton) - - - // example of usage: - MySingleton* m = MySingleton::getInstance(); // creates the singleton if there isn't already one. - - ... - - MySingleton::deleteInstance(); // safely deletes the singleton (if it's been created). - - @endcode - - If doNotRecreateAfterDeletion = true, it won't allow the object to be created more - than once during the process's lifetime - i.e. after you've created and deleted the - object, getInstance() will refuse to create another one. This can be useful to stop - objects being accidentally re-created during your app's shutdown code. - - If you know that your object will only be created and deleted by a single thread, you - can use the slightly more efficient beast_DeclareSingleton_SingleThreaded() macro instead - of this one. - - @see beast_ImplementSingleton, beast_DeclareSingleton_SingleThreaded -*/ -#define beast_DeclareSingleton(classname, doNotRecreateAfterDeletion) \ -\ - static classname* _singletonInstance; \ - static beast::CriticalSection _singletonLock; \ -\ - static classname* BEAST_CALLTYPE getInstance() \ - { \ - if (_singletonInstance == nullptr) \ - {\ - const beast::ScopedLock sl (_singletonLock); \ -\ - if (_singletonInstance == nullptr) \ - { \ - static bool alreadyInside = false; \ - static bool createdOnceAlready = false; \ -\ - const bool problem = alreadyInside || ((doNotRecreateAfterDeletion) && createdOnceAlready); \ - bassert (! problem); \ - if (! problem) \ - { \ - createdOnceAlready = true; \ - alreadyInside = true; \ - classname* newObject = new classname(); /* (use a stack variable to avoid setting the newObject value before the class has finished its constructor) */ \ - alreadyInside = false; \ -\ - _singletonInstance = newObject; \ - } \ - } \ - } \ -\ - return _singletonInstance; \ - } \ -\ - static inline classname* BEAST_CALLTYPE getInstanceWithoutCreating() noexcept\ - { \ - return _singletonInstance; \ - } \ -\ - static void BEAST_CALLTYPE deleteInstance() \ - { \ - const beast::ScopedLock sl (_singletonLock); \ - if (_singletonInstance != nullptr) \ - { \ - classname* const old = _singletonInstance; \ - _singletonInstance = nullptr; \ - delete old; \ - } \ - } \ -\ - void clearSingletonInstance() noexcept\ - { \ - if (_singletonInstance == this) \ - _singletonInstance = nullptr; \ - } - - -//============================================================================== -/** This is a counterpart to the beast_DeclareSingleton macro. - - After adding the beast_DeclareSingleton to the class definition, this macro has - to be used in the cpp file. -*/ -#define beast_ImplementSingleton(classname) \ -\ - classname* classname::_singletonInstance = nullptr; \ - beast::CriticalSection classname::_singletonLock; - - -//============================================================================== -/** - Macro to declare member variables and methods for a singleton class. - - This is exactly the same as beast_DeclareSingleton, but doesn't use a critical - section to make access to it thread-safe. If you know that your object will - only ever be created or deleted by a single thread, then this is a - more efficient version to use. - - If doNotRecreateAfterDeletion = true, it won't allow the object to be created more - than once during the process's lifetime - i.e. after you've created and deleted the - object, getInstance() will refuse to create another one. This can be useful to stop - objects being accidentally re-created during your app's shutdown code. - - See the documentation for beast_DeclareSingleton for more information about - how to use it, the only difference being that you have to use - beast_ImplementSingleton_SingleThreaded instead of beast_ImplementSingleton. - - @see beast_ImplementSingleton_SingleThreaded, beast_DeclareSingleton, beast_DeclareSingleton_SingleThreaded_Minimal -*/ -#define beast_DeclareSingleton_SingleThreaded(classname, doNotRecreateAfterDeletion) \ -\ - static classname* _singletonInstance; \ -\ - static classname* getInstance() \ - { \ - if (_singletonInstance == nullptr) \ - { \ - static bool alreadyInside = false; \ - static bool createdOnceAlready = false; \ -\ - const bool problem = alreadyInside || ((doNotRecreateAfterDeletion) && createdOnceAlready); \ - bassert (! problem); \ - if (! problem) \ - { \ - createdOnceAlready = true; \ - alreadyInside = true; \ - classname* newObject = new classname(); /* (use a stack variable to avoid setting the newObject value before the class has finished its constructor) */ \ - alreadyInside = false; \ -\ - _singletonInstance = newObject; \ - } \ - } \ -\ - return _singletonInstance; \ - } \ -\ - static inline classname* getInstanceWithoutCreating() noexcept\ - { \ - return _singletonInstance; \ - } \ -\ - static void deleteInstance() \ - { \ - if (_singletonInstance != nullptr) \ - { \ - classname* const old = _singletonInstance; \ - _singletonInstance = nullptr; \ - delete old; \ - } \ - } \ -\ - void clearSingletonInstance() noexcept\ - { \ - if (_singletonInstance == this) \ - _singletonInstance = nullptr; \ - } - -//============================================================================== -/** - Macro to declare member variables and methods for a singleton class. - - This is like beast_DeclareSingleton_SingleThreaded, but doesn't do any checking - for recursion or repeated instantiation. It's intended for use as a lightweight - version of a singleton, where you're using it in very straightforward - circumstances and don't need the extra checking. - - Beast use the normal beast_ImplementSingleton_SingleThreaded as the counterpart - to this declaration, as you would with beast_DeclareSingleton_SingleThreaded. - - See the documentation for beast_DeclareSingleton for more information about - how to use it, the only difference being that you have to use - beast_ImplementSingleton_SingleThreaded instead of beast_ImplementSingleton. - - @see beast_ImplementSingleton_SingleThreaded, beast_DeclareSingleton -*/ -#define beast_DeclareSingleton_SingleThreaded_Minimal(classname) \ -\ - static classname* _singletonInstance; \ -\ - static classname* getInstance() \ - { \ - if (_singletonInstance == nullptr) \ - _singletonInstance = new classname(); \ -\ - return _singletonInstance; \ - } \ -\ - static inline classname* getInstanceWithoutCreating() noexcept\ - { \ - return _singletonInstance; \ - } \ -\ - static void deleteInstance() \ - { \ - if (_singletonInstance != nullptr) \ - { \ - classname* const old = _singletonInstance; \ - _singletonInstance = nullptr; \ - delete old; \ - } \ - } \ -\ - void clearSingletonInstance() noexcept\ - { \ - if (_singletonInstance == this) \ - _singletonInstance = nullptr; \ - } - - -//============================================================================== -/** This is a counterpart to the beast_DeclareSingleton_SingleThreaded macro. - - After adding beast_DeclareSingleton_SingleThreaded or beast_DeclareSingleton_SingleThreaded_Minimal - to the class definition, this macro has to be used somewhere in the cpp file. -*/ -#define beast_ImplementSingleton_SingleThreaded(classname) \ -\ - classname* classname::_singletonInstance = nullptr; - - - -#endif // BEAST_SINGLETON_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_StaticObject.h b/Subtrees/beast/modules/beast_core/memory/beast_StaticObject.h similarity index 90% rename from Subtrees/beast/modules/beast_basics/memory/beast_StaticObject.h rename to Subtrees/beast/modules/beast_core/memory/beast_StaticObject.h index 663e42b650..d1533bee38 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_StaticObject.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_StaticObject.h @@ -20,8 +20,6 @@ #ifndef BEAST_STATICOBJECT_BEASTHEADER #define BEAST_STATICOBJECT_BEASTHEADER -#include "../threads/beast_SpinDelay.h" - // // A full suite of thread-safe objects designed for static storage duration. // @@ -122,26 +120,19 @@ char Storage ::s_storage [sizeof (ObjectType)]; class Initializer { public: - /* - bool inited () const - { - return m_state.get () == stateInitialized; - } - */ - // If the condition is not initialized, the first caller will // receive true, while concurrent callers get blocked until // initialization completes. // - bool begin () + bool beginConstruction () { - bool shouldInitialize; + bool needsInitialization = false; - if (m_state.get () == stateUninitialized) + if (m_state.get () != stateInitialized) { if (m_state.compareAndSetBool (stateInitializing, stateUninitialized)) { - shouldInitialize = true; + needsInitialization = true; } else { @@ -152,21 +143,15 @@ public: delay.pause (); } while (m_state.get () != stateInitialized); - - shouldInitialize = false; } } - else - { - shouldInitialize = false; - } - return shouldInitialize; + return needsInitialization; } // Called to signal that the initialization is complete // - void end () + void endConstruction () { m_state.set (stateInitialized); } diff --git a/Subtrees/beast/modules/beast_basics/memory/beast_Uncopyable.h b/Subtrees/beast/modules/beast_core/memory/beast_Uncopyable.h similarity index 71% rename from Subtrees/beast/modules/beast_basics/memory/beast_Uncopyable.h rename to Subtrees/beast/modules/beast_core/memory/beast_Uncopyable.h index 0ce3c4fa2b..e1f1a614b1 100644 --- a/Subtrees/beast/modules/beast_basics/memory/beast_Uncopyable.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_Uncopyable.h @@ -20,10 +20,39 @@ #ifndef BEAST_UNCOPYABLE_BEASTHEADER #define BEAST_UNCOPYABLE_BEASTHEADER -// Prevents warnings about missing copy -// constructors and assignment operators. +/** Prevent copy construction and assignment. -// Ideas based on boost + This is used to suppress warnings and prevent unsafe operations on + objects which cannot be passed by value. Ideas based on Boost. + + For example, instead of + + @code + + class MyClass + { + public: + //... + + private: + MyClass (const MyClass&); + MyClass& operator= (const MyClass&); + }; + + @endcode + + ..you can just write: + + @code + + class MyClass : Uncopyable + { + public: + //... + }; + + @endcode +*/ class Uncopyable { protected: diff --git a/Subtrees/beast/modules/beast_core/memory/beast_WeakReference.h b/Subtrees/beast/modules/beast_core/memory/beast_WeakReference.h index 192ef1a623..42d396c70f 100644 --- a/Subtrees/beast/modules/beast_core/memory/beast_WeakReference.h +++ b/Subtrees/beast/modules/beast_core/memory/beast_WeakReference.h @@ -128,7 +128,9 @@ public: in your code! @see WeakReference */ - class SharedPointer : public ReferenceCountingType + class SharedPointer + : public ReferenceCountingType + , Uncopyable { public: explicit SharedPointer (ObjectType* const obj) noexcept : owner (obj) {} @@ -138,8 +140,6 @@ public: private: ObjectType* volatile owner; - - BEAST_DECLARE_NON_COPYABLE (SharedPointer) }; typedef ReferenceCountedObjectPtr SharedRef; @@ -150,7 +150,7 @@ public: See the WeakReference class notes for an example of how to use this class. @see WeakReference */ - class Master + class Master : Uncopyable { public: Master() noexcept {} @@ -192,8 +192,6 @@ public: private: SharedRef sharedPointer; - - BEAST_DECLARE_NON_COPYABLE (Master) }; private: diff --git a/Subtrees/beast/modules/beast_core/misc/beast_Uuid.h b/Subtrees/beast/modules/beast_core/misc/beast_Uuid.h index 33f4524c0d..358b8ce444 100644 --- a/Subtrees/beast/modules/beast_core/misc/beast_Uuid.h +++ b/Subtrees/beast/modules/beast_core/misc/beast_Uuid.h @@ -37,7 +37,7 @@ The class includes methods for saving the ID as a string or as raw binary data. */ -class BEAST_API Uuid +class BEAST_API Uuid : LeakChecked { public: //============================================================================== @@ -101,8 +101,6 @@ public: private: //============================================================================== uint8 uuid[16]; - - BEAST_LEAK_DETECTOR (Uuid) }; diff --git a/Subtrees/beast/modules/beast_core/misc/beast_WindowsRegistry.h b/Subtrees/beast/modules/beast_core/misc/beast_WindowsRegistry.h index e1c1f811d8..86b95cd246 100644 --- a/Subtrees/beast/modules/beast_core/misc/beast_WindowsRegistry.h +++ b/Subtrees/beast/modules/beast_core/misc/beast_WindowsRegistry.h @@ -30,7 +30,7 @@ Contains some static helper functions for manipulating the MS Windows registry (Only available on Windows, of course!) */ -class WindowsRegistry +class WindowsRegistry : Uncopyable { public: //============================================================================== @@ -112,7 +112,6 @@ public: private: WindowsRegistry(); - BEAST_DECLARE_NON_COPYABLE (WindowsRegistry) }; #endif diff --git a/Subtrees/beast/modules/beast_core/native/beast_BasicNativeHeaders.h b/Subtrees/beast/modules/beast_core/native/beast_BasicNativeHeaders.h index 5af5a11d72..7a39aac765 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_BasicNativeHeaders.h +++ b/Subtrees/beast/modules/beast_core/native/beast_BasicNativeHeaders.h @@ -188,8 +188,19 @@ #if BEAST_BSD #include + #include + #include + #include + #include + #include #include #include + #include + #include + + // This has to be in the global namespace + extern char** environ; + #else #include #include diff --git a/Subtrees/beast/modules/beast_core/native/beast_android_Files.cpp b/Subtrees/beast/modules/beast_core/native/beast_android_Files.cpp index ed53676ac8..5e3972347b 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_android_Files.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_android_Files.cpp @@ -127,7 +127,7 @@ File File::getSpecialLocation (const SpecialLocationType type) break; } - return File::nonexistent; + return File::nonexistent (); } //============================================================================== @@ -148,7 +148,7 @@ bool File::moveToTrash() const } //============================================================================== -class DirectoryIterator::NativeIterator::Pimpl +class DirectoryIterator::NativeIterator::Pimpl : Uncopyable { public: Pimpl (const File& directory, const String& wildCard_) @@ -202,8 +202,6 @@ public: private: String parentDir, wildCard; DIR* dir; - - BEAST_DECLARE_NON_COPYABLE (Pimpl) }; diff --git a/Subtrees/beast/modules/beast_core/native/beast_android_JNIHelpers.h b/Subtrees/beast/modules/beast_core/native/beast_android_JNIHelpers.h index 3681c5c6a4..4393f17fa5 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_android_JNIHelpers.h +++ b/Subtrees/beast/modules/beast_core/native/beast_android_JNIHelpers.h @@ -167,7 +167,7 @@ namespace } //============================================================================== -class JNIClassBase +class JNIClassBase : Uncopyable { public: explicit JNIClassBase (const char* classPath); @@ -193,8 +193,6 @@ private: static Array& getClasses(); void initialise (JNIEnv*); void release (JNIEnv*); - - BEAST_DECLARE_NON_COPYABLE (JNIClassBase) }; //============================================================================== diff --git a/Subtrees/beast/modules/beast_core/native/beast_android_Network.cpp b/Subtrees/beast/modules/beast_core/native/beast_android_Network.cpp index 6fdd4f6339..86ea9596b2 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_android_Network.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_android_Network.cpp @@ -60,7 +60,9 @@ bool Process::openEmailWithAttachments (const String& targetEmailAddress, //============================================================================== -class WebInputStream : public InputStream +class WebInputStream + : public InputStream + , LeakChecked { public: //============================================================================== @@ -154,9 +156,6 @@ public: //============================================================================== GlobalRef stream; - -private: - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (WebInputStream) }; InputStream* URL::createNativeStream (const String& address, bool isPost, const MemoryBlock& postData, diff --git a/Subtrees/beast/modules/beast_core/native/beast_bsd_Files.cpp b/Subtrees/beast/modules/beast_core/native/beast_bsd_Files.cpp new file mode 100644 index 0000000000..ec8bccdb64 --- /dev/null +++ b/Subtrees/beast/modules/beast_core/native/beast_bsd_Files.cpp @@ -0,0 +1,371 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Portions of this file are from JUCE. + Copyright (c) 2013 - Raw Material Software Ltd. + Please visit http://www.juce.com + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +enum +{ + U_ISOFS_SUPER_MAGIC = 5, + U_MSDOS_SUPER_MAGIC = 2, + U_NFS_SUPER_MAGIC = 1, + U_SMB_SUPER_MAGIC = 8 +}; + +//============================================================================== +bool File::copyInternal (const File& dest) const +{ + FileInputStream in (*this); + + if (dest.deleteFile()) + { + { + FileOutputStream out (dest); + + if (out.failedToOpen()) + return false; + + if (out.writeFromInputStream (in, -1) == getSize()) + return true; + } + + dest.deleteFile(); + } + + return false; +} + +void File::findFileSystemRoots (Array& destArray) +{ + destArray.add (File ("/")); +} + +//============================================================================== +bool File::isOnCDRomDrive() const +{ + struct statfs buf; + + return statfs (getFullPathName().toUTF8(), &buf) == 0 + && buf.f_type == (short) U_ISOFS_SUPER_MAGIC; +} + +bool File::isOnHardDisk() const +{ + struct statfs buf; + + if (statfs (getFullPathName().toUTF8(), &buf) == 0) + { + switch (buf.f_type) + { + case U_ISOFS_SUPER_MAGIC: // CD-ROM + case U_MSDOS_SUPER_MAGIC: // Probably floppy (but could be mounted FAT filesystem) + case U_NFS_SUPER_MAGIC: // Network NFS + case U_SMB_SUPER_MAGIC: // Network Samba + return false; + + default: + // Assume anything else is a hard-disk (but note it could + // be a RAM disk. There isn't a good way of determining + // this for sure) + return true; + } + } + + // Assume so if this fails for some reason + return true; +} + +bool File::isOnRemovableDrive() const +{ + bassertfalse; // XXX not implemented for FreeBSD! + return false; +} + +bool File::isHidden() const +{ + return getFileName().startsWithChar ('.'); +} + +//============================================================================== +namespace +{ + File beast_readlink (const String& file, const File& defaultFile) + { + const size_t size = 8192; + HeapBlock buffer; + buffer.malloc (size + 4); + + const size_t numBytes = readlink (file.toUTF8(), buffer, size); + + if (numBytes > 0 && numBytes <= size) + return File (file).getSiblingFile (String::fromUTF8 (buffer, (int) numBytes)); + + return defaultFile; + } +} + +File File::getLinkedTarget() const +{ + return beast_readlink (getFullPathName().toUTF8(), *this); +} + +//============================================================================== +static File resolveXDGFolder (const char* const type, const char* const fallbackFolder) +{ + StringArray confLines; + File ("~/.config/user-dirs.dirs").readLines (confLines); + + for (int i = 0; i < confLines.size(); ++i) + { + const String line (confLines[i].trimStart()); + + if (line.startsWith (type)) + { + // eg. resolve XDG_MUSIC_DIR="$HOME/Music" to /home/user/Music + const File f (line.replace ("$HOME", File ("~").getFullPathName()) + .fromFirstOccurrenceOf ("=", false, false) + .trim().unquoted()); + + if (f.isDirectory()) + return f; + } + } + + return File (fallbackFolder); +} + +const char* const* beast_argv = nullptr; +int beast_argc = 0; + +File File::getSpecialLocation (const SpecialLocationType type) +{ + switch (type) + { + case userHomeDirectory: + { + const char* homeDir = getenv ("HOME"); + + if (homeDir == nullptr) + { + struct passwd* const pw = getpwuid (getuid()); + if (pw != nullptr) + homeDir = pw->pw_dir; + } + + return File (CharPointer_UTF8 (homeDir)); + } + + case userDocumentsDirectory: return resolveXDGFolder ("XDG_DOCUMENTS_DIR", "~"); + case userMusicDirectory: return resolveXDGFolder ("XDG_MUSIC_DIR", "~"); + case userMoviesDirectory: return resolveXDGFolder ("XDG_VIDEOS_DIR", "~"); + case userPicturesDirectory: return resolveXDGFolder ("XDG_PICTURES_DIR", "~"); + case userDesktopDirectory: return resolveXDGFolder ("XDG_DESKTOP_DIR", "~/Desktop"); + case userApplicationDataDirectory: return File ("~"); + case commonApplicationDataDirectory: return File ("/var"); + case globalApplicationsDirectory: return File ("/usr"); + + case tempDirectory: + { + File tmp ("/var/tmp"); + + if (! tmp.isDirectory()) + { + tmp = "/tmp"; + + if (! tmp.isDirectory()) + tmp = File::getCurrentWorkingDirectory(); + } + + return tmp; + } + + case invokedExecutableFile: + if (beast_argv != nullptr && beast_argc > 0) + return File (CharPointer_UTF8 (beast_argv[0])); + // deliberate fall-through... + + case currentExecutableFile: + case currentApplicationFile: + return beast_getExecutableFile(); + + case hostApplicationPath: + return beast_readlink ("/proc/self/exe", beast_getExecutableFile()); + + default: + bassertfalse; // unknown type? + break; + } + + return File::nonexistent (); +} + +//============================================================================== +String File::getVersion() const +{ + return String::empty; // xxx not yet implemented +} + +//============================================================================== +bool File::moveToTrash() const +{ + if (! exists()) + return true; + + File trashCan ("~/.Trash"); + + if (! trashCan.isDirectory()) + trashCan = "~/.local/share/Trash/files"; + + if (! trashCan.isDirectory()) + return false; + + return moveFileTo (trashCan.getNonexistentChildFile (getFileNameWithoutExtension(), + getFileExtension())); +} + +//============================================================================== +class DirectoryIterator::NativeIterator::Pimpl : Uncopyable +{ +public: + Pimpl (const File& directory, const String& wildCard_) + : parentDir (File::addTrailingSeparator (directory.getFullPathName())), + wildCard (wildCard_), + dir (opendir (directory.getFullPathName().toUTF8())) + { + } + + ~Pimpl() + { + if (dir != nullptr) + closedir (dir); + } + + bool next (String& filenameFound, + bool* const isDir, bool* const isHidden, int64* const fileSize, + Time* const modTime, Time* const creationTime, bool* const isReadOnly) + { + if (dir != nullptr) + { + const char* wildcardUTF8 = nullptr; + + for (;;) + { + struct dirent* const de = readdir (dir); + + if (de == nullptr) + break; + + if (wildcardUTF8 == nullptr) + wildcardUTF8 = wildCard.toUTF8(); + + if (fnmatch (wildcardUTF8, de->d_name, FNM_CASEFOLD) == 0) + { + filenameFound = CharPointer_UTF8 (de->d_name); + + updateStatInfoForFile (parentDir + filenameFound, isDir, fileSize, modTime, creationTime, isReadOnly); + + if (isHidden != nullptr) + *isHidden = filenameFound.startsWithChar ('.'); + + return true; + } + } + } + + return false; + } + +private: + String parentDir, wildCard; + DIR* dir; +}; + +DirectoryIterator::NativeIterator::NativeIterator (const File& directory, const String& wildCard) + : pimpl (new DirectoryIterator::NativeIterator::Pimpl (directory, wildCard)) +{ +} + +DirectoryIterator::NativeIterator::~NativeIterator() +{ +} + +bool DirectoryIterator::NativeIterator::next (String& filenameFound, + bool* const isDir, bool* const isHidden, int64* const fileSize, + Time* const modTime, Time* const creationTime, bool* const isReadOnly) +{ + return pimpl->next (filenameFound, isDir, isHidden, fileSize, modTime, creationTime, isReadOnly); +} + + +//============================================================================== +static bool isFileExecutable (const String& filename) +{ + beast_statStruct info; + + return beast_stat (filename, info) + && S_ISREG (info.st_mode) + && access (filename.toUTF8(), X_OK) == 0; +} + +bool Process::openDocument (const String& fileName, const String& parameters) +{ + String cmdString (fileName.replace (" ", "\\ ",false)); + cmdString << " " << parameters; + + if (URL::isProbablyAWebsiteURL (fileName) + || cmdString.startsWithIgnoreCase ("file:") + || URL::isProbablyAnEmailAddress (fileName) + || File::createFileWithoutCheckingPath (fileName).isDirectory() + || ! isFileExecutable (fileName)) + { + // create a command that tries to launch a bunch of likely browsers + const char* const browserNames[] = { "xdg-open", "firefox", "seamonkey", + "chrome", "opera", "konqueror" }; + StringArray cmdLines; + + for (int i = 0; i < numElementsInArray (browserNames); ++i) + cmdLines.add (String (browserNames[i]) + " " + cmdString.trim().quoted()); + + cmdString = cmdLines.joinIntoString (" || "); + } + + const char* const argv[4] = { "/bin/sh", "-c", cmdString.toUTF8(), 0 }; + + const int cpid = fork(); + + if (cpid == 0) + { + setsid(); + + // Child process + execve (argv[0], (char**) argv, environ); + exit (0); + } + + return cpid >= 0; +} + +void File::revealToUser() const +{ + if (isDirectory()) + startAsProcess(); + else if (getParentDirectory().exists()) + getParentDirectory().startAsProcess(); +} diff --git a/Subtrees/beast/modules/beast_core/native/beast_bsd_Network.cpp b/Subtrees/beast/modules/beast_core/native/beast_bsd_Network.cpp new file mode 100644 index 0000000000..45167cd463 --- /dev/null +++ b/Subtrees/beast/modules/beast_core/native/beast_bsd_Network.cpp @@ -0,0 +1,455 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Portions of this file are from JUCE. + Copyright (c) 2013 - Raw Material Software Ltd. + Please visit http://www.juce.com + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +void MACAddress::findAllAddresses (Array& result) +{ + ifaddrs* addrs = nullptr; + + if (getifaddrs (&addrs) == 0) + { + for (const ifaddrs* cursor = addrs; cursor != nullptr; cursor = cursor->ifa_next) + { + sockaddr_storage* sto = (sockaddr_storage*) cursor->ifa_addr; + if (sto->ss_family == AF_LINK) + { + const sockaddr_dl* const sadd = (const sockaddr_dl*) cursor->ifa_addr; + + #ifndef IFT_ETHER + #define IFT_ETHER 6 + #endif + + if (sadd->sdl_type == IFT_ETHER) + result.addIfNotAlreadyThere (MACAddress (((const uint8*) sadd->sdl_data) + sadd->sdl_nlen)); + } + } + + freeifaddrs (addrs); + } +} + + +bool Process::openEmailWithAttachments (const String& /* targetEmailAddress */, + const String& /* emailSubject */, + const String& /* bodyText */, + const StringArray& /* filesToAttach */) +{ + bassertfalse; // xxx todo + + return false; +} + + +//============================================================================== +class WebInputStream + : public InputStream + , LeakChecked +{ +public: + WebInputStream (const String& address_, bool isPost_, const MemoryBlock& postData_, + URL::OpenStreamProgressCallback* progressCallback, void* progressCallbackContext, + const String& headers_, int timeOutMs_, StringPairArray* responseHeaders) + : socketHandle (-1), levelsOfRedirection (0), + address (address_), headers (headers_), postData (postData_), position (0), + finished (false), isPost (isPost_), timeOutMs (timeOutMs_) + { + createConnection (progressCallback, progressCallbackContext); + + if (responseHeaders != nullptr && ! isError()) + { + for (int i = 0; i < headerLines.size(); ++i) + { + const String& headersEntry = headerLines[i]; + const String key (headersEntry.upToFirstOccurrenceOf (": ", false, false)); + const String value (headersEntry.fromFirstOccurrenceOf (": ", false, false)); + const String previousValue ((*responseHeaders) [key]); + responseHeaders->set (key, previousValue.isEmpty() ? value : (previousValue + "," + value)); + } + } + } + + ~WebInputStream() + { + closeSocket(); + } + + //============================================================================== + bool isError() const { return socketHandle < 0; } + bool isExhausted() { return finished; } + int64 getPosition() { return position; } + + int64 getTotalLength() + { + //xxx to do + return -1; + } + + int read (void* buffer, int bytesToRead) + { + if (finished || isError()) + return 0; + + fd_set readbits; + FD_ZERO (&readbits); + FD_SET (socketHandle, &readbits); + + struct timeval tv; + tv.tv_sec = bmax (1, timeOutMs / 1000); + tv.tv_usec = 0; + + if (select (socketHandle + 1, &readbits, 0, 0, &tv) <= 0) + return 0; // (timeout) + + const int bytesRead = bmax (0, (int) recv (socketHandle, buffer, bytesToRead, MSG_WAITALL)); + if (bytesRead == 0) + finished = true; + position += bytesRead; + return bytesRead; + } + + bool setPosition (int64 wantedPos) + { + if (isError()) + return false; + + if (wantedPos != position) + { + finished = false; + + if (wantedPos < position) + { + closeSocket(); + position = 0; + createConnection (0, 0); + } + + skipNextBytes (wantedPos - position); + } + + return true; + } + + //============================================================================== +private: + int socketHandle, levelsOfRedirection; + StringArray headerLines; + String address, headers; + MemoryBlock postData; + int64 position; + bool finished; + const bool isPost; + const int timeOutMs; + + void closeSocket() + { + if (socketHandle >= 0) + close (socketHandle); + + socketHandle = -1; + levelsOfRedirection = 0; + } + + void createConnection (URL::OpenStreamProgressCallback* progressCallback, void* progressCallbackContext) + { + closeSocket(); + + uint32 timeOutTime = Time::getMillisecondCounter(); + + if (timeOutMs == 0) + timeOutTime += 60000; + else if (timeOutMs < 0) + timeOutTime = 0xffffffff; + else + timeOutTime += timeOutMs; + + String hostName, hostPath; + int hostPort; + if (! decomposeURL (address, hostName, hostPath, hostPort)) + return; + + String serverName, proxyName, proxyPath; + int proxyPort = 0; + int port = 0; + + const String proxyURL (getenv ("http_proxy")); + if (proxyURL.startsWithIgnoreCase ("http://")) + { + if (! decomposeURL (proxyURL, proxyName, proxyPath, proxyPort)) + return; + + serverName = proxyName; + port = proxyPort; + } + else + { + serverName = hostName; + port = hostPort; + } + + struct addrinfo hints; + zerostruct (hints); + + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_NUMERICSERV; + + struct addrinfo* result = nullptr; + if (getaddrinfo (serverName.toUTF8(), String (port).toUTF8(), &hints, &result) != 0 || result == 0) + return; + + socketHandle = socket (result->ai_family, result->ai_socktype, 0); + + if (socketHandle == -1) + { + freeaddrinfo (result); + return; + } + + int receiveBufferSize = 16384; + setsockopt (socketHandle, SOL_SOCKET, SO_RCVBUF, (char*) &receiveBufferSize, sizeof (receiveBufferSize)); + setsockopt (socketHandle, SOL_SOCKET, SO_KEEPALIVE, 0, 0); + + #if BEAST_MAC + setsockopt (socketHandle, SOL_SOCKET, SO_NOSIGPIPE, 0, 0); + #endif + + if (connect (socketHandle, result->ai_addr, result->ai_addrlen) == -1) + { + closeSocket(); + freeaddrinfo (result); + return; + } + + freeaddrinfo (result); + + { + const MemoryBlock requestHeader (createRequestHeader (hostName, hostPort, proxyName, proxyPort, + hostPath, address, headers, postData, isPost)); + + if (! sendHeader (socketHandle, requestHeader, timeOutTime, progressCallback, progressCallbackContext)) + { + closeSocket(); + return; + } + } + + String responseHeader (readResponse (socketHandle, timeOutTime)); + + if (responseHeader.isNotEmpty()) + { + headerLines = StringArray::fromLines (responseHeader); + + const int statusCode = responseHeader.fromFirstOccurrenceOf (" ", false, false) + .substring (0, 3).getIntValue(); + + //int contentLength = findHeaderItem (lines, "Content-Length:").getIntValue(); + //bool isChunked = findHeaderItem (lines, "Transfer-Encoding:").equalsIgnoreCase ("chunked"); + + String location (findHeaderItem (headerLines, "Location:")); + + if (statusCode >= 300 && statusCode < 400 && location.isNotEmpty()) + { + if (! location.startsWithIgnoreCase ("http://")) + location = "http://" + location; + + if (++levelsOfRedirection <= 3) + { + address = location; + createConnection (progressCallback, progressCallbackContext); + return; + } + } + else + { + levelsOfRedirection = 0; + return; + } + } + + closeSocket(); + } + + //============================================================================== + static String readResponse (const int socketHandle, const uint32 timeOutTime) + { + int bytesRead = 0, numConsecutiveLFs = 0; + MemoryBlock buffer (1024, true); + + while (numConsecutiveLFs < 2 && bytesRead < 32768 + && Time::getMillisecondCounter() <= timeOutTime) + { + fd_set readbits; + FD_ZERO (&readbits); + FD_SET (socketHandle, &readbits); + + struct timeval tv; + tv.tv_sec = bmax (1, (int) (timeOutTime - Time::getMillisecondCounter()) / 1000); + tv.tv_usec = 0; + + if (select (socketHandle + 1, &readbits, 0, 0, &tv) <= 0) + return String::empty; // (timeout) + + buffer.ensureSize (bytesRead + 8, true); + char* const dest = (char*) buffer.getData() + bytesRead; + + if (recv (socketHandle, dest, 1, 0) == -1) + return String::empty; + + const char lastByte = *dest; + ++bytesRead; + + if (lastByte == '\n') + ++numConsecutiveLFs; + else if (lastByte != '\r') + numConsecutiveLFs = 0; + } + + const String header (CharPointer_UTF8 ((const char*) buffer.getData())); + + if (header.startsWithIgnoreCase ("HTTP/")) + return header.trimEnd(); + + return String::empty; + } + + static void writeValueIfNotPresent (MemoryOutputStream& dest, const String& headers, const String& key, const String& value) + { + if (! headers.containsIgnoreCase (key)) + dest << "\r\n" << key << ' ' << value; + } + + static void writeHost (MemoryOutputStream& dest, const bool isPost, const String& path, const String& host, const int port) + { + dest << (isPost ? "POST " : "GET ") << path << " HTTP/1.0\r\nHost: " << host; + + if (port > 0) + dest << ':' << port; + } + + static MemoryBlock createRequestHeader (const String& hostName, const int hostPort, + const String& proxyName, const int proxyPort, + const String& hostPath, const String& originalURL, + const String& userHeaders, const MemoryBlock& postData, + const bool isPost) + { + MemoryOutputStream header; + + if (proxyName.isEmpty()) + writeHost (header, isPost, hostPath, hostName, hostPort); + else + writeHost (header, isPost, originalURL, proxyName, proxyPort); + + writeValueIfNotPresent (header, userHeaders, "User-Agent:", "BEAST/" BEAST_STRINGIFY(BEAST_MAJOR_VERSION) + "." BEAST_STRINGIFY(BEAST_MINOR_VERSION) + "." BEAST_STRINGIFY(BEAST_BUILDNUMBER)); + writeValueIfNotPresent (header, userHeaders, "Connection:", "Close"); + + if (isPost) + writeValueIfNotPresent (header, userHeaders, "Content-Length:", String ((int) postData.getSize())); + + header << "\r\n" << userHeaders + << "\r\n" << postData; + + return header.getMemoryBlock(); + } + + static bool sendHeader (int socketHandle, const MemoryBlock& requestHeader, const uint32 timeOutTime, + URL::OpenStreamProgressCallback* progressCallback, void* progressCallbackContext) + { + size_t totalHeaderSent = 0; + + while (totalHeaderSent < requestHeader.getSize()) + { + if (Time::getMillisecondCounter() > timeOutTime) + return false; + + const int numToSend = bmin (1024, (int) (requestHeader.getSize() - totalHeaderSent)); + + if (send (socketHandle, static_cast (requestHeader.getData()) + totalHeaderSent, numToSend, 0) != numToSend) + return false; + + totalHeaderSent += numToSend; + + if (progressCallback != nullptr && ! progressCallback (progressCallbackContext, totalHeaderSent, requestHeader.getSize())) + return false; + } + + return true; + } + + static bool decomposeURL (const String& url, String& host, String& path, int& port) + { + if (! url.startsWithIgnoreCase ("http://")) + return false; + + const int nextSlash = url.indexOfChar (7, '/'); + int nextColon = url.indexOfChar (7, ':'); + if (nextColon > nextSlash && nextSlash > 0) + nextColon = -1; + + if (nextColon >= 0) + { + host = url.substring (7, nextColon); + + if (nextSlash >= 0) + port = url.substring (nextColon + 1, nextSlash).getIntValue(); + else + port = url.substring (nextColon + 1).getIntValue(); + } + else + { + port = 80; + + if (nextSlash >= 0) + host = url.substring (7, nextSlash); + else + host = url.substring (7); + } + + if (nextSlash >= 0) + path = url.substring (nextSlash); + else + path = "/"; + + return true; + } + + static String findHeaderItem (const StringArray& lines, const String& itemName) + { + for (int i = 0; i < lines.size(); ++i) + if (lines[i].startsWithIgnoreCase (itemName)) + return lines[i].substring (itemName.length()).trim(); + + return String::empty; + } +}; + +InputStream* URL::createNativeStream (const String& address, bool isPost, const MemoryBlock& postData, + OpenStreamProgressCallback* progressCallback, void* progressCallbackContext, + const String& headers, const int timeOutMs, StringPairArray* responseHeaders) +{ + ScopedPointer wi (new WebInputStream (address, isPost, postData, + progressCallback, progressCallbackContext, + headers, timeOutMs, responseHeaders)); + + return wi->isError() ? nullptr : wi.release(); +} diff --git a/Subtrees/beast/modules/beast_core/native/beast_bsd_SystemStats.cpp b/Subtrees/beast/modules/beast_core/native/beast_bsd_SystemStats.cpp new file mode 100644 index 0000000000..613ca15296 --- /dev/null +++ b/Subtrees/beast/modules/beast_core/native/beast_bsd_SystemStats.cpp @@ -0,0 +1,346 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Portions of this file are from JUCE. + Copyright (c) 2013 - Raw Material Software Ltd. + Please visit http://www.juce.com + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +// sysinfo() from sysinfo-bsd +// https://code.google.com/p/sysinfo-bsd/ +/* + * Copyright (C) 2010 Kostas Petrikas, All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name(s) of the author(s) may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#define SI_LOAD_SHIFT 16 +struct sysinfo { + long uptime; /* Seconds since boot */ + unsigned long loads[3]; /* 1, 5, and 15 minute load averages */ + unsigned long totalram; /* Total usable main memory size */ + unsigned long freeram; /* Available memory size */ + unsigned long sharedram; /* Amount of shared memory */ + unsigned long bufferram; /* Memory used by buffers */ + unsigned long totalswap; /* Total swap space size */ + unsigned long freeswap; /* swap space still available */ + unsigned short procs; /* Number of current processes */ + unsigned short pad; /* leaving this for linux compatability */ + unsigned long totalhigh; /* Total high memory size */ + unsigned long freehigh; /* Available high memory size */ + unsigned int mem_unit; /* Memory unit size in bytes */ + char _f[20-2*sizeof(long)-sizeof(int)]; /* leaving this for linux compatability */ +}; + +#define NLOADS 3 +#define UNIT_S 1024 /*Kb*/ +#define R_IGNORE -1 + +/*the macros*/ +#define R_ERROR(_EC) {if(_EC > R_IGNORE)errno = _EC; return -1;} +#define GETSYSCTL(name, var) getsysctl((char*)name, &(var), sizeof(var)) +#define PAGE_2_UNIT(_PAGE) (((double)_PAGE * page_s) / UNIT_S) + +/*sysctl wrapper*/ +static int getsysctl(char *name, void *ptr, size_t len){ + size_t nlen = len; + if (sysctlbyname(name, ptr, &nlen, NULL, 0) == -1) + return -1; + + if (nlen != len) + return -1; + + return 0; +} + +int sysinfo(struct sysinfo *info){ + kvm_t *kvmh; + double load_avg[NLOADS]; + int page_s = getpagesize(); + + if (info == NULL) + R_ERROR(EFAULT); + + memset(info, 0, sizeof(struct sysinfo)); + info -> mem_unit = UNIT_S; + + /*kvm init*/ + if ((kvmh = kvm_open(NULL, "/dev/null", "/dev/null", + O_RDONLY, "kvm_open")) == NULL) + R_ERROR(0); + + /*load averages*/ + if (kvm_getloadavg(kvmh, load_avg, NLOADS) == -1) + R_ERROR(0); + + info -> loads[0] = (u_long)((float)load_avg[0] * USHRT_MAX); + info -> loads[1] = (u_long)((float)load_avg[1] * USHRT_MAX); + info -> loads[2] = (u_long)((float)load_avg[2] * USHRT_MAX); + + /*swap space*/ + struct kvm_swap k_swap; + + if (kvm_getswapinfo(kvmh, &k_swap, 1, 0) == -1) + R_ERROR(0); + + info -> totalswap = + (u_long)PAGE_2_UNIT(k_swap.ksw_total); + info -> freeswap = info -> totalswap - + (u_long)PAGE_2_UNIT(k_swap.ksw_used); + + /*processes*/ + int n_procs; + + if (kvm_getprocs(kvmh, KERN_PROC_ALL, 0, &n_procs) == NULL) + R_ERROR(0); + + info -> procs = (u_short)n_procs; + + /*end of kvm session*/ + if (kvm_close(kvmh) == -1) + R_ERROR(0); + + /*uptime*/ + struct timespec ts; + + if (clock_gettime(CLOCK_UPTIME, &ts) == -1) + R_ERROR(R_IGNORE); + + info -> uptime = (long)ts.tv_sec; + + /*ram*/ + int total_pages, + free_pages, + active_pages, + inactive_pages; + u_long shmmax; + + if (GETSYSCTL("vm.stats.vm.v_page_count", total_pages) == -1) + R_ERROR(R_IGNORE); + if (GETSYSCTL("vm.stats.vm.v_free_count", free_pages) == -1) + R_ERROR(R_IGNORE); + if (GETSYSCTL("vm.stats.vm.v_active_count", active_pages) == -1) + R_ERROR(R_IGNORE); + if (GETSYSCTL("vm.stats.vm.v_inactive_count", inactive_pages) == -1) + R_ERROR(R_IGNORE); + if (GETSYSCTL("kern.ipc.shmmax", shmmax) == -1) + R_ERROR(R_IGNORE); + + info -> totalram = (u_long)PAGE_2_UNIT(total_pages); + info -> freeram = (u_long)PAGE_2_UNIT(free_pages); + info -> bufferram = (u_long)PAGE_2_UNIT(active_pages); + info -> sharedram = shmmax / UNIT_S; + + /*high mem (todo)*/ + info -> totalhigh = 0; /*Does this supose to refer to HMA or reserved ram?*/ + info -> freehigh = 0; + + return 0; +} + +//============================================================================== + +void Logger::outputDebugString (const String& text) +{ + std::cerr << text << std::endl; +} + +//============================================================================== +SystemStats::OperatingSystemType SystemStats::getOperatingSystemType() +{ + return FreeBSD; +} + +String SystemStats::getOperatingSystemName() +{ + return "FreeBSD"; +} + +bool SystemStats::isOperatingSystem64Bit() + +{ + #if BEAST_64BIT + return true; + #else + //xxx not sure how to find this out?.. + return false; + #endif +} + +//============================================================================== +namespace BSDStatsHelpers +{ + String getCpuInfo (const char* const key) + { + StringArray lines; + File ("/proc/cpuinfo").readLines (lines); + + for (int i = lines.size(); --i >= 0;) // (NB - it's important that this runs in reverse order) + if (lines[i].startsWithIgnoreCase (key)) + return lines[i].fromFirstOccurrenceOf (":", false, false).trim(); + + return String::empty; + } +} + +String SystemStats::getCpuVendor() +{ + return BSDStatsHelpers::getCpuInfo ("vendor_id"); +} + +int SystemStats::getCpuSpeedInMegaherz() +{ + return roundToInt (BSDStatsHelpers::getCpuInfo ("cpu MHz").getFloatValue()); +} + +int SystemStats::getMemorySizeInMegabytes() +{ + struct sysinfo sysi; + + if (sysinfo (&sysi) == 0) + return (sysi.totalram * sysi.mem_unit / (1024 * 1024)); + + return 0; +} + +int SystemStats::getPageSize() +{ + return sysconf (_SC_PAGESIZE); +} + +//============================================================================== +String SystemStats::getLogonName() +{ + const char* user = getenv ("USER"); + + if (user == nullptr) + { + struct passwd* const pw = getpwuid (getuid()); + if (pw != nullptr) + user = pw->pw_name; + } + + return CharPointer_UTF8 (user); +} + +String SystemStats::getFullUserName() +{ + return getLogonName(); +} + +String SystemStats::getComputerName() +{ + char name [256] = { 0 }; + if (gethostname (name, sizeof (name) - 1) == 0) + return name; + + return String::empty; +} + +String getLocaleValue (nl_item key) +{ + const char* oldLocale = ::setlocale (LC_ALL, ""); + return String (const_cast (nl_langinfo (key))); + ::setlocale (LC_ALL, oldLocale); +} + +String SystemStats::getUserLanguage() +{ + return "Uknown user language"; +} + +String SystemStats::getUserRegion() +{ + return "Unknown user region"; +} + +String SystemStats::getDisplayLanguage() +{ + return getUserLanguage(); +} + +//============================================================================== +SystemStats::CPUFlags::CPUFlags() +{ + const String flags (BSDStatsHelpers::getCpuInfo ("flags")); + hasMMX = flags.contains ("mmx"); + hasSSE = flags.contains ("sse"); + hasSSE2 = flags.contains ("sse2"); + has3DNow = flags.contains ("3dnow"); + + numCpus = BSDStatsHelpers::getCpuInfo ("processor").getIntValue() + 1; +} + +//============================================================================== +uint32 beast_millisecondsSinceStartup() noexcept +{ + timespec t; + clock_gettime (CLOCK_MONOTONIC, &t); + + return t.tv_sec * 1000 + t.tv_nsec / 1000000; +} + +int64 Time::getHighResolutionTicks() noexcept +{ + timespec t; + clock_gettime (CLOCK_MONOTONIC, &t); + + return (t.tv_sec * (int64) 1000000) + (t.tv_nsec / 1000); +} + +int64 Time::getHighResolutionTicksPerSecond() noexcept +{ + return 1000000; // (microseconds) +} + +double Time::getMillisecondCounterHiRes() noexcept +{ + return getHighResolutionTicks() * 0.001; +} + +bool Time::setSystemTimeToThisTime() const +{ + timeval t; + t.tv_sec = millisSinceEpoch / 1000; + t.tv_usec = (millisSinceEpoch - t.tv_sec * 1000) * 1000; + + return settimeofday (&t, 0) == 0; +} diff --git a/Subtrees/beast/modules/beast_core/native/beast_bsd_Threads.cpp b/Subtrees/beast/modules/beast_core/native/beast_bsd_Threads.cpp new file mode 100644 index 0000000000..1255f375ef --- /dev/null +++ b/Subtrees/beast/modules/beast_core/native/beast_bsd_Threads.cpp @@ -0,0 +1,74 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Portions of this file are from JUCE. + Copyright (c) 2013 - Raw Material Software Ltd. + Please visit http://www.juce.com + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +/* + Note that a lot of methods that you'd expect to find in this file actually + live in beast_posix_SharedCode.h! +*/ + +//============================================================================== +void Process::setPriority (const ProcessPriority prior) +{ + const int policy = (prior <= NormalPriority) ? SCHED_OTHER : SCHED_RR; + const int minp = sched_get_priority_min (policy); + const int maxp = sched_get_priority_max (policy); + + struct sched_param param; + + switch (prior) + { + case LowPriority: + case NormalPriority: param.sched_priority = 0; break; + case HighPriority: param.sched_priority = minp + (maxp - minp) / 4; break; + case RealtimePriority: param.sched_priority = minp + (3 * (maxp - minp) / 4); break; + default: bassertfalse; break; + } + + pthread_setschedparam (pthread_self(), policy, ¶m); +} + +void Process::terminate() +{ + exit (0); +} + +BEAST_API bool BEAST_CALLTYPE beast_isRunningUnderDebugger() +{ + // XXX not implemented for FreeBSD! + bassertfalse; + return false; +} + +BEAST_API bool BEAST_CALLTYPE Process::isRunningUnderDebugger() +{ + return beast_isRunningUnderDebugger(); +} + +static void swapUserAndEffectiveUser() +{ + (void) setreuid (geteuid(), getuid()); + (void) setregid (getegid(), getgid()); +} + +void Process::raisePrivilege() { if (geteuid() != 0 && getuid() == 0) swapUserAndEffectiveUser(); } +void Process::lowerPrivilege() { if (geteuid() == 0 && getuid() != 0) swapUserAndEffectiveUser(); } diff --git a/Subtrees/beast/modules/beast_core/native/beast_linux_Files.cpp b/Subtrees/beast/modules/beast_core/native/beast_linux_Files.cpp index 6487f80fb3..e29f551dca 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_linux_Files.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_linux_Files.cpp @@ -213,7 +213,7 @@ File File::getSpecialLocation (const SpecialLocationType type) break; } - return File::nonexistent; + return File::nonexistent (); } //============================================================================== @@ -241,7 +241,7 @@ bool File::moveToTrash() const } //============================================================================== -class DirectoryIterator::NativeIterator::Pimpl +class DirectoryIterator::NativeIterator::Pimpl : Uncopyable { public: Pimpl (const File& directory, const String& wildCard_) @@ -295,8 +295,6 @@ public: private: String parentDir, wildCard; DIR* dir; - - BEAST_DECLARE_NON_COPYABLE (Pimpl) }; DirectoryIterator::NativeIterator::NativeIterator (const File& directory, const String& wildCard) diff --git a/Subtrees/beast/modules/beast_core/native/beast_linux_Network.cpp b/Subtrees/beast/modules/beast_core/native/beast_linux_Network.cpp index 33b8012e7d..137718093c 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_linux_Network.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_linux_Network.cpp @@ -62,7 +62,9 @@ bool Process::openEmailWithAttachments (const String& /* targetEmailAddress */, //============================================================================== -class WebInputStream : public InputStream +class WebInputStream + : public InputStream + , LeakChecked { public: WebInputStream (const String& address_, bool isPost_, const MemoryBlock& postData_, @@ -441,8 +443,6 @@ private: return String::empty; } - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (WebInputStream) }; InputStream* URL::createNativeStream (const String& address, bool isPost, const MemoryBlock& postData, diff --git a/Subtrees/beast/modules/beast_core/native/beast_linux_SystemStats.cpp b/Subtrees/beast/modules/beast_core/native/beast_linux_SystemStats.cpp index 3764f477c4..32a6d92e7f 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_linux_SystemStats.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_linux_SystemStats.cpp @@ -78,7 +78,7 @@ int SystemStats::getMemorySizeInMegabytes() struct sysinfo sysi; if (sysinfo (&sysi) == 0) - return (sysi.totalram * sysi.mem_unit / (1024 * 1024)); + return sysi.totalram * sysi.mem_unit / (1024 * 1024); return 0; } @@ -94,11 +94,8 @@ String SystemStats::getLogonName() const char* user = getenv ("USER"); if (user == nullptr) - { - struct passwd* const pw = getpwuid (getuid()); - if (pw != nullptr) + if (passwd* const pw = getpwuid (getuid())) user = pw->pw_name; - } return CharPointer_UTF8 (user); } @@ -117,11 +114,12 @@ String SystemStats::getComputerName() return String::empty; } -String getLocaleValue (nl_item key) +static String getLocaleValue (nl_item key) { const char* oldLocale = ::setlocale (LC_ALL, ""); - return String (const_cast (nl_langinfo (key))); + String result (String::fromUTF8 (nl_langinfo (key))); ::setlocale (LC_ALL, oldLocale); + return result; } String SystemStats::getUserLanguage() { return getLocaleValue (_NL_IDENTIFICATION_LANGUAGE); } diff --git a/Subtrees/beast/modules/beast_core/native/beast_mac_Files.mm b/Subtrees/beast/modules/beast_core/native/beast_mac_Files.mm index 020a06514c..183492670c 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_mac_Files.mm +++ b/Subtrees/beast/modules/beast_core/native/beast_mac_Files.mm @@ -251,7 +251,7 @@ File File::getSpecialLocation (const SpecialLocationType type) return File (resultPath.convertToPrecomposedUnicode()); } - return File::nonexistent; + return File::nonexistent (); } //============================================================================== @@ -308,7 +308,7 @@ bool File::moveToTrash() const } //============================================================================== -class DirectoryIterator::NativeIterator::Pimpl +class DirectoryIterator::NativeIterator::Pimpl : Uncopyable { public: Pimpl (const File& directory, const String& wildCard_) @@ -364,8 +364,6 @@ public: private: String parentDir, wildCard; NSDirectoryEnumerator* enumerator; - - BEAST_DECLARE_NON_COPYABLE (Pimpl) }; DirectoryIterator::NativeIterator::NativeIterator (const File& directory, const String& wildcard) diff --git a/Subtrees/beast/modules/beast_core/native/beast_mac_Network.mm b/Subtrees/beast/modules/beast_core/native/beast_mac_Network.mm index ddba3b886e..8f3410185e 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_mac_Network.mm +++ b/Subtrees/beast/modules/beast_core/native/beast_mac_Network.mm @@ -97,7 +97,10 @@ bool Process::openEmailWithAttachments (const String& targetEmailAddress, } //============================================================================== -class URLConnectionState : public Thread +class URLConnectionState + : public Thread + , LeakChecked + , Uncopyable { public: URLConnectionState (NSURLRequest* req) @@ -290,13 +293,13 @@ private: getState (self)->finishedLoading(); } }; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (URLConnectionState) }; //============================================================================== -class WebInputStream : public InputStream +class WebInputStream + : public InputStream + , LeakChecked { public: WebInputStream (const String& address_, bool isPost_, const MemoryBlock& postData_, @@ -411,8 +414,6 @@ private: connection = nullptr; } } - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (WebInputStream) }; InputStream* URL::createNativeStream (const String& address, bool isPost, const MemoryBlock& postData, diff --git a/Subtrees/beast/modules/beast_core/native/beast_osx_ObjCHelpers.h b/Subtrees/beast/modules/beast_core/native/beast_osx_ObjCHelpers.h index d565e9d845..1f0c10e2e6 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_osx_ObjCHelpers.h +++ b/Subtrees/beast/modules/beast_core/native/beast_osx_ObjCHelpers.h @@ -64,7 +64,7 @@ struct NSObjectRetainer //============================================================================== template -struct ObjCClass +struct ObjCClass : Uncopyable { ObjCClass (const char* nameRoot) : cls (objc_allocateClassPair ([SuperclassType class], getRandomisedName (nameRoot).toUTF8(), 0)) @@ -145,8 +145,6 @@ private: { return root + String::toHexString (beast::Random::getSystemRandom().nextInt64()); } - - BEAST_DECLARE_NON_COPYABLE (ObjCClass) }; diff --git a/Subtrees/beast/modules/beast_basics/native/beast_posix_FPUFlags.cpp b/Subtrees/beast/modules/beast_core/native/beast_posix_FPUFlags.cpp similarity index 100% rename from Subtrees/beast/modules/beast_basics/native/beast_posix_FPUFlags.cpp rename to Subtrees/beast/modules/beast_core/native/beast_posix_FPUFlags.cpp diff --git a/Subtrees/beast/modules/beast_core/native/beast_posix_NamedPipe.cpp b/Subtrees/beast/modules/beast_core/native/beast_posix_NamedPipe.cpp index ed5a6b1f0a..07c2725d6e 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_posix_NamedPipe.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_posix_NamedPipe.cpp @@ -21,7 +21,7 @@ */ //============================================================================== -class NamedPipe::Pimpl +class NamedPipe::Pimpl : LeakChecked , Uncopyable { public: Pimpl (const String& pipePath, bool createPipe) @@ -32,7 +32,7 @@ public: stopReadOperation (false) { signal (SIGPIPE, signalHandler); - siginterrupt (SIGPIPE, 1); + beast_siginterrupt (SIGPIPE, 1); } ~Pimpl() @@ -164,8 +164,6 @@ private: select (handle + 1, &rset, nullptr, 0, &timeout); } - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (Pimpl) }; void NamedPipe::close() diff --git a/Subtrees/beast/modules/beast_core/native/beast_posix_SharedCode.h b/Subtrees/beast/modules/beast_core/native/beast_posix_SharedCode.h index 56d017b975..936be35f40 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_posix_SharedCode.h +++ b/Subtrees/beast/modules/beast_core/native/beast_posix_SharedCode.h @@ -174,6 +174,21 @@ bool File::setAsCurrentWorkingDirectory() const return chdir (getFullPathName().toUTF8()) == 0; } +//============================================================================== +// The unix siginterrupt function is deprecated - this does the same job. +int beast_siginterrupt (int sig, int flag) +{ + struct ::sigaction act; + (void) ::sigaction (sig, nullptr, &act); + + if (flag != 0) + act.sa_flags &= ~SA_RESTART; + else + act.sa_flags |= SA_RESTART; + + return ::sigaction (sig, &act, nullptr); +} + //============================================================================== namespace { @@ -973,7 +988,7 @@ void* DynamicLibrary::getFunction (const String& functionName) noexcept //============================================================================== -class ChildProcess::ActiveProcess +class ChildProcess::ActiveProcess : LeakChecked , Uncopyable { public: ActiveProcess (const StringArray& arguments) @@ -1066,8 +1081,6 @@ public: private: int pipeHandle; FILE* readHandle; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ActiveProcess) }; bool ChildProcess::start (const String& command) @@ -1104,7 +1117,7 @@ bool ChildProcess::kill() } //============================================================================== -struct HighResolutionTimer::Pimpl +struct HighResolutionTimer::Pimpl : Uncopyable { Pimpl (HighResolutionTimer& t) : owner (t), thread (0), shouldStop (false) { @@ -1192,7 +1205,7 @@ private: uint64_t time, delta; - #elif BEAST_ANDROID + #elif BEAST_ANDROID || BEAST_BSD Clock (double millis) noexcept : delta ((uint64) (millis * 1000000)) { } @@ -1221,12 +1234,7 @@ private: struct timespec t; t.tv_sec = (time_t) (time / 1000000000); t.tv_nsec = (long) (time % 1000000000); - -#if BEAST_BSD - bassertfalse; // unimplemented -#else clock_nanosleep (CLOCK_MONOTONIC, TIMER_ABSTIME, &t, nullptr); -#endif } uint64 time, delta; @@ -1255,6 +1263,4 @@ private: #endif } - - BEAST_DECLARE_NON_COPYABLE (Pimpl) }; diff --git a/Subtrees/beast/modules/beast_basics/native/beast_win32_FPUFlags.cpp b/Subtrees/beast/modules/beast_core/native/beast_win32_FPUFlags.cpp similarity index 100% rename from Subtrees/beast/modules/beast_basics/native/beast_win32_FPUFlags.cpp rename to Subtrees/beast/modules/beast_core/native/beast_win32_FPUFlags.cpp diff --git a/Subtrees/beast/modules/beast_core/native/beast_win32_Files.cpp b/Subtrees/beast/modules/beast_core/native/beast_win32_Files.cpp index 56f17582f5..444bc51c3e 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_win32_Files.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_win32_Files.cpp @@ -88,7 +88,7 @@ namespace WindowsFileHelpers if (SHGetSpecialFolderPath (0, path, type, FALSE)) return File (String (path)); - return File::nonexistent; + return File::nonexistent (); } File getModuleFileName (HINSTANCE moduleHandle) @@ -536,7 +536,7 @@ File BEAST_CALLTYPE File::getSpecialLocation (const SpecialLocationType type) default: bassertfalse; // unknown type? - return File::nonexistent; + return File::nonexistent (); } return WindowsFileHelpers::getSpecialFolderPath (csidlType); @@ -628,6 +628,8 @@ bool File::createLink (const String& description, const File& linkFileToCreate) //============================================================================== class DirectoryIterator::NativeIterator::Pimpl + : LeakChecked + , Uncopyable { public: Pimpl (const File& directory, const String& wildCard) @@ -677,8 +679,6 @@ public: private: const String directoryWithWildCard; HANDLE handle; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (Pimpl) }; DirectoryIterator::NativeIterator::NativeIterator (const File& directory, const String& wildCard) @@ -731,7 +731,7 @@ void File::revealToUser() const } //============================================================================== -class NamedPipe::Pimpl +class NamedPipe::Pimpl : LeakChecked , Uncopyable { public: Pimpl (const String& pipeName, const bool createPipe) @@ -914,8 +914,6 @@ private: CancelIo (pipeH); return false; } - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (Pimpl) }; void NamedPipe::close() diff --git a/Subtrees/beast/modules/beast_core/native/beast_win32_Misc.cpp b/Subtrees/beast/modules/beast_core/native/beast_win32_Misc.cpp new file mode 100644 index 0000000000..4a2f785a22 --- /dev/null +++ b/Subtrees/beast/modules/beast_core/native/beast_win32_Misc.cpp @@ -0,0 +1,648 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Portions of this file are from JUCE. + Copyright (c) 2013 - Raw Material Software Ltd. + Please visit http://www.juce.com + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +HWND beast_messageWindowHandle = 0; // (this is used by other parts of the codebase) + +//============================================================================== +#if ! BEAST_USE_INTRINSICS +// In newer compilers, the inline versions of these are used (in beast_Atomic.h), but in +// older ones we have to actually call the ops as win32 functions.. +long beast_InterlockedExchange (volatile long* a, long b) noexcept { return InterlockedExchange (a, b); } +long beast_InterlockedIncrement (volatile long* a) noexcept { return InterlockedIncrement (a); } +long beast_InterlockedDecrement (volatile long* a) noexcept { return InterlockedDecrement (a); } +long beast_InterlockedExchangeAdd (volatile long* a, long b) noexcept { return InterlockedExchangeAdd (a, b); } +long beast_InterlockedCompareExchange (volatile long* a, long b, long c) noexcept { return InterlockedCompareExchange (a, b, c); } + +__int64 beast_InterlockedCompareExchange64 (volatile __int64* value, __int64 newValue, __int64 valueToCompare) noexcept +{ + bassertfalse; // This operation isn't available in old MS compiler versions! + + __int64 oldValue = *value; + if (oldValue == valueToCompare) + *value = newValue; + + return oldValue; +} + +#endif + +//============================================================================== +CriticalSection::CriticalSection() noexcept +{ + // (just to check the MS haven't changed this structure and broken things...) + #if BEAST_VC7_OR_EARLIER + static_bassert (sizeof (CRITICAL_SECTION) <= 24); + #else + static_bassert (sizeof (CRITICAL_SECTION) <= sizeof (internal)); + #endif + + InitializeCriticalSection ((CRITICAL_SECTION*) internal); +} + +CriticalSection::~CriticalSection() noexcept +{ + DeleteCriticalSection ((CRITICAL_SECTION*) internal); +} + +void CriticalSection::enter() const noexcept +{ + EnterCriticalSection ((CRITICAL_SECTION*) internal); +} + +bool CriticalSection::tryEnter() const noexcept +{ + return TryEnterCriticalSection ((CRITICAL_SECTION*) internal) != FALSE; +} + +void CriticalSection::exit() const noexcept +{ + LeaveCriticalSection ((CRITICAL_SECTION*) internal); +} + +//============================================================================== +WaitableEvent::WaitableEvent (const bool manualReset) noexcept + : internal (CreateEvent (0, manualReset ? TRUE : FALSE, FALSE, 0)) +{ +} + +WaitableEvent::~WaitableEvent() noexcept +{ + CloseHandle (internal); +} + +bool WaitableEvent::wait (const int timeOutMillisecs) const noexcept +{ + return WaitForSingleObject (internal, (DWORD) timeOutMillisecs) == WAIT_OBJECT_0; +} + +void WaitableEvent::signal() const noexcept +{ + SetEvent (internal); +} + +void WaitableEvent::reset() const noexcept +{ + ResetEvent (internal); +} + +//============================================================================== +void BEAST_API beast_threadEntryPoint (void*); + +static unsigned int __stdcall threadEntryProc (void* userData) +{ + if (beast_messageWindowHandle != 0) + AttachThreadInput (GetWindowThreadProcessId (beast_messageWindowHandle, 0), + GetCurrentThreadId(), TRUE); + + beast_threadEntryPoint (userData); + + _endthreadex (0); + return 0; +} + +void Thread::launchThread() +{ + unsigned int newThreadId; + threadHandle = (void*) _beginthreadex (0, 0, &threadEntryProc, this, 0, &newThreadId); + threadId = (ThreadID) newThreadId; +} + +void Thread::closeThreadHandle() +{ + CloseHandle ((HANDLE) threadHandle); + threadId = 0; + threadHandle = 0; +} + +void Thread::killThread() +{ + if (threadHandle != 0) + { + #if BEAST_DEBUG + OutputDebugStringA ("** Warning - Forced thread termination **\n"); + #endif + TerminateThread (threadHandle, 0); + } +} + +void Thread::setCurrentThreadName (const String& name) +{ + #if BEAST_DEBUG && BEAST_MSVC + struct + { + DWORD dwType; + LPCSTR szName; + DWORD dwThreadID; + DWORD dwFlags; + } info; + + info.dwType = 0x1000; + info.szName = name.toUTF8(); + info.dwThreadID = GetCurrentThreadId(); + info.dwFlags = 0; + + __try + { + RaiseException (0x406d1388 /*MS_VC_EXCEPTION*/, 0, sizeof (info) / sizeof (ULONG_PTR), (ULONG_PTR*) &info); + } + __except (EXCEPTION_CONTINUE_EXECUTION) + {} + #else + (void) name; + #endif +} + +Thread::ThreadID Thread::getCurrentThreadId() +{ + return (ThreadID) (pointer_sized_int) GetCurrentThreadId(); +} + +bool Thread::setThreadPriority (void* handle, int priority) +{ + int pri = THREAD_PRIORITY_TIME_CRITICAL; + + if (priority < 1) pri = THREAD_PRIORITY_IDLE; + else if (priority < 2) pri = THREAD_PRIORITY_LOWEST; + else if (priority < 5) pri = THREAD_PRIORITY_BELOW_NORMAL; + else if (priority < 7) pri = THREAD_PRIORITY_NORMAL; + else if (priority < 9) pri = THREAD_PRIORITY_ABOVE_NORMAL; + else if (priority < 10) pri = THREAD_PRIORITY_HIGHEST; + + if (handle == 0) + handle = GetCurrentThread(); + + return SetThreadPriority (handle, pri) != FALSE; +} + +void Thread::setCurrentThreadAffinityMask (const uint32 affinityMask) +{ + SetThreadAffinityMask (GetCurrentThread(), affinityMask); +} + +//============================================================================== +struct SleepEvent +{ + SleepEvent() noexcept + : handle (CreateEvent (nullptr, FALSE, FALSE, + #if BEAST_DEBUG + _T("BEAST Sleep Event"))) + #else + nullptr)) + #endif + {} + + ~SleepEvent() noexcept + { + CloseHandle (handle); + handle = 0; + } + + HANDLE handle; +}; + +static SleepEvent sleepEvent; + +void BEAST_CALLTYPE Thread::sleep (const int millisecs) +{ + if (millisecs >= 10 || sleepEvent.handle == 0) + { + Sleep ((DWORD) millisecs); + } + else + { + // unlike Sleep() this is guaranteed to return to the current thread after + // the time expires, so we'll use this for short waits, which are more likely + // to need to be accurate + WaitForSingleObject (sleepEvent.handle, (DWORD) millisecs); + } +} + +void Thread::yield() +{ + Sleep (0); +} + +//============================================================================== +static int lastProcessPriority = -1; + +// called by WindowDriver because Windows does weird things to process priority +// when you swap apps, and this forces an update when the app is brought to the front. +void beast_repeatLastProcessPriority() +{ + if (lastProcessPriority >= 0) // (avoid changing this if it's not been explicitly set by the app..) + { + DWORD p; + + switch (lastProcessPriority) + { + case Process::LowPriority: p = IDLE_PRIORITY_CLASS; break; + case Process::NormalPriority: p = NORMAL_PRIORITY_CLASS; break; + case Process::HighPriority: p = HIGH_PRIORITY_CLASS; break; + case Process::RealtimePriority: p = REALTIME_PRIORITY_CLASS; break; + default: bassertfalse; return; // bad priority value + } + + SetPriorityClass (GetCurrentProcess(), p); + } +} + +void Process::setPriority (ProcessPriority prior) +{ + if (lastProcessPriority != (int) prior) + { + lastProcessPriority = (int) prior; + beast_repeatLastProcessPriority(); + } +} + +BEAST_API bool BEAST_CALLTYPE beast_isRunningUnderDebugger() +{ + return IsDebuggerPresent() != FALSE; +} + +bool BEAST_CALLTYPE Process::isRunningUnderDebugger() +{ + return beast_isRunningUnderDebugger(); +} + +BEAST_API void BEAST_CALLTYPE Process::breakPoint () +{ +#if BEAST_DEBUG + if (beast_isRunningUnderDebugger ()) + beast_breakDebugger; + +#else + bassertfalse; + +#endif +} + +//------------------------------------------------------------------------------ + +static void* currentModuleHandle = nullptr; + +void* Process::getCurrentModuleInstanceHandle() noexcept +{ + if (currentModuleHandle == nullptr) + currentModuleHandle = GetModuleHandleA (nullptr); + + return currentModuleHandle; +} + +void Process::setCurrentModuleInstanceHandle (void* const newHandle) noexcept +{ + currentModuleHandle = newHandle; +} + +void Process::raisePrivilege() +{ + bassertfalse; // xxx not implemented +} + +void Process::lowerPrivilege() +{ + bassertfalse; // xxx not implemented +} + +void Process::terminate() +{ + #if BEAST_MSVC && BEAST_CHECK_MEMORY_LEAKS + _CrtDumpMemoryLeaks(); + #endif + + // bullet in the head in case there's a problem shutting down.. + ExitProcess (0); +} + +bool beast_isRunningInWine() +{ + HMODULE ntdll = GetModuleHandleA ("ntdll"); + return ntdll != 0 && GetProcAddress (ntdll, "wine_get_version") != nullptr; +} + +//============================================================================== +bool DynamicLibrary::open (const String& name) +{ + close(); + + BEAST_TRY + { + handle = LoadLibrary (name.toWideCharPointer()); + } + BEAST_CATCH_ALL + + return handle != nullptr; +} + +void DynamicLibrary::close() +{ + BEAST_TRY + { + if (handle != nullptr) + { + FreeLibrary ((HMODULE) handle); + handle = nullptr; + } + } + BEAST_CATCH_ALL +} + +void* DynamicLibrary::getFunction (const String& functionName) noexcept +{ + return handle != nullptr ? (void*) GetProcAddress ((HMODULE) handle, functionName.toUTF8()) // (void* cast is required for mingw) + : nullptr; +} + + +//============================================================================== +class InterProcessLock::Pimpl +{ +public: + Pimpl (String name, const int timeOutMillisecs) + : handle (0), refCount (1) + { + name = name.replaceCharacter ('\\', '/'); + handle = CreateMutexW (0, TRUE, ("Global\\" + name).toWideCharPointer()); + + // Not 100% sure why a global mutex sometimes can't be allocated, but if it fails, fall back to + // a local one. (A local one also sometimes fails on other machines so neither type appears to be + // universally reliable) + if (handle == 0) + handle = CreateMutexW (0, TRUE, ("Local\\" + name).toWideCharPointer()); + + if (handle != 0 && GetLastError() == ERROR_ALREADY_EXISTS) + { + if (timeOutMillisecs == 0) + { + close(); + return; + } + + switch (WaitForSingleObject (handle, timeOutMillisecs < 0 ? INFINITE : timeOutMillisecs)) + { + case WAIT_OBJECT_0: + case WAIT_ABANDONED: + break; + + case WAIT_TIMEOUT: + default: + close(); + break; + } + } + } + + ~Pimpl() + { + close(); + } + + void close() + { + if (handle != 0) + { + ReleaseMutex (handle); + CloseHandle (handle); + handle = 0; + } + } + + HANDLE handle; + int refCount; +}; + +InterProcessLock::InterProcessLock (const String& name_) + : name (name_) +{ +} + +InterProcessLock::~InterProcessLock() +{ +} + +bool InterProcessLock::enter (const int timeOutMillisecs) +{ + const ScopedLock sl (lock); + + if (pimpl == nullptr) + { + pimpl = new Pimpl (name, timeOutMillisecs); + + if (pimpl->handle == 0) + pimpl = nullptr; + } + else + { + pimpl->refCount++; + } + + return pimpl != nullptr; +} + +void InterProcessLock::exit() +{ + const ScopedLock sl (lock); + + // Trying to release the lock too many times! + bassert (pimpl != nullptr); + + if (pimpl != nullptr && --(pimpl->refCount) == 0) + pimpl = nullptr; +} + +//============================================================================== +class ChildProcess::ActiveProcess +{ +public: + ActiveProcess (const String& command) + : ok (false), readPipe (0), writePipe (0) + { + SECURITY_ATTRIBUTES securityAtts = { 0 }; + securityAtts.nLength = sizeof (securityAtts); + securityAtts.bInheritHandle = TRUE; + + if (CreatePipe (&readPipe, &writePipe, &securityAtts, 0) + && SetHandleInformation (readPipe, HANDLE_FLAG_INHERIT, 0)) + { + STARTUPINFOW startupInfo = { 0 }; + startupInfo.cb = sizeof (startupInfo); + startupInfo.hStdError = writePipe; + startupInfo.hStdOutput = writePipe; + startupInfo.dwFlags = STARTF_USESTDHANDLES; + + ok = CreateProcess (nullptr, const_cast (command.toWideCharPointer()), + nullptr, nullptr, TRUE, CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT, + nullptr, nullptr, &startupInfo, &processInfo) != FALSE; + } + } + + ~ActiveProcess() + { + if (ok) + { + CloseHandle (processInfo.hThread); + CloseHandle (processInfo.hProcess); + } + + if (readPipe != 0) + CloseHandle (readPipe); + + if (writePipe != 0) + CloseHandle (writePipe); + } + + bool isRunning() const + { + return WaitForSingleObject (processInfo.hProcess, 0) != WAIT_OBJECT_0; + } + + int read (void* dest, int numNeeded) const + { + int total = 0; + + while (ok && numNeeded > 0) + { + DWORD available = 0; + + if (! PeekNamedPipe ((HANDLE) readPipe, nullptr, 0, nullptr, &available, nullptr)) + break; + + const int numToDo = bmin ((int) available, numNeeded); + + if (available == 0) + { + if (! isRunning()) + break; + + Thread::yield(); + } + else + { + DWORD numRead = 0; + if (! ReadFile ((HANDLE) readPipe, dest, numToDo, &numRead, nullptr)) + break; + + total += numRead; + dest = addBytesToPointer (dest, numRead); + numNeeded -= numRead; + } + } + + return total; + } + + bool killProcess() const + { + return TerminateProcess (processInfo.hProcess, 0) != FALSE; + } + + bool ok; + +private: + HANDLE readPipe, writePipe; + PROCESS_INFORMATION processInfo; + + BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ActiveProcess) +}; + +bool ChildProcess::start (const String& command) +{ + activeProcess = new ActiveProcess (command); + + if (! activeProcess->ok) + activeProcess = nullptr; + + return activeProcess != nullptr; +} + +bool ChildProcess::start (const StringArray& args) +{ + return start (args.joinIntoString (" ")); +} + +bool ChildProcess::isRunning() const +{ + return activeProcess != nullptr && activeProcess->isRunning(); +} + +int ChildProcess::readProcessOutput (void* dest, int numBytes) +{ + return activeProcess != nullptr ? activeProcess->read (dest, numBytes) : 0; +} + +bool ChildProcess::kill() +{ + return activeProcess == nullptr || activeProcess->killProcess(); +} + +//============================================================================== +struct HighResolutionTimer::Pimpl +{ + Pimpl (HighResolutionTimer& t) noexcept : owner (t), periodMs (0) + { + } + + ~Pimpl() + { + bassert (periodMs == 0); + } + + void start (int newPeriod) + { + if (newPeriod != periodMs) + { + stop(); + periodMs = newPeriod; + + TIMECAPS tc; + if (timeGetDevCaps (&tc, sizeof (tc)) == TIMERR_NOERROR) + { + const int actualPeriod = blimit ((int) tc.wPeriodMin, (int) tc.wPeriodMax, newPeriod); + + timerID = timeSetEvent (actualPeriod, tc.wPeriodMin, callbackFunction, (DWORD_PTR) this, + TIME_PERIODIC | TIME_CALLBACK_FUNCTION | 0x100 /*TIME_KILL_SYNCHRONOUS*/); + } + } + } + + void stop() + { + periodMs = 0; + timeKillEvent (timerID); + } + + HighResolutionTimer& owner; + int periodMs; + +private: + unsigned int timerID; + + static void __stdcall callbackFunction (UINT, UINT, DWORD_PTR userInfo, DWORD_PTR, DWORD_PTR) + { + if (Pimpl* const timer = reinterpret_cast (userInfo)) + if (timer->periodMs != 0) + timer->owner.hiResTimerCallback(); + } + + BEAST_DECLARE_NON_COPYABLE (Pimpl) +}; diff --git a/Subtrees/beast/modules/beast_core/native/beast_win32_Network.cpp b/Subtrees/beast/modules/beast_core/native/beast_win32_Network.cpp index 7edad70888..f239d190c7 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_win32_Network.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_win32_Network.cpp @@ -30,7 +30,9 @@ #endif //============================================================================== -class WebInputStream : public InputStream +class WebInputStream + : public InputStream + , LeakChecked { public: WebInputStream (const String& address_, bool isPost_, const MemoryBlock& postData_, @@ -296,8 +298,6 @@ private: close(); } - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (WebInputStream) }; InputStream* URL::createNativeStream (const String& address, bool isPost, const MemoryBlock& postData, diff --git a/Subtrees/beast/modules/beast_core/native/beast_win32_Registry.cpp b/Subtrees/beast/modules/beast_core/native/beast_win32_Registry.cpp index adc7599c9a..af37aaa93a 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_win32_Registry.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_win32_Registry.cpp @@ -21,7 +21,7 @@ */ //============================================================================== -struct RegistryKeyWrapper +struct RegistryKeyWrapper : Uncopyable { RegistryKeyWrapper (String name, const bool createForWriting, const DWORD wow64Flags) : key (0), wideCharValueName (nullptr) @@ -130,8 +130,6 @@ struct RegistryKeyWrapper HKEY key; const wchar_t* wideCharValueName; String valueName; - - BEAST_DECLARE_NON_COPYABLE (RegistryKeyWrapper) }; uint32 WindowsRegistry::getBinaryValue (const String& regValuePath, MemoryBlock& result) diff --git a/Subtrees/beast/modules/beast_core/native/beast_win32_SystemStats.cpp b/Subtrees/beast/modules/beast_core/native/beast_win32_SystemStats.cpp index f6568e1bd9..5ba5a1f3b6 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_win32_SystemStats.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_win32_SystemStats.cpp @@ -115,44 +115,52 @@ SystemStats::CPUFlags::CPUFlags() numCpus = (int) systemInfo.dwNumberOfProcessors; } -#if BEAST_MSVC && BEAST_CHECK_MEMORY_LEAKS -struct DebugFlagsInitialiser -{ - DebugFlagsInitialiser() - { - _CrtSetDbgFlag (_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); - } -}; - -static DebugFlagsInitialiser debugFlagsInitialiser; -#endif - //============================================================================== -SystemStats::OperatingSystemType SystemStats::getOperatingSystemType() +static bool isWindowsVersionOrLater (SystemStats::OperatingSystemType target) { - OSVERSIONINFO info; - info.dwOSVersionInfoSize = sizeof (info); - GetVersionEx (&info); + OSVERSIONINFOEX info; + zerostruct (info); + info.dwOSVersionInfoSize = sizeof (OSVERSIONINFOEX); - if (info.dwPlatformId == VER_PLATFORM_WIN32_NT) + if (target >= SystemStats::WinVista) { - if (info.dwMajorVersion == 5) - return (info.dwMinorVersion == 0) ? Win2000 : WinXP; + info.dwMajorVersion = 6; - if (info.dwMajorVersion == 6) + switch (target) { - switch (info.dwMinorVersion) - { - case 0: return WinVista; - case 1: return Windows7; - case 2: return Windows8; - - default: - bassertfalse; // new version needs to be added here! - return Windows8; - } + case SystemStats::WinVista: info.dwMinorVersion = 0; break; + case SystemStats::Windows7: info.dwMinorVersion = 1; break; + case SystemStats::Windows8: info.dwMinorVersion = 2; break; + default: bassertfalse; break; } } + else + { + info.dwMajorVersion = 5; + info.dwMinorVersion = target >= SystemStats::WinXP ? 1 : 0; + } + + DWORDLONG mask = 0; + + VER_SET_CONDITION (mask, VER_MAJORVERSION, VER_GREATER_EQUAL); + VER_SET_CONDITION (mask, VER_MINORVERSION, VER_GREATER_EQUAL); + VER_SET_CONDITION (mask, VER_SERVICEPACKMAJOR, VER_GREATER_EQUAL); + VER_SET_CONDITION (mask, VER_SERVICEPACKMINOR, VER_GREATER_EQUAL); + + return VerifyVersionInfo (&info, + VER_MAJORVERSION | VER_MINORVERSION + | VER_SERVICEPACKMAJOR | VER_SERVICEPACKMINOR, + mask) != FALSE; +} + +SystemStats::OperatingSystemType SystemStats::getOperatingSystemType() +{ + const SystemStats::OperatingSystemType types[] + = { Windows8, Windows7, WinVista, WinXP, Win2000 }; + + for (int i = 0; i < numElementsInArray (types); ++i) + if (isWindowsVersionOrLater (types[i])) + return types[i]; bassertfalse; // need to support whatever new version is running! return UnknownOS; diff --git a/Subtrees/beast/modules/beast_core/native/beast_win32_Threads.cpp b/Subtrees/beast/modules/beast_core/native/beast_win32_Threads.cpp index 2274bec659..6c9f455a77 100644 --- a/Subtrees/beast/modules/beast_core/native/beast_win32_Threads.cpp +++ b/Subtrees/beast/modules/beast_core/native/beast_win32_Threads.cpp @@ -285,6 +285,8 @@ bool BEAST_CALLTYPE Process::isRunningUnderDebugger() return beast_isRunningUnderDebugger(); } +//------------------------------------------------------------------------------ + static void* currentModuleHandle = nullptr; void* Process::getCurrentModuleInstanceHandle() noexcept @@ -457,7 +459,7 @@ void InterProcessLock::exit() } //============================================================================== -class ChildProcess::ActiveProcess +class ChildProcess::ActiveProcess : LeakChecked , Uncopyable { public: ActiveProcess (const String& command) @@ -547,8 +549,6 @@ public: private: HANDLE readPipe, writePipe; PROCESS_INFORMATION processInfo; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ActiveProcess) }; bool ChildProcess::start (const String& command) @@ -582,7 +582,7 @@ bool ChildProcess::kill() } //============================================================================== -struct HighResolutionTimer::Pimpl +struct HighResolutionTimer::Pimpl : Uncopyable { Pimpl (HighResolutionTimer& t) noexcept : owner (t), periodMs (0) { @@ -629,6 +629,4 @@ private: if (timer->periodMs != 0) timer->owner.hiResTimerCallback(); } - - BEAST_DECLARE_NON_COPYABLE (Pimpl) }; diff --git a/Subtrees/beast/modules/beast_core/network/beast_NamedPipe.h b/Subtrees/beast/modules/beast_core/network/beast_NamedPipe.h index ae9c24c5ff..3e73e09ecc 100644 --- a/Subtrees/beast/modules/beast_core/network/beast_NamedPipe.h +++ b/Subtrees/beast/modules/beast_core/network/beast_NamedPipe.h @@ -34,7 +34,7 @@ @see InterprocessConnection */ -class BEAST_API NamedPipe +class BEAST_API NamedPipe : LeakChecked , Uncopyable { public: //============================================================================== @@ -92,8 +92,6 @@ private: ReadWriteLock lock; bool openInternal (const String& pipeName, const bool createPipe); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (NamedPipe) }; diff --git a/Subtrees/beast/modules/beast_core/network/beast_Socket.cpp b/Subtrees/beast/modules/beast_core/network/beast_Socket.cpp index 30013e06a0..60e997d2b7 100644 --- a/Subtrees/beast/modules/beast_core/network/beast_Socket.cpp +++ b/Subtrees/beast/modules/beast_core/network/beast_Socket.cpp @@ -81,7 +81,7 @@ namespace SocketHelpers servTmpAddr.sin_addr.s_addr = htonl (INADDR_ANY); servTmpAddr.sin_port = htons ((uint16) port); - return bind (handle, (struct sockaddr*) &servTmpAddr, sizeof (struct sockaddr_in)) >= 0; + return ::bind (handle, (struct sockaddr*) &servTmpAddr, sizeof (struct sockaddr_in)) >= 0; } static int readSocket (const SocketHandle handle, @@ -415,7 +415,7 @@ bool StreamingSocket::createListener (const int newPortNumber, const String& loc const int reuse = 1; setsockopt (handle, SOL_SOCKET, SO_REUSEADDR, (const char*) &reuse, sizeof (reuse)); - if (bind (handle, (struct sockaddr*) &servTmpAddr, sizeof (struct sockaddr_in)) < 0 + if (::bind (handle, (struct sockaddr*) &servTmpAddr, sizeof (struct sockaddr_in)) < 0 || listen (handle, SOMAXCONN) < 0) { close(); diff --git a/Subtrees/beast/modules/beast_core/network/beast_Socket.h b/Subtrees/beast/modules/beast_core/network/beast_Socket.h index 91db75fe83..3b526f6f4e 100644 --- a/Subtrees/beast/modules/beast_core/network/beast_Socket.h +++ b/Subtrees/beast/modules/beast_core/network/beast_Socket.h @@ -36,7 +36,7 @@ @see DatagramSocket, InterprocessConnection, InterprocessConnectionServer */ -class BEAST_API StreamingSocket +class BEAST_API StreamingSocket : LeakChecked , Uncopyable { public: //============================================================================== @@ -163,8 +163,6 @@ private: bool connected, isListener; StreamingSocket (const String& hostname, int portNumber, int handle); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (StreamingSocket) }; @@ -177,7 +175,7 @@ private: @see StreamingSocket, InterprocessConnection, InterprocessConnectionServer */ -class BEAST_API DatagramSocket +class BEAST_API DatagramSocket : LeakChecked , Uncopyable { public: //============================================================================== @@ -294,8 +292,6 @@ private: void* serverAddress; DatagramSocket (const String& hostname, int portNumber, int handle, int localPortNumber); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (DatagramSocket) }; diff --git a/Subtrees/beast/modules/beast_core/network/beast_URL.h b/Subtrees/beast/modules/beast_core/network/beast_URL.h index dd6963341f..31e14afe17 100644 --- a/Subtrees/beast/modules/beast_core/network/beast_URL.h +++ b/Subtrees/beast/modules/beast_core/network/beast_URL.h @@ -37,7 +37,7 @@ class XmlElement; This class can be used to launch URLs in browsers, and also to create InputStreams that can read from remote http or ftp sources. */ -class BEAST_API URL +class BEAST_API URL : LeakChecked { public: //============================================================================== @@ -338,7 +338,6 @@ private: OpenStreamProgressCallback* progressCallback, void* progressCallbackContext, const String& headers, const int timeOutMs, StringPairArray* responseHeaders); - BEAST_LEAK_DETECTOR (URL) }; diff --git a/Subtrees/beast/modules/beast_core/streams/beast_BufferedInputStream.h b/Subtrees/beast/modules/beast_core/streams/beast_BufferedInputStream.h index 54ef9668e1..c2d35440ea 100644 --- a/Subtrees/beast/modules/beast_core/streams/beast_BufferedInputStream.h +++ b/Subtrees/beast/modules/beast_core/streams/beast_BufferedInputStream.h @@ -37,7 +37,9 @@ so that the source stream gets accessed in larger chunk sizes, meaning less work for the underlying stream. */ -class BEAST_API BufferedInputStream : public InputStream +class BEAST_API BufferedInputStream + : public InputStream + , LeakChecked { public: //============================================================================== @@ -84,8 +86,6 @@ private: int64 position, lastReadPos, bufferStart, bufferOverlap; HeapBlock buffer; void ensureBuffered(); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (BufferedInputStream) }; #endif // BEAST_BUFFEREDINPUTSTREAM_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/streams/beast_FileInputSource.h b/Subtrees/beast/modules/beast_core/streams/beast_FileInputSource.h index 8726144f42..ff217974cc 100644 --- a/Subtrees/beast/modules/beast_core/streams/beast_FileInputSource.h +++ b/Subtrees/beast/modules/beast_core/streams/beast_FileInputSource.h @@ -34,7 +34,10 @@ @see InputSource */ -class BEAST_API FileInputSource : public InputSource +class BEAST_API FileInputSource + : public InputSource + , LeakChecked + , Uncopyable { public: //============================================================================== @@ -56,8 +59,6 @@ private: //============================================================================== const File file; bool useFileTimeInHashGeneration; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (FileInputSource) }; diff --git a/Subtrees/beast/modules/beast_core/streams/beast_InputSource.h b/Subtrees/beast/modules/beast_core/streams/beast_InputSource.h index cd47a678c1..1936df3a61 100644 --- a/Subtrees/beast/modules/beast_core/streams/beast_InputSource.h +++ b/Subtrees/beast/modules/beast_core/streams/beast_InputSource.h @@ -35,7 +35,7 @@ @see FileInputSource */ -class BEAST_API InputSource +class BEAST_API InputSource : LeakChecked { public: //============================================================================== @@ -63,11 +63,6 @@ public: /** Returns a hash code that uniquely represents this item. */ virtual int64 hashCode() const = 0; - - -private: - //============================================================================== - BEAST_LEAK_DETECTOR (InputSource) }; diff --git a/Subtrees/beast/modules/beast_core/streams/beast_InputStream.h b/Subtrees/beast/modules/beast_core/streams/beast_InputStream.h index d90bb5c599..7d7e643234 100644 --- a/Subtrees/beast/modules/beast_core/streams/beast_InputStream.h +++ b/Subtrees/beast/modules/beast_core/streams/beast_InputStream.h @@ -37,6 +37,8 @@ class MemoryBlock; @see OutputStream, MemoryInputStream, BufferedInputStream, FileInputStream */ class BEAST_API InputStream + : public Uncopyable + , LeakChecked { public: /** Destructor. */ @@ -285,9 +287,6 @@ public: protected: //============================================================================== InputStream() noexcept {} - -private: - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (InputStream) }; #endif // BEAST_INPUTSTREAM_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/streams/beast_MemoryInputStream.h b/Subtrees/beast/modules/beast_core/streams/beast_MemoryInputStream.h index ce39cc7897..d3e0d0b8d5 100644 --- a/Subtrees/beast/modules/beast_core/streams/beast_MemoryInputStream.h +++ b/Subtrees/beast/modules/beast_core/streams/beast_MemoryInputStream.h @@ -35,7 +35,9 @@ This can either be used to refer to a shared block of memory, or can make its own internal copy of the data when the MemoryInputStream is created. */ -class BEAST_API MemoryInputStream : public InputStream +class BEAST_API MemoryInputStream + : public InputStream + , LeakChecked { public: //============================================================================== @@ -88,8 +90,6 @@ private: HeapBlock internalCopy; void createInternalCopy(); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (MemoryInputStream) }; #endif // BEAST_MEMORYINPUTSTREAM_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/streams/beast_MemoryOutputStream.h b/Subtrees/beast/modules/beast_core/streams/beast_MemoryOutputStream.h index 98ea3daf90..1413dcbf3d 100644 --- a/Subtrees/beast/modules/beast_core/streams/beast_MemoryOutputStream.h +++ b/Subtrees/beast/modules/beast_core/streams/beast_MemoryOutputStream.h @@ -36,7 +36,9 @@ The data that was written into the stream can then be accessed later as a contiguous block of memory. */ -class BEAST_API MemoryOutputStream : public OutputStream +class BEAST_API MemoryOutputStream + : public OutputStream + , LeakChecked { public: //============================================================================== @@ -120,8 +122,6 @@ private: void trimExternalBlockSize(); char* prepareToWrite (size_t); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (MemoryOutputStream) }; /** Copies all the data that has been written to a MemoryOutputStream into another stream. */ diff --git a/Subtrees/beast/modules/beast_core/streams/beast_OutputStream.h b/Subtrees/beast/modules/beast_core/streams/beast_OutputStream.h index 052cbc4fc7..9365041ba8 100644 --- a/Subtrees/beast/modules/beast_core/streams/beast_OutputStream.h +++ b/Subtrees/beast/modules/beast_core/streams/beast_OutputStream.h @@ -41,6 +41,8 @@ class File; @see InputStream, MemoryOutputStream, FileOutputStream */ class BEAST_API OutputStream + : public Uncopyable + , LeakChecked { protected: //============================================================================== @@ -220,8 +222,6 @@ public: private: //============================================================================== String newLineString; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (OutputStream) }; //============================================================================== diff --git a/Subtrees/beast/modules/beast_core/streams/beast_SubregionStream.h b/Subtrees/beast/modules/beast_core/streams/beast_SubregionStream.h index f98cdecfe5..cc01fed0a3 100644 --- a/Subtrees/beast/modules/beast_core/streams/beast_SubregionStream.h +++ b/Subtrees/beast/modules/beast_core/streams/beast_SubregionStream.h @@ -34,7 +34,9 @@ This lets you take a subsection of a stream and present it as an entire stream in its own right. */ -class BEAST_API SubregionStream : public InputStream +class BEAST_API SubregionStream + : public InputStream + , LeakChecked { public: //============================================================================== @@ -80,8 +82,6 @@ public: private: OptionalScopedPointer source; const int64 startPositionInSourceStream, lengthOfSourceStream; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (SubregionStream) }; #endif // BEAST_SUBREGIONSTREAM_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/system/beast_Functional.h b/Subtrees/beast/modules/beast_core/system/beast_Functional.h new file mode 100644 index 0000000000..ef178d2c19 --- /dev/null +++ b/Subtrees/beast/modules/beast_core/system/beast_Functional.h @@ -0,0 +1,329 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef BEAST_FUNCTIONAL_BEASTHEADER +#define BEAST_FUNCTIONAL_BEASTHEADER + +/* Brings functional support into our namespace, based on environment. + + Notes on bind + + Difference between boost::bind and std::bind + http://stackoverflow.com/questions/10555566/is-there-any-difference-between-c11-stdbind-and-boostbind + + Resolving conflict between boost::shared_ptr and std::shared_ptr + http://stackoverflow.com/questions/4682343/how-to-resolve-conflict-between-boostshared-ptr-and-using-stdshared-ptr +*/ + +#ifndef BEAST_BIND_PLACEHOLDERS_N +# if BEAST_MSVC && BEAST_BIND_USES_STD +# define BEAST_BIND_PLACEHOLDERS_N 20 // Visual Studio 2012 +# else +# define BEAST_BIND_PLACEHOLDERS_N 8 // Seems a reasonable number +# endif +#endif + +/** Max number of arguments to bind, total. +*/ +#if BEAST_MSVC +# ifdef _VARIADIC_MAX +# define BEAST_VARIADIC_MAX _VARIADIC_MAX +# else +# define BEAST_VARIADIC_MAX 9 +# endif +#else +# define BEAST_VARIADIC_MAX 9 +#endif + +//------------------------------------------------------------------------------ + +#if BEAST_BIND_USES_STD + +using std::ref; +using std::cref; +using std::bind; +using std::function; + +#if BEAST_BIND_PLACEHOLDERS_N >= 1 +using std::placeholders::_1; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 2 +using std::placeholders::_2; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 3 +using std::placeholders::_3; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 4 +using std::placeholders::_4; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 5 +using std::placeholders::_5; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 6 +using std::placeholders::_6; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 7 +using std::placeholders::_7; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 8 +using std::placeholders::_8; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 9 +using std::placeholders::_9; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 10 +using std::placeholders::_10; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 11 +using std::placeholders::_11; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 12 +using std::placeholders::_12; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 13 +using std::placeholders::_13; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 14 +using std::placeholders::_14; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 15 +using std::placeholders::_15; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 16 +using std::placeholders::_16; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 17 +using std::placeholders::_17; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 18 +using std::placeholders::_18; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 19 +using std::placeholders::_19; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 20 +using std::placeholders::_20; +#endif + +//------------------------------------------------------------------------------ + +#elif BEAST_BIND_USES_TR1 + +using std::tr1::ref; +using std::tr1::cref; +using std::tr1::bind; +using std::tr1::function; + +#if BEAST_BIND_PLACEHOLDERS_N >= 1 +using std::tr1::placeholders::_1; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 2 +using std::tr1::placeholders::_2; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 3 +using std::tr1::placeholders::_3; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 4 +using std::tr1::placeholders::_4; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 5 +using std::tr1::placeholders::_5; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 6 +using std::tr1::placeholders::_6; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 7 +using std::tr1::placeholders::_7; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 8 +using std::tr1::placeholders::_8; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 9 +using std::tr1::placeholders::_9; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 10 +using std::tr1::placeholders::_10; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 11 +using std::tr1::placeholders::_11; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 12 +using std::tr1::placeholders::_12; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 13 +using std::tr1::placeholders::_13; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 14 +using std::tr1::placeholders::_14; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 15 +using std::tr1::placeholders::_15; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 16 +using std::tr1::placeholders::_16; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 17 +using std::tr1::placeholders::_17; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 18 +using std::tr1::placeholders::_18; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 19 +using std::tr1::placeholders::_19; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 20 +using std::tr1::placeholders::_20; +#endif + +//------------------------------------------------------------------------------ + +#elif BEAST_BIND_USES_BOOST + +using boost::ref; +using boost::cref; +using boost::bind; +using boost::function; + +#if BEAST_BIND_PLACEHOLDERS_N >= 1 +using boost::placeholders::_1; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 2 +using boost::placeholders::_2; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 3 +using boost::placeholders::_3; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 4 +using boost::placeholders::_4; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 5 +using boost::placeholders::_5; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 6 +using boost::placeholders::_6; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 7 +using boost::placeholders::_7; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 8 +using boost::placeholders::_8; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 9 +using boost::placeholders::_9; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 10 +using boost::placeholders::_10; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 11 +using boost::placeholders::_11; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 12 +using boost::placeholders::_12; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 13 +using boost::placeholders::_13; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 14 +using boost::placeholders::_14; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 15 +using boost::placeholders::_15; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 16 +using boost::placeholders::_16; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 17 +using boost::placeholders::_17; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 18 +using boost::placeholders::_18; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 19 +using boost::placeholders::_19; +#endif + +#if BEAST_BIND_PLACEHOLDERS_N >= 20 +using boost::placeholders::_20; +#endif + +//------------------------------------------------------------------------------ + +#else + +#error Unknown bind source in beast_Functional.h + +#endif + +#endif diff --git a/Subtrees/beast/modules/beast_core/system/beast_PlatformDefs.h b/Subtrees/beast/modules/beast_core/system/beast_PlatformDefs.h index dd3ff1051e..9af6796ec4 100644 --- a/Subtrees/beast/modules/beast_core/system/beast_PlatformDefs.h +++ b/Subtrees/beast/modules/beast_core/system/beast_PlatformDefs.h @@ -24,107 +24,113 @@ #ifndef BEAST_PLATFORMDEFS_BEASTHEADER #define BEAST_PLATFORMDEFS_BEASTHEADER -//============================================================================== -/* This file defines miscellaneous macros for debugging, assertions, etc. -*/ +// This file defines miscellaneous macros for debugging, assertions, etc. -//============================================================================== #ifdef BEAST_FORCE_DEBUG - #undef BEAST_DEBUG - - #if BEAST_FORCE_DEBUG - #define BEAST_DEBUG 1 - #endif +# undef BEAST_DEBUG +# if BEAST_FORCE_DEBUG +# define BEAST_DEBUG 1 +# endif #endif -/** This macro defines the C calling convention used as the standard for Beast calls. */ +/** This macro defines the C calling convention used as the standard for Beast calls. +*/ #if BEAST_MSVC - #define BEAST_CALLTYPE __stdcall - #define BEAST_CDECL __cdecl +# define BEAST_CALLTYPE __stdcall +# define BEAST_CDECL __cdecl #else - #define BEAST_CALLTYPE - #define BEAST_CDECL +# define BEAST_CALLTYPE +# define BEAST_CDECL #endif -//============================================================================== // Debugging and assertion macros #if BEAST_LOG_ASSERTIONS || BEAST_DEBUG - #define beast_LogCurrentAssertion beast::logAssertion (__FILE__, __LINE__); +#define beast_LogCurrentAssertion beast::logAssertion (__FILE__, __LINE__); #else - #define beast_LogCurrentAssertion +#define beast_LogCurrentAssertion #endif -//============================================================================== #if BEAST_IOS || BEAST_LINUX || BEAST_ANDROID || BEAST_PPC - /** This will try to break into the debugger if the app is currently being debugged. - If called by an app that's not being debugged, the behaiour isn't defined - it may crash or not, depending - on the platform. - @see bassert() - */ - #define beast_breakDebugger { ::kill (0, SIGTRAP); } +/** This will try to break into the debugger if the app is currently being debugged. + If called by an app that's not being debugged, the behaiour isn't defined - it may crash or not, depending + on the platform. + @see bassert() +*/ +# define beast_breakDebugger { ::kill (0, SIGTRAP); } #elif BEAST_USE_INTRINSICS - #ifndef __INTEL_COMPILER - #pragma intrinsic (__debugbreak) - #endif - #define beast_breakDebugger { __debugbreak(); } +# ifndef __INTEL_COMPILER +# pragma intrinsic (__debugbreak) +# endif +# define beast_breakDebugger { __debugbreak(); } #elif BEAST_GCC || BEAST_MAC - #if BEAST_NO_INLINE_ASM - #define beast_breakDebugger { } - #else - #define beast_breakDebugger { asm ("int $3"); } - #endif +# if BEAST_NO_INLINE_ASM +# define beast_breakDebugger { } +# else +# define beast_breakDebugger { asm ("int $3"); } +# endif #else - #define beast_breakDebugger { __asm int 3 } +# define beast_breakDebugger { __asm int 3 } #endif +#if BEAST_CLANG && defined (__has_feature) && ! defined (BEAST_ANALYZER_NORETURN) +# if __has_feature (attribute_analyzer_noreturn) + inline void __attribute__((analyzer_noreturn)) beast_assert_noreturn() {} +# define BEAST_ANALYZER_NORETURN beast_assert_noreturn(); +# endif +#endif + +#ifndef BEAST_ANALYZER_NORETURN +#define BEAST_ANALYZER_NORETURN +#endif + +//------------------------------------------------------------------------------ -//============================================================================== #if BEAST_DEBUG || DOXYGEN - /** Writes a string to the standard error stream. - This is only compiled in a debug build. - @see Logger::outputDebugString - */ - #define DBG(dbgtext) { beast::String tempDbgBuf; tempDbgBuf << dbgtext; beast::Logger::outputDebugString (tempDbgBuf); } - //============================================================================== - /** This will always cause an assertion failure. - It is only compiled in a debug build, (unless BEAST_LOG_ASSERTIONS is enabled for your build). - @see bassert - */ - #define bassertfalse { beast_LogCurrentAssertion; if (beast::beast_isRunningUnderDebugger()) beast_breakDebugger; } +/** Writes a string to the standard error stream. + This is only compiled in a debug build. + @see Logger::outputDebugString +*/ +#define DBG(dbgtext) { beast::String tempDbgBuf; tempDbgBuf << dbgtext; beast::Logger::outputDebugString (tempDbgBuf); } - //============================================================================== - /** Platform-independent assertion macro. +/** This will always cause an assertion failure. + It is only compiled in a debug build, (unless BEAST_LOG_ASSERTIONS is enabled for your build). + @see bassert +*/ +#define bassertfalse { beast_LogCurrentAssertion; if (beast::beast_isRunningUnderDebugger()) beast_breakDebugger; BEAST_ANALYZER_NORETURN } - This macro gets turned into a no-op when you're building with debugging turned off, so be - careful that the expression you pass to it doesn't perform any actions that are vital for the - correct behaviour of your program! - @see bassertfalse +/** Platform-independent assertion macro. + + This macro gets turned into a no-op when you're building with debugging turned off, so be + careful that the expression you pass to it doesn't perform any actions that are vital for the + correct behaviour of your program! + @see bassertfalse */ - #define bassert(expression) { if (! (expression)) bassertfalse; } +#define bassert(expression) { if (! (expression)) bassertfalse; } #else - //============================================================================== - // If debugging is disabled, these dummy debug and assertion macros are used.. - #define DBG(dbgtext) - #define bassertfalse { beast_LogCurrentAssertion } +// If debugging is disabled, these dummy debug and assertion macros are used.. - #if BEAST_LOG_ASSERTIONS - #define bassert(expression) { if (! (expression)) bassertfalse; } - #else - #define bassert(a) {} - #endif +#define DBG(dbgtext) +#define bassertfalse { beast_LogCurrentAssertion } + +# if BEAST_LOG_ASSERTIONS +# define bassert(expression) { if (! (expression)) bassertfalse; } +# else +# define bassert(a) {} +# endif #endif -//============================================================================== +//------------------------------------------------------------------------------ + #ifndef DOXYGEN namespace beast { - template struct BeastStaticAssert; - template <> struct BeastStaticAssert { static void dummy() {} }; +template struct BeastStaticAssert; +template <> struct BeastStaticAssert { static void dummy() {} }; } #endif @@ -135,41 +141,6 @@ namespace beast */ #define static_bassert(expression) beast::BeastStaticAssert::dummy(); -/** This is a shorthand macro for declaring stubs for a class's copy constructor and operator=. - - For example, instead of - @code - class MyClass - { - etc.. - - private: - MyClass (const MyClass&); - MyClass& operator= (const MyClass&); - };@endcode - - ..you can just write: - - @code - class MyClass - { - etc.. - - private: - BEAST_DECLARE_NON_COPYABLE (MyClass) - };@endcode -*/ -#define BEAST_DECLARE_NON_COPYABLE(className) \ - className (const className&);\ - className& operator= (const className&); - -/** This is a shorthand way of writing both a BEAST_DECLARE_NON_COPYABLE and - BEAST_LEAK_DETECTOR macro for a class. -*/ -#define BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(className) \ - BEAST_DECLARE_NON_COPYABLE(className) \ - BEAST_LEAK_DETECTOR(className) - /** This macro can be added to class definitions to disable the use of new/delete to allocate the object on the heap, forcing it to only be used as a stack or member variable. */ @@ -178,8 +149,8 @@ namespace beast static void* operator new (size_t); \ static void operator delete (void*); +//------------------------------------------------------------------------------ -//============================================================================== #if ! DOXYGEN #define BEAST_JOIN_MACRO_HELPER(a, b) a ## b #define BEAST_STRINGIFY_MACRO_HELPER(a) #a @@ -195,158 +166,142 @@ namespace beast */ #define BEAST_STRINGIFY(item) BEAST_STRINGIFY_MACRO_HELPER (item) +//------------------------------------------------------------------------------ -//============================================================================== #if BEAST_CATCH_UNHANDLED_EXCEPTIONS - - #define BEAST_TRY try - - #define BEAST_CATCH_ALL catch (...) {} - #define BEAST_CATCH_ALL_ASSERT catch (...) { bassertfalse; } - - #if ! BEAST_MODULE_AVAILABLE_beast_gui_basics - #define BEAST_CATCH_EXCEPTION BEAST_CATCH_ALL - #else - /** Used in try-catch blocks, this macro will send exceptions to the BEASTApplication - object so they can be logged by the application if it wants to. - */ - #define BEAST_CATCH_EXCEPTION \ - catch (const std::exception& e) \ - { \ - beast::BEASTApplication::sendUnhandledException (&e, __FILE__, __LINE__); \ - } \ - catch (...) \ - { \ - beast::BEASTApplication::sendUnhandledException (nullptr, __FILE__, __LINE__); \ - } - #endif +# define BEAST_TRY try +# define BEAST_CATCH_ALL catch (...) {} +# define BEAST_CATCH_ALL_ASSERT catch (...) { bassertfalse; } +# define BEAST_CATCH_EXCEPTION BEAST_CATCH_ALL #else - - #define BEAST_TRY - #define BEAST_CATCH_EXCEPTION - #define BEAST_CATCH_ALL - #define BEAST_CATCH_ALL_ASSERT +# define BEAST_TRY +# define BEAST_CATCH_EXCEPTION +# define BEAST_CATCH_ALL +# define BEAST_CATCH_ALL_ASSERT #endif -//============================================================================== +//------------------------------------------------------------------------------ + #if BEAST_DEBUG || DOXYGEN - /** A platform-independent way of forcing an inline function. - Use the syntax: @code - forcedinline void myfunction (int x) - @endcode - */ - #define forcedinline inline +/** A platform-independent way of forcing an inline function. + Use the syntax: @code + forcedinline void myfunction (int x) + @endcode +*/ +# define forcedinline inline +#elif BEAST_MSVC +# define forcedinline __forceinline #else - #if BEAST_MSVC - #define forcedinline __forceinline - #else - #define forcedinline inline __attribute__((always_inline)) - #endif +# define forcedinline inline __attribute__((always_inline)) #endif #if BEAST_MSVC || DOXYGEN - /** This can be placed before a stack or member variable declaration to tell the compiler - to align it to the specified number of bytes. */ - #define BEAST_ALIGN(bytes) __declspec (align (bytes)) +/** This can be placed before a stack or member variable declaration to tell + the compiler to align it to the specified number of bytes. +*/ +#define BEAST_ALIGN(bytes) __declspec (align (bytes)) #else - #define BEAST_ALIGN(bytes) __attribute__ ((aligned (bytes))) +#define BEAST_ALIGN(bytes) __attribute__ ((aligned (bytes))) #endif -//============================================================================== +//------------------------------------------------------------------------------ + // Cross-compiler deprecation macros.. -#if DOXYGEN || (BEAST_MSVC && ! BEAST_NO_DEPRECATION_WARNINGS) - /** This can be used to wrap a function which has been deprecated. */ - #define BEAST_DEPRECATED(functionDef) __declspec(deprecated) functionDef -#elif BEAST_GCC && ! BEAST_NO_DEPRECATION_WARNINGS - #define BEAST_DEPRECATED(functionDef) functionDef __attribute__ ((deprecated)) +#ifdef DOXYGEN + /** This macro can be used to wrap a function which has been deprecated. */ + #define BEAST_DEPRECATED(functionDef) +#elif BEAST_MSVC && ! BEAST_NO_DEPRECATION_WARNINGS + #define BEAST_DEPRECATED(functionDef) __declspec(deprecated) functionDef +#elif BEAST_GCC && ! BEAST_NO_DEPRECATION_WARNINGS + #define BEAST_DEPRECATED(functionDef) functionDef __attribute__ ((deprecated)) #else - #define BEAST_DEPRECATED(functionDef) functionDef + #define BEAST_DEPRECATED(functionDef) functionDef #endif -//============================================================================== +//------------------------------------------------------------------------------ + #if BEAST_ANDROID && ! DOXYGEN - #define BEAST_MODAL_LOOPS_PERMITTED 0 +# define BEAST_MODAL_LOOPS_PERMITTED 0 #elif ! defined (BEAST_MODAL_LOOPS_PERMITTED) - /** Some operating environments don't provide a modal loop mechanism, so this flag can be - used to disable any functions that try to run a modal loop. */ + /** Some operating environments don't provide a modal loop mechanism, so this + flag can be used to disable any functions that try to run a modal loop. + */ #define BEAST_MODAL_LOOPS_PERMITTED 1 #endif -//============================================================================== +//------------------------------------------------------------------------------ + #if BEAST_GCC - #define BEAST_PACKED __attribute__((packed)) +# define BEAST_PACKED __attribute__((packed)) #elif ! DOXYGEN - #define BEAST_PACKED +# define BEAST_PACKED #endif -//============================================================================== +//------------------------------------------------------------------------------ + // Here, we'll check for C++11 compiler support, and if it's not available, define // a few workarounds, so that we can still use some of the newer language features. #if defined (__GXX_EXPERIMENTAL_CXX0X__) && defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__) >= 405 - #define BEAST_COMPILER_SUPPORTS_NOEXCEPT 1 - #define BEAST_COMPILER_SUPPORTS_NULLPTR 1 - #define BEAST_COMPILER_SUPPORTS_MOVE_SEMANTICS 1 - - #if (__GNUC__ * 100 + __GNUC_MINOR__) >= 407 && ! defined (BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL) - #define BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL 1 - #endif +# define BEAST_COMPILER_SUPPORTS_NOEXCEPT 1 +# define BEAST_COMPILER_SUPPORTS_NULLPTR 1 +# define BEAST_COMPILER_SUPPORTS_MOVE_SEMANTICS 1 +# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 407 && ! defined (BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL) +# define BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL 1 +# endif #endif #if BEAST_CLANG && defined (__has_feature) - #if __has_feature (cxx_nullptr) - #define BEAST_COMPILER_SUPPORTS_NULLPTR 1 - #endif - - #if __has_feature (cxx_noexcept) - #define BEAST_COMPILER_SUPPORTS_NOEXCEPT 1 - #endif - - #if __has_feature (cxx_rvalue_references) - #define BEAST_COMPILER_SUPPORTS_MOVE_SEMANTICS 1 - #endif - - #ifndef BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL - #define BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL 1 - #endif - - #ifndef BEAST_COMPILER_SUPPORTS_ARC - #define BEAST_COMPILER_SUPPORTS_ARC 1 - #endif +# if __has_feature (cxx_nullptr) +# define BEAST_COMPILER_SUPPORTS_NULLPTR 1 +# endif +# if __has_feature (cxx_noexcept) +# define BEAST_COMPILER_SUPPORTS_NOEXCEPT 1 +# endif +# if __has_feature (cxx_rvalue_references) +# define BEAST_COMPILER_SUPPORTS_MOVE_SEMANTICS 1 +# endif +# ifndef BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL +# define BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL 1 +# endif +# ifndef BEAST_COMPILER_SUPPORTS_ARC +# define BEAST_COMPILER_SUPPORTS_ARC 1 +# endif #endif #if defined (_MSC_VER) && _MSC_VER >= 1600 - #define BEAST_COMPILER_SUPPORTS_NULLPTR 1 - #define BEAST_COMPILER_SUPPORTS_MOVE_SEMANTICS 1 +# define BEAST_COMPILER_SUPPORTS_NULLPTR 1 +# define BEAST_COMPILER_SUPPORTS_MOVE_SEMANTICS 1 #endif #if defined (_MSC_VER) && _MSC_VER >= 1700 - #define BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL 1 +# define BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL 1 #endif -//============================================================================== +//------------------------------------------------------------------------------ + // Declare some fake versions of nullptr and noexcept, for older compilers: #if ! (DOXYGEN || BEAST_COMPILER_SUPPORTS_NOEXCEPT) - #ifdef noexcept - #undef noexcept - #endif - #define noexcept throw() - #if defined (_MSC_VER) && _MSC_VER > 1600 - #define _ALLOW_KEYWORD_MACROS 1 // (to stop VC2012 complaining) - #endif +# ifdef noexcept +# undef noexcept +# endif +# define noexcept throw() +# if defined (_MSC_VER) && _MSC_VER > 1600 +# define _ALLOW_KEYWORD_MACROS 1 // (to stop VC2012 complaining) +# endif #endif #if ! (DOXYGEN || BEAST_COMPILER_SUPPORTS_NULLPTR) - #ifdef nullptr - #undef nullptr - #endif - #define nullptr (0) +#ifdef nullptr +#undef nullptr +#endif +#define nullptr (0) #endif #if ! (DOXYGEN || BEAST_COMPILER_SUPPORTS_OVERRIDE_AND_FINAL) - #undef override - #define override +#undef override +#define override #endif -#endif // BEAST_PLATFORMDEFS_BEASTHEADER +#endif diff --git a/Subtrees/beast/modules/beast_core/system/beast_StandardHeader.h b/Subtrees/beast/modules/beast_core/system/beast_StandardHeader.h index ce4e4f7efe..15d5bff463 100644 --- a/Subtrees/beast/modules/beast_core/system/beast_StandardHeader.h +++ b/Subtrees/beast/modules/beast_core/system/beast_StandardHeader.h @@ -24,10 +24,11 @@ #ifndef BEAST_STANDARDHEADER_BEASTHEADER #define BEAST_STANDARDHEADER_BEASTHEADER -//============================================================================== +//------------------------------------------------------------------------------ + /** Current BEAST version number. - See also SystemStats::getBEASTVersion() for a string version. + See also SystemStats::getBeastVersion() for a string version. */ #define BEAST_MAJOR_VERSION 0 #define BEAST_MINOR_VERSION 0 @@ -39,61 +40,87 @@ Bits 8 to 16 = minor version. Bits 0 to 8 = point release. - See also SystemStats::getBEASTVersion() for a string version. + See also SystemStats::getBeastVersion() for a string version. */ #define BEAST_VERSION ((BEAST_MAJOR_VERSION << 16) + (BEAST_MINOR_VERSION << 8) + BEAST_BUILDNUMBER) +//------------------------------------------------------------------------------ -//============================================================================== #include "beast_TargetPlatform.h" // (sets up the various BEAST_WINDOWS, BEAST_MAC, etc flags) #include "beast_PlatformDefs.h" -//============================================================================== // Now we'll include some common OS headers.. #if BEAST_MSVC - #pragma warning (push) - #pragma warning (disable: 4514 4245 4100) +#pragma warning (push) +#pragma warning (disable: 4514 4245 4100) #endif -#include -#include +#include +#include #include -#include #include -#include -#include -#include -#include +#include +#include #include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #if BEAST_USE_INTRINSICS - #include +# include #endif #if BEAST_MAC || BEAST_IOS - #include +# include #endif #if BEAST_LINUX - #include - - #if __INTEL_COMPILER - #if __ia64__ - #include - #else - #include - #endif - #endif +# include +# if __INTEL_COMPILER +# if __ia64__ +# include +# else +# include +# endif +# endif #endif #if BEAST_MSVC && BEAST_DEBUG - #include +# include +# include +# include #endif #if BEAST_MSVC - #pragma warning (pop) +#pragma warning (pop) #endif #if BEAST_ANDROID @@ -107,8 +134,12 @@ #undef max #undef min -//============================================================================== +//------------------------------------------------------------------------------ + // DLL building settings on Windows +// +// VFALCO TODO Deprecate this +// #if BEAST_MSVC #ifdef BEAST_DLL_BUILD #define BEAST_API __declspec (dllexport) @@ -124,43 +155,31 @@ #define BEAST_API __attribute__ ((visibility("default"))) #endif -//============================================================================== +//------------------------------------------------------------------------------ + #ifndef BEAST_API - #define BEAST_API /**< This macro is added to all beast public class declarations. */ +#define BEAST_API /**< This macro is added to all beast public class declarations. */ #endif #if BEAST_MSVC && BEAST_DLL_BUILD - #define BEAST_PUBLIC_IN_DLL_BUILD(declaration) public: declaration; private: +#define BEAST_PUBLIC_IN_DLL_BUILD(decl) public: decl; private: #else - #define BEAST_PUBLIC_IN_DLL_BUILD(declaration) declaration; +#define BEAST_PUBLIC_IN_DLL_BUILD(decl) decl; #endif /** This macro is added to all beast public function declarations. */ #define BEAST_PUBLIC_FUNCTION BEAST_API BEAST_CALLTYPE -#if (! defined (BEAST_CATCH_DEPRECATED_CODE_MISUSE)) && BEAST_DEBUG && ! DOXYGEN - /** This turns on some non-essential bits of code that should prevent old code from compiling - in cases where method signatures have changed, etc. - */ - #define BEAST_CATCH_DEPRECATED_CODE_MISUSE 1 -#endif +//------------------------------------------------------------------------------ -#ifndef DOXYGEN - #define BEAST_NAMESPACE beast // This old macro is deprecated: you should just use the beast namespace directly. -#endif - -//============================================================================== -// Now include some common headers... namespace beast { - extern BEAST_API bool BEAST_CALLTYPE beast_isRunningUnderDebugger(); - extern BEAST_API void BEAST_CALLTYPE logAssertion (const char* file, int line) noexcept; - #include "../memory/beast_Memory.h" - #include "../maths/beast_MathsFunctions.h" - #include "../memory/beast_ByteOrder.h" - #include "../logging/beast_Logger.h" - #include "../memory/beast_LeakedObjectDetector.h" +extern BEAST_API bool BEAST_CALLTYPE beast_isRunningUnderDebugger(); +extern BEAST_API void BEAST_CALLTYPE logAssertion (char const* file, int line) noexcept; + +// These are so common that we include them early +// ? } -#endif // BEAST_STANDARDHEADER_BEASTHEADER +#endif diff --git a/Subtrees/beast/modules/beast_core/system/beast_SystemStats.cpp b/Subtrees/beast/modules/beast_core/system/beast_SystemStats.cpp index 07a8b43c8c..1e50c7176d 100644 --- a/Subtrees/beast/modules/beast_core/system/beast_SystemStats.cpp +++ b/Subtrees/beast/modules/beast_core/system/beast_SystemStats.cpp @@ -27,7 +27,7 @@ const SystemStats::CPUFlags& SystemStats::getCPUFlags() return cpuFlags; } -String SystemStats::getBEASTVersion() +String SystemStats::getBeastVersion() { // Some basic tests, to keep an eye on things and make sure these types work ok // on all platforms. Let me know if any of these assertions fail on your system! @@ -41,7 +41,7 @@ String SystemStats::getBEASTVersion() static_bassert (sizeof (int64) == 8); static_bassert (sizeof (uint64) == 8); - return "BEAST v" BEAST_STRINGIFY(BEAST_MAJOR_VERSION) + return "Beast v" BEAST_STRINGIFY(BEAST_MAJOR_VERSION) "." BEAST_STRINGIFY(BEAST_MINOR_VERSION) "." BEAST_STRINGIFY(BEAST_BUILDNUMBER); } @@ -55,7 +55,7 @@ String SystemStats::getBEASTVersion() { BeastVersionPrinter() { - DBG (SystemStats::getBEASTVersion()); + DBG (SystemStats::getBeastVersion()); } }; @@ -79,7 +79,7 @@ String SystemStats::getStackBacktrace() int frames = (int) CaptureStackBackTrace (0, numElementsInArray (stack), stack, nullptr); HeapBlock symbol; - symbol.calloc (sizeof(SYMBOL_INFO) + 256, 1); + symbol.calloc (sizeof (SYMBOL_INFO) + 256, 1); symbol->MaxNameLen = 255; symbol->SizeOfStruct = sizeof (SYMBOL_INFO); @@ -131,6 +131,8 @@ static void handleCrash (int) globalCrashHandler(); kill (getpid(), SIGKILL); } + +int beast_siginterrupt (int sig, int flag); #endif void SystemStats::setApplicationCrashHandler (CrashHandlerFunction handler) @@ -146,7 +148,7 @@ void SystemStats::setApplicationCrashHandler (CrashHandlerFunction handler) for (int i = 0; i < numElementsInArray (signals); ++i) { ::signal (signals[i], handleCrash); - ::siginterrupt (signals[i], 1); + beast_siginterrupt (signals[i], 1); } #endif } diff --git a/Subtrees/beast/modules/beast_core/system/beast_SystemStats.h b/Subtrees/beast/modules/beast_core/system/beast_SystemStats.h index 761f7c91bf..41a6e79562 100644 --- a/Subtrees/beast/modules/beast_core/system/beast_SystemStats.h +++ b/Subtrees/beast/modules/beast_core/system/beast_SystemStats.h @@ -26,19 +26,18 @@ #include "../text/beast_StringArray.h" - //============================================================================== /** Contains methods for finding out about the current hardware and OS configuration. */ -class BEAST_API SystemStats +class BEAST_API SystemStats : Uncopyable { public: //============================================================================== /** Returns the current version of BEAST, See also the BEAST_VERSION, BEAST_MAJOR_VERSION and BEAST_MINOR_VERSION macros. */ - static String getBEASTVersion(); + static String getBeastVersion(); //============================================================================== /** The set of possible results of the getOperatingSystemType() method. */ @@ -46,16 +45,16 @@ public: { UnknownOS = 0, - Linux = 0x2000, - Android = 0x3000, - iOS = 0x8000, - MacOSX_10_4 = 0x1004, MacOSX_10_5 = 0x1005, MacOSX_10_6 = 0x1006, MacOSX_10_7 = 0x1007, MacOSX_10_8 = 0x1008, + Linux = 0x2000, + FreeBSD = 0x2001, + Android = 0x3000, + Win2000 = 0x4105, WinXP = 0x4106, WinVista = 0x4107, @@ -64,6 +63,8 @@ public: Windows = 0x4000, /**< To test whether any version of Windows is running, you can use the expression ((getOperatingSystemType() & Windows) != 0). */ + + iOS = 0x8000 }; /** Returns the type of operating system we're running on. @@ -193,8 +194,6 @@ private: SystemStats(); static const CPUFlags& getCPUFlags(); - - BEAST_DECLARE_NON_COPYABLE (SystemStats) }; diff --git a/Subtrees/beast/modules/beast_core/text/beast_LocalisedStrings.h b/Subtrees/beast/modules/beast_core/text/beast_LocalisedStrings.h index 8e805ec007..c8163b318c 100644 --- a/Subtrees/beast/modules/beast_core/text/beast_LocalisedStrings.h +++ b/Subtrees/beast/modules/beast_core/text/beast_LocalisedStrings.h @@ -70,7 +70,7 @@ get a list of all the messages by searching for the TRANS() macro in the Beast source code). */ -class BEAST_API LocalisedStrings +class BEAST_API LocalisedStrings : LeakChecked { public: //============================================================================== @@ -176,8 +176,6 @@ private: StringPairArray translations; void loadFromText (const String&, bool ignoreCase); - - BEAST_LEAK_DETECTOR (LocalisedStrings) }; //============================================================================== diff --git a/Subtrees/beast/modules/beast_core/text/beast_String.cpp b/Subtrees/beast/modules/beast_core/text/beast_String.cpp index 45a64ed674..8bd0c53179 100644 --- a/Subtrees/beast/modules/beast_core/text/beast_String.cpp +++ b/Subtrees/beast/modules/beast_core/text/beast_String.cpp @@ -131,10 +131,11 @@ public: if (start.getAddress() == nullptr || start.isEmpty()) return getEmpty(); - const size_t numBytes = (size_t) (end.getAddress() - start.getAddress()); - const CharPointerType dest (createUninitialisedBytes (numBytes + 1)); + const size_t numBytes = (size_t)( reinterpret_cast (end.getAddress()) + - reinterpret_cast (start.getAddress())); + const CharPointerType dest (createUninitialisedBytes (numBytes + sizeof (CharType))); memcpy (dest.getAddress(), start, numBytes); - dest.getAddress()[numBytes] = 0; + dest.getAddress()[numBytes / sizeof (CharType)] = 0; return dest; } @@ -1199,8 +1200,8 @@ public: dest = result.getCharPointer(); } - StringCreationHelper (const String::CharPointerType& source_) - : source (source_), dest (nullptr), allocatedBytes (StringHolder::getAllocatedNumBytes (source)), bytesWritten (0) + StringCreationHelper (const String::CharPointerType s) + : source (s), dest (nullptr), allocatedBytes (StringHolder::getAllocatedNumBytes (s)), bytesWritten (0) { result.preallocateBytes (allocatedBytes); dest = result.getCharPointer(); @@ -1530,7 +1531,8 @@ String String::quoted (const beast_wchar quoteCharacter) const } //============================================================================== -static String::CharPointerType findTrimmedEnd (const String::CharPointerType& start, String::CharPointerType end) +static String::CharPointerType findTrimmedEnd (const String::CharPointerType start, + String::CharPointerType end) { while (end > start) { diff --git a/Subtrees/beast/modules/beast_core/text/beast_String.h b/Subtrees/beast/modules/beast_core/text/beast_String.h index 3a4ba8df9a..e3c41b651d 100644 --- a/Subtrees/beast/modules/beast_core/text/beast_String.h +++ b/Subtrees/beast/modules/beast_core/text/beast_String.h @@ -1219,7 +1219,7 @@ private: explicit String (const PreallocationBytes&); // This constructor preallocates a certain amount of memory void appendFixedLength (const char* text, int numExtraChars); size_t getByteOffsetOfEnd() const noexcept; - BEAST_DEPRECATED (String (const String& stringToCopy, size_t charsToAllocate)); + BEAST_DEPRECATED (String (const String&, size_t)); // This private cast operator should prevent strings being accidentally cast // to bools (this is possible because the compiler can add an implicit cast diff --git a/Subtrees/beast/modules/beast_core/text/beast_StringArray.h b/Subtrees/beast/modules/beast_core/text/beast_StringArray.h index 26d1536cea..1b0978fe1e 100644 --- a/Subtrees/beast/modules/beast_core/text/beast_StringArray.h +++ b/Subtrees/beast/modules/beast_core/text/beast_StringArray.h @@ -34,7 +34,7 @@ @see String, StringPairArray */ -class BEAST_API StringArray +class BEAST_API StringArray : LeakChecked { public: //============================================================================== @@ -396,10 +396,7 @@ public: private: - //============================================================================== Array strings; - - BEAST_LEAK_DETECTOR (StringArray) }; diff --git a/Subtrees/beast/modules/beast_core/text/beast_StringPairArray.h b/Subtrees/beast/modules/beast_core/text/beast_StringPairArray.h index a3eac9b3bf..3e3fdfc93f 100644 --- a/Subtrees/beast/modules/beast_core/text/beast_StringPairArray.h +++ b/Subtrees/beast/modules/beast_core/text/beast_StringPairArray.h @@ -33,7 +33,7 @@ @see StringArray */ -class BEAST_API StringPairArray +class BEAST_API StringPairArray : LeakChecked { public: //============================================================================== @@ -150,8 +150,6 @@ private: //============================================================================== StringArray keys, values; bool ignoreCase; - - BEAST_LEAK_DETECTOR (StringPairArray) }; diff --git a/Subtrees/beast/modules/beast_core/text/beast_TextDiff.cpp b/Subtrees/beast/modules/beast_core/text/beast_TextDiff.cpp index 5739da32ea..18645157d6 100644 --- a/Subtrees/beast/modules/beast_core/text/beast_TextDiff.cpp +++ b/Subtrees/beast/modules/beast_core/text/beast_TextDiff.cpp @@ -30,14 +30,14 @@ struct TextDiffHelpers StringRegion (const String& s) noexcept : text (s.getCharPointer()), start (0), length (s.length()) {} - StringRegion (const String::CharPointerType& t, int s, int len) noexcept + StringRegion (const String::CharPointerType t, int s, int len) noexcept : text (t), start (s), length (len) {} String::CharPointerType text; int start, length; }; - static void addInsertion (TextDiff& td, const String::CharPointerType& text, int index, int length) + static void addInsertion (TextDiff& td, const String::CharPointerType text, int index, int length) { TextDiff::Change c; c.insertedText = String (text, (size_t) length); @@ -99,7 +99,7 @@ struct TextDiffHelpers } static int findLongestCommonSubstring (String::CharPointerType a, const int lenA, - const String::CharPointerType& b, const int lenB, + const String::CharPointerType b, const int lenB, int& indexInA, int& indexInB) { if (lenA == 0 || lenB == 0) diff --git a/Subtrees/beast/modules/beast_core/threads/beast_ChildProcess.h b/Subtrees/beast/modules/beast_core/threads/beast_ChildProcess.h index 4875efd608..9efead13b1 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_ChildProcess.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_ChildProcess.h @@ -32,7 +32,7 @@ This class lets you launch an executable, and read its output. You can also use it to check whether the child process has finished. */ -class BEAST_API ChildProcess +class BEAST_API ChildProcess : LeakChecked , Uncopyable { public: //============================================================================== @@ -92,8 +92,6 @@ private: class ActiveProcess; friend class ScopedPointer; ScopedPointer activeProcess; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ChildProcess) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_CriticalSection.h b/Subtrees/beast/modules/beast_core/threads/beast_CriticalSection.h index d0da916418..7c8015e8a9 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_CriticalSection.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_CriticalSection.h @@ -37,7 +37,7 @@ @see ScopedLock, ScopedTryLock, ScopedUnlock, SpinLock, ReadWriteLock, Thread, InterProcessLock */ -class BEAST_API CriticalSection +class BEAST_API CriticalSection : Uncopyable { public: //============================================================================== @@ -111,8 +111,6 @@ private: #else mutable pthread_mutex_t internal; #endif - - BEAST_DECLARE_NON_COPYABLE (CriticalSection) }; @@ -126,7 +124,7 @@ private: @see CriticalSection, Array, OwnedArray, ReferenceCountedArray */ -class BEAST_API DummyCriticalSection +class BEAST_API DummyCriticalSection : Uncopyable { public: inline DummyCriticalSection() noexcept {} @@ -145,9 +143,6 @@ public: /** A dummy scoped-unlocker type to use with a dummy critical section. */ typedef ScopedLockType ScopedUnlockType; - -private: - BEAST_DECLARE_NON_COPYABLE (DummyCriticalSection) }; //============================================================================== diff --git a/Subtrees/beast/modules/beast_core/threads/beast_DynamicLibrary.h b/Subtrees/beast/modules/beast_core/threads/beast_DynamicLibrary.h index 9b99c6a8a3..0330163a96 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_DynamicLibrary.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_DynamicLibrary.h @@ -31,7 +31,7 @@ Since the DLL is freed when this object is deleted, it's handy for managing library lifetimes using RAII. */ -class BEAST_API DynamicLibrary +class BEAST_API DynamicLibrary : LeakChecked , Uncopyable { public: /** Creates an unopened DynamicLibrary object. @@ -72,8 +72,6 @@ public: private: void* handle; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (DynamicLibrary) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_HighResolutionTimer.h b/Subtrees/beast/modules/beast_core/threads/beast_HighResolutionTimer.h index a42674f911..01f38fdc29 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_HighResolutionTimer.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_HighResolutionTimer.h @@ -38,7 +38,7 @@ @see Timer */ -class BEAST_API HighResolutionTimer +class BEAST_API HighResolutionTimer : LeakChecked , Uncopyable { protected: /** Creates a HighResolutionTimer. @@ -96,8 +96,6 @@ private: friend struct Pimpl; friend class ScopedPointer; ScopedPointer pimpl; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (HighResolutionTimer) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_InterProcessLock.h b/Subtrees/beast/modules/beast_core/threads/beast_InterProcessLock.h index 946f29580d..7e4469f8c6 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_InterProcessLock.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_InterProcessLock.h @@ -34,7 +34,7 @@ @see CriticalSection */ -class BEAST_API InterProcessLock +class BEAST_API InterProcessLock : Uncopyable { public: //============================================================================== @@ -71,7 +71,7 @@ public: @see ScopedLock */ - class ScopedLockType + class ScopedLockType : Uncopyable { public: //============================================================================== @@ -106,8 +106,6 @@ public: //============================================================================== InterProcessLock& ipLock; bool lockWasSuccessful; - - BEAST_DECLARE_NON_COPYABLE (ScopedLockType) }; private: @@ -118,8 +116,6 @@ private: CriticalSection lock; String name; - - BEAST_DECLARE_NON_COPYABLE (InterProcessLock) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_Process.h b/Subtrees/beast/modules/beast_core/threads/beast_Process.h index 05a9f1f367..87b8262d80 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_Process.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_Process.h @@ -35,7 +35,7 @@ @see Thread, BEASTApplication */ -class BEAST_API Process +class BEAST_API Process : Uncopyable { public: //============================================================================== @@ -90,11 +90,9 @@ public: */ static void lowerPrivilege(); - //============================================================================== /** Returns true if this process is being hosted by a debugger. */ static bool BEAST_CALLTYPE isRunningUnderDebugger(); - //============================================================================== /** Tries to launch the OS's default reader application for a given file or URL. */ static bool openDocument (const String& documentURL, const String& parameters); @@ -140,7 +138,6 @@ public: private: Process(); - BEAST_DECLARE_NON_COPYABLE (Process) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_ReadWriteLock.h b/Subtrees/beast/modules/beast_core/threads/beast_ReadWriteLock.h index 1b9d961e02..880cae41b1 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_ReadWriteLock.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_ReadWriteLock.h @@ -50,7 +50,7 @@ @see ScopedReadLock, ScopedWriteLock, CriticalSection */ -class BEAST_API ReadWriteLock +class BEAST_API ReadWriteLock : Uncopyable { public: //============================================================================== @@ -146,8 +146,6 @@ private: }; mutable Array readerThreads; - - BEAST_DECLARE_NON_COPYABLE (ReadWriteLock) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_ScopedLock.h b/Subtrees/beast/modules/beast_core/threads/beast_ScopedLock.h index cd19d5df60..91cc5473de 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_ScopedLock.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_ScopedLock.h @@ -51,7 +51,7 @@ @see GenericScopedUnlock, CriticalSection, SpinLock, ScopedLock, ScopedUnlock */ template -class GenericScopedLock +class GenericScopedLock : Uncopyable { public: //============================================================================== @@ -76,8 +76,6 @@ public: private: //============================================================================== const LockType& lock_; - - BEAST_DECLARE_NON_COPYABLE (GenericScopedLock) }; @@ -120,7 +118,7 @@ private: @see GenericScopedLock, CriticalSection, ScopedLock, ScopedUnlock */ template -class GenericScopedUnlock +class GenericScopedUnlock : Uncopyable { public: //============================================================================== @@ -149,8 +147,6 @@ public: private: //============================================================================== const LockType& lock_; - - BEAST_DECLARE_NON_COPYABLE (GenericScopedUnlock) }; @@ -190,7 +186,7 @@ private: @see CriticalSection::tryEnter, GenericScopedLock, GenericScopedUnlock */ template -class GenericScopedTryLock +class GenericScopedTryLock : Uncopyable { public: //============================================================================== @@ -224,8 +220,6 @@ private: //============================================================================== const LockType& lock_; const bool lockWasSuccessful; - - BEAST_DECLARE_NON_COPYABLE (GenericScopedTryLock) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_ScopedReadLock.h b/Subtrees/beast/modules/beast_core/threads/beast_ScopedReadLock.h index 0ec698c6ae..9ec3c1ff55 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_ScopedReadLock.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_ScopedReadLock.h @@ -50,7 +50,7 @@ @see ReadWriteLock, ScopedWriteLock */ -class BEAST_API ScopedReadLock +class BEAST_API ScopedReadLock : Uncopyable { public: //============================================================================== @@ -79,8 +79,6 @@ public: private: //============================================================================== const ReadWriteLock& lock_; - - BEAST_DECLARE_NON_COPYABLE (ScopedReadLock) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_ScopedWriteLock.h b/Subtrees/beast/modules/beast_core/threads/beast_ScopedWriteLock.h index fc0894fdb1..8bc88ccbdd 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_ScopedWriteLock.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_ScopedWriteLock.h @@ -50,7 +50,7 @@ @see ReadWriteLock, ScopedReadLock */ -class BEAST_API ScopedWriteLock +class BEAST_API ScopedWriteLock : Uncopyable { public: //============================================================================== @@ -79,8 +79,6 @@ public: private: //============================================================================== const ReadWriteLock& lock_; - - BEAST_DECLARE_NON_COPYABLE (ScopedWriteLock) }; diff --git a/Subtrees/beast/modules/beast_basics/native/beast_win32_Threads.cpp b/Subtrees/beast/modules/beast_core/threads/beast_SpinDelay.cpp similarity index 89% rename from Subtrees/beast/modules/beast_basics/native/beast_win32_Threads.cpp rename to Subtrees/beast/modules/beast_core/threads/beast_SpinDelay.cpp index b6865169f0..959dc42f60 100644 --- a/Subtrees/beast/modules/beast_basics/native/beast_win32_Threads.cpp +++ b/Subtrees/beast/modules/beast_core/threads/beast_SpinDelay.cpp @@ -16,3 +16,14 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== + +SpinDelay::SpinDelay () + : m_count (0) +{ +} + +void SpinDelay::pause () +{ + if (++m_count > 20) + Thread::yield (); +} diff --git a/Subtrees/beast/modules/beast_basics/threads/beast_SpinDelay.h b/Subtrees/beast/modules/beast_core/threads/beast_SpinDelay.h similarity index 86% rename from Subtrees/beast/modules/beast_basics/threads/beast_SpinDelay.h rename to Subtrees/beast/modules/beast_core/threads/beast_SpinDelay.h index 03694706a1..c51d4ce4ce 100644 --- a/Subtrees/beast/modules/beast_basics/threads/beast_SpinDelay.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_SpinDelay.h @@ -20,22 +20,14 @@ #ifndef BEAST_SPINDELAY_BEASTHEADER #define BEAST_SPINDELAY_BEASTHEADER -// -// Synchronization element -// - -class SpinDelay +/** A simple delay used to synchronize threads. +*/ +class BEAST_API SpinDelay { public: - SpinDelay () : m_count (0) - { - } + SpinDelay (); - inline void pause () - { - if (++m_count > 20) - Thread::yield (); - } + void pause (); private: int m_count; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_SpinLock.h b/Subtrees/beast/modules/beast_core/threads/beast_SpinLock.h index 4d773a6042..f8eec63a7b 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_SpinLock.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_SpinLock.h @@ -40,7 +40,7 @@ @see CriticalSection */ -class BEAST_API SpinLock +class BEAST_API SpinLock : Uncopyable { public: inline SpinLock() noexcept {} @@ -80,8 +80,6 @@ public: private: //============================================================================== mutable Atomic lock; - - BEAST_DECLARE_NON_COPYABLE (SpinLock) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_Thread.cpp b/Subtrees/beast/modules/beast_core/threads/beast_Thread.cpp index b891733651..dfebf33aaa 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_Thread.cpp +++ b/Subtrees/beast/modules/beast_core/threads/beast_Thread.cpp @@ -48,14 +48,12 @@ Thread::~Thread() //============================================================================== // Use a ref-counted object to hold this shared data, so that it can outlive its static // shared pointer when threads are still running during static shutdown. -struct CurrentThreadHolder : public ReferenceCountedObject +struct CurrentThreadHolder : public ReferenceCountedObject { CurrentThreadHolder() noexcept {} typedef ReferenceCountedObjectPtr Ptr; ThreadLocalValue value; - - BEAST_DECLARE_NON_COPYABLE (CurrentThreadHolder) }; static char currentThreadHolderLock [sizeof (SpinLock)]; // (statically initialised to zeros). diff --git a/Subtrees/beast/modules/beast_core/threads/beast_Thread.h b/Subtrees/beast/modules/beast_core/threads/beast_Thread.h index e553913da6..e0d28b64ae 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_Thread.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_Thread.h @@ -42,7 +42,7 @@ @see CriticalSection, WaitableEvent, Process, ThreadWithProgressWindow, MessageManagerLock */ -class BEAST_API Thread +class BEAST_API Thread : LeakChecked , Uncopyable { public: //============================================================================== @@ -280,8 +280,6 @@ private: void killThread(); void threadEntryPoint(); static bool setThreadPriority (void*, int); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (Thread) }; #endif // BEAST_THREAD_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/threads/beast_ThreadLocalValue.h b/Subtrees/beast/modules/beast_core/threads/beast_ThreadLocalValue.h index 2a0bd68fe4..e3aede01fb 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_ThreadLocalValue.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_ThreadLocalValue.h @@ -51,7 +51,7 @@ is deleted. */ template -class ThreadLocalValue +class ThreadLocalValue : Uncopyable { public: /** */ @@ -170,7 +170,7 @@ public: private: //============================================================================== #if BEAST_NO_COMPILER_THREAD_LOCAL - struct ObjectHolder + struct ObjectHolder : Uncopyable { ObjectHolder (const Thread::ThreadID& tid) : threadId (tid), object() @@ -179,15 +179,11 @@ private: Thread::ThreadID threadId; ObjectHolder* next; Type object; - - BEAST_DECLARE_NON_COPYABLE (ObjectHolder) }; mutable Atomic first; SpinLock lock; #endif - - BEAST_DECLARE_NON_COPYABLE (ThreadLocalValue) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_ThreadPool.cpp b/Subtrees/beast/modules/beast_core/threads/beast_ThreadPool.cpp index c59e1550b3..951383066c 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_ThreadPool.cpp +++ b/Subtrees/beast/modules/beast_core/threads/beast_ThreadPool.cpp @@ -53,7 +53,9 @@ void ThreadPoolJob::signalJobShouldExit() } //============================================================================== -class ThreadPool::ThreadPoolThread : public Thread +class ThreadPool::ThreadPoolThread + : public Thread + , LeakChecked { public: ThreadPoolThread (ThreadPool& pool_) @@ -73,8 +75,6 @@ public: private: ThreadPool& pool; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ThreadPoolThread) }; //============================================================================== diff --git a/Subtrees/beast/modules/beast_core/threads/beast_ThreadPool.h b/Subtrees/beast/modules/beast_core/threads/beast_ThreadPool.h index d4d5dfe835..81bc7a73b5 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_ThreadPool.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_ThreadPool.h @@ -46,7 +46,7 @@ class ThreadPoolThread; @see ThreadPool, Thread */ -class BEAST_API ThreadPoolJob +class BEAST_API ThreadPoolJob : LeakChecked , Uncopyable { public: //============================================================================== @@ -125,8 +125,6 @@ private: String jobName; ThreadPool* pool; bool shouldStop, isActive, shouldBeDeleted; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ThreadPoolJob) }; @@ -139,7 +137,7 @@ private: @see ThreadPoolJob, Thread */ -class BEAST_API ThreadPool +class BEAST_API ThreadPool : LeakChecked , Uncopyable { public: //============================================================================== @@ -305,8 +303,6 @@ private: // Note that this method has changed, and no longer has a parameter to indicate // whether the jobs should be deleted - see the new method for details. void removeAllJobs (bool, int, bool); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ThreadPool) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_TimeSliceThread.h b/Subtrees/beast/modules/beast_core/threads/beast_TimeSliceThread.h index fb783d233a..ca0b7cc25a 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_TimeSliceThread.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_TimeSliceThread.h @@ -79,7 +79,9 @@ private: @see TimeSliceClient, Thread */ -class BEAST_API TimeSliceThread : public Thread +class BEAST_API TimeSliceThread + : public Thread + , LeakChecked { public: //============================================================================== @@ -139,8 +141,6 @@ private: TimeSliceClient* clientBeingCalled; TimeSliceClient* getNextClient (int index) const; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (TimeSliceThread) }; diff --git a/Subtrees/beast/modules/beast_core/threads/beast_WaitableEvent.h b/Subtrees/beast/modules/beast_core/threads/beast_WaitableEvent.h index e95ef02044..1a01d7e5c5 100644 --- a/Subtrees/beast/modules/beast_core/threads/beast_WaitableEvent.h +++ b/Subtrees/beast/modules/beast_core/threads/beast_WaitableEvent.h @@ -35,7 +35,7 @@ calling thread until another thread wakes it up by calling the signal() method. */ -class BEAST_API WaitableEvent +class BEAST_API WaitableEvent : LeakChecked , Uncopyable { public: //============================================================================== @@ -45,7 +45,7 @@ public: method is called. If manualReset is true, then once the event is signalled, the only way to reset it will be by calling the reset() method. */ - WaitableEvent (bool manualReset = false) noexcept; + explicit WaitableEvent (bool manualReset = false) noexcept; /** Destructor. @@ -106,8 +106,6 @@ private: mutable pthread_mutex_t mutex; mutable bool triggered, manualReset; #endif - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (WaitableEvent) }; diff --git a/Subtrees/beast/modules/beast_core/time/beast_PerformanceCounter.cpp b/Subtrees/beast/modules/beast_core/time/beast_PerformanceCounter.cpp index fe1f82c8af..6231e11b1e 100644 --- a/Subtrees/beast/modules/beast_core/time/beast_PerformanceCounter.cpp +++ b/Subtrees/beast/modules/beast_core/time/beast_PerformanceCounter.cpp @@ -30,7 +30,7 @@ PerformanceCounter::PerformanceCounter (const String& name_, totalTime (0), outputFile (loggingFile) { - if (outputFile != File::nonexistent) + if (outputFile != File::nonexistent ()) { String s ("**** Counter for \""); s << name_ << "\" started at: " @@ -81,7 +81,7 @@ void PerformanceCounter::printStatistics() s << newLine; - if (outputFile != File::nonexistent) + if (outputFile != File::nonexistent ()) outputFile.appendText (s, false, false); numRuns = 0; diff --git a/Subtrees/beast/modules/beast_core/time/beast_PerformanceCounter.h b/Subtrees/beast/modules/beast_core/time/beast_PerformanceCounter.h index f826511382..039876487a 100644 --- a/Subtrees/beast/modules/beast_core/time/beast_PerformanceCounter.h +++ b/Subtrees/beast/modules/beast_core/time/beast_PerformanceCounter.h @@ -57,12 +57,13 @@ public: @param counterName the name used when printing out the statistics @param runsPerPrintout the number of start/stop iterations before calling printStatistics() - @param loggingFile a file to dump the results to - if this is File::nonexistent, - the results are just written to the debugger output + @param loggingFile a file to dump the results to - if this is + File::nonexistent (), the results are just written + to the debugger output */ PerformanceCounter (const String& counterName, int runsPerPrintout = 100, - const File& loggingFile = File::nonexistent); + const File& loggingFile = File::nonexistent ()); /** Destructor. */ ~PerformanceCounter(); diff --git a/Subtrees/beast/modules/beast_basics/events/beast_PerformedAtExit.cpp b/Subtrees/beast/modules/beast_core/time/beast_PerformedAtExit.cpp similarity index 70% rename from Subtrees/beast/modules/beast_basics/events/beast_PerformedAtExit.cpp rename to Subtrees/beast/modules/beast_core/time/beast_PerformedAtExit.cpp index 896dc98c00..7babb0d2dd 100644 --- a/Subtrees/beast/modules/beast_basics/events/beast_PerformedAtExit.cpp +++ b/Subtrees/beast/modules/beast_core/time/beast_PerformedAtExit.cpp @@ -17,14 +17,16 @@ */ //============================================================================== -class PerformedAtExit::Performer +class PerformedAtExit::ExitHook { public: typedef Static::Storage , PerformedAtExit> StackType; private: - ~Performer () + ~ExitHook () { + // Call all PerformedAtExit objects + // PerformedAtExit* object = s_list->pop_front (); while (object != nullptr) @@ -34,7 +36,9 @@ private: object = s_list->pop_front (); } - LeakCheckedBase::detectAllLeaks (); + // Now do the leak checking + // + LeakCheckedBase::checkForLeaks (); } public: @@ -48,19 +52,23 @@ private: static StackType s_list; - static Performer s_performer; + static ExitHook s_performer; }; -PerformedAtExit::Performer PerformedAtExit::Performer::s_performer; -PerformedAtExit::Performer::StackType PerformedAtExit::Performer::s_list; +PerformedAtExit::ExitHook PerformedAtExit::ExitHook::s_performer; +PerformedAtExit::ExitHook::StackType PerformedAtExit::ExitHook::s_list; PerformedAtExit::PerformedAtExit () { #if BEAST_IOS - // TODO: PerformedAtExit::Performer::push_front crashes on iOS if s_storage is not accessed before used - char* hack = PerformedAtExit::Performer::s_list.s_storage; + // Patrick Dehne: + // PerformedAtExit::ExitHook::push_front crashes on iOS + // if s_storage is not accessed before used + // + // VFALCO TODO Figure out why and fix it cleanly if needed. + // + char* hack = PerformedAtExit::ExitHook::s_list.s_storage; #endif - Performer::push_front (this); + ExitHook::push_front (this); } - diff --git a/Subtrees/beast/modules/beast_basics/events/beast_PerformedAtExit.h b/Subtrees/beast/modules/beast_core/time/beast_PerformedAtExit.h similarity index 84% rename from Subtrees/beast/modules/beast_basics/events/beast_PerformedAtExit.h rename to Subtrees/beast/modules/beast_core/time/beast_PerformedAtExit.h index 7bd0abbd41..0c79d3646f 100644 --- a/Subtrees/beast/modules/beast_basics/events/beast_PerformedAtExit.h +++ b/Subtrees/beast/modules/beast_core/time/beast_PerformedAtExit.h @@ -20,8 +20,6 @@ #ifndef BEAST_PERFORMEDATEXIT_BEASTHEADER #define BEAST_PERFORMEDATEXIT_BEASTHEADER -#include "../containers/beast_LockFreeStack.h" - /*============================================================================*/ /** Perform an action at program exit @@ -32,8 +30,15 @@ @ingroup beast_core */ -class PerformedAtExit : public LockFreeStack ::Node +// VFALCO TODO Make the linked list element a private type and use composition +// instead of inheritance, so that PerformedAtExit doesn't expose +// lock free stack node interfaces. +// +class BEAST_API PerformedAtExit : public LockFreeStack ::Node { +public: + class ExitHook; + protected: PerformedAtExit (); virtual ~PerformedAtExit () { } @@ -42,9 +47,6 @@ protected: /** Called at program exit. */ virtual void performAtExit () = 0; - -private: - class Performer; }; #endif diff --git a/Subtrees/beast/modules/beast_core/unit_tests/beast_UnitTest.h b/Subtrees/beast/modules/beast_core/unit_tests/beast_UnitTest.h index d155f0d7b5..96eb2c89b0 100644 --- a/Subtrees/beast/modules/beast_core/unit_tests/beast_UnitTest.h +++ b/Subtrees/beast/modules/beast_core/unit_tests/beast_UnitTest.h @@ -67,7 +67,7 @@ class UnitTestRunner; @see UnitTestRunner */ -class BEAST_API UnitTest +class BEAST_API UnitTest : Uncopyable { public: //============================================================================== @@ -164,8 +164,6 @@ private: //============================================================================== const String name; UnitTestRunner* runner; - - BEAST_DECLARE_NON_COPYABLE (UnitTest) }; @@ -181,7 +179,7 @@ private: @see UnitTest */ -class BEAST_API UnitTestRunner +class BEAST_API UnitTestRunner : Uncopyable { public: //============================================================================== @@ -277,8 +275,6 @@ private: void addPass(); void addFail (const String& failureMessage); - - BEAST_DECLARE_NON_COPYABLE (UnitTestRunner) }; diff --git a/Subtrees/beast/modules/beast_core/xml/beast_XmlDocument.h b/Subtrees/beast/modules/beast_core/xml/beast_XmlDocument.h index 443902225e..3e51feecad 100644 --- a/Subtrees/beast/modules/beast_core/xml/beast_XmlDocument.h +++ b/Subtrees/beast/modules/beast_core/xml/beast_XmlDocument.h @@ -67,7 +67,7 @@ class InputSource; @see XmlElement */ -class BEAST_API XmlDocument +class BEAST_API XmlDocument : LeakChecked , Uncopyable { public: //============================================================================== @@ -173,8 +173,6 @@ private: String expandEntity (const String& entity); String expandExternalEntity (const String& entity); String getParameterEntity (const String& entity); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (XmlDocument) }; diff --git a/Subtrees/beast/modules/beast_core/xml/beast_XmlElement.h b/Subtrees/beast/modules/beast_core/xml/beast_XmlElement.h index 9705b78ea8..042c2274f6 100644 --- a/Subtrees/beast/modules/beast_core/xml/beast_XmlElement.h +++ b/Subtrees/beast/modules/beast_core/xml/beast_XmlElement.h @@ -139,7 +139,7 @@ @see XmlDocument */ -class BEAST_API XmlElement +class BEAST_API XmlElement : LeakChecked { public: //============================================================================== @@ -726,8 +726,6 @@ private: void writeElementAsText (OutputStream&, int indentationLevel, int lineWrapLength) const; void getChildElementsAsArray (XmlElement**) const noexcept; void reorderChildElements (XmlElement**, int) noexcept; - - BEAST_LEAK_DETECTOR (XmlElement) }; diff --git a/Subtrees/beast/modules/beast_core/zip/beast_GZIPCompressorOutputStream.cpp b/Subtrees/beast/modules/beast_core/zip/beast_GZIPCompressorOutputStream.cpp index a2467d15f8..725aba87f1 100644 --- a/Subtrees/beast/modules/beast_core/zip/beast_GZIPCompressorOutputStream.cpp +++ b/Subtrees/beast/modules/beast_core/zip/beast_GZIPCompressorOutputStream.cpp @@ -26,7 +26,7 @@ ============================================================================== */ -class GZIPCompressorOutputStream::GZIPCompressorHelper +class GZIPCompressorOutputStream::GZIPCompressorHelper : Uncopyable { public: GZIPCompressorHelper (const int compressionLevel, const int windowBits) @@ -114,8 +114,6 @@ private: return false; } - - BEAST_DECLARE_NON_COPYABLE (GZIPCompressorHelper) }; //============================================================================== diff --git a/Subtrees/beast/modules/beast_core/zip/beast_GZIPCompressorOutputStream.h b/Subtrees/beast/modules/beast_core/zip/beast_GZIPCompressorOutputStream.h index 6d7a66d6f5..c083afc45f 100644 --- a/Subtrees/beast/modules/beast_core/zip/beast_GZIPCompressorOutputStream.h +++ b/Subtrees/beast/modules/beast_core/zip/beast_GZIPCompressorOutputStream.h @@ -44,7 +44,9 @@ @see GZIPDecompressorInputStream */ -class BEAST_API GZIPCompressorOutputStream : public OutputStream +class BEAST_API GZIPCompressorOutputStream + : public OutputStream + , LeakChecked { public: //============================================================================== @@ -98,8 +100,6 @@ private: class GZIPCompressorHelper; friend class ScopedPointer ; ScopedPointer helper; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (GZIPCompressorOutputStream) }; #endif // BEAST_GZIPCOMPRESSOROUTPUTSTREAM_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/zip/beast_GZIPDecompressorInputStream.cpp b/Subtrees/beast/modules/beast_core/zip/beast_GZIPDecompressorInputStream.cpp index ad1c8d4dd0..5f69fa68aa 100644 --- a/Subtrees/beast/modules/beast_core/zip/beast_GZIPDecompressorInputStream.cpp +++ b/Subtrees/beast/modules/beast_core/zip/beast_GZIPDecompressorInputStream.cpp @@ -82,7 +82,7 @@ namespace zlibNamespace //============================================================================== // internal helper object that holds the zlib structures so they don't have to be // included publicly. -class GZIPDecompressorInputStream::GZIPDecompressHelper +class GZIPDecompressorInputStream::GZIPDecompressHelper : Uncopyable { public: GZIPDecompressHelper (const bool dontWrap) @@ -160,8 +160,6 @@ private: zlibNamespace::z_stream stream; uint8* data; size_t dataSize; - - BEAST_DECLARE_NON_COPYABLE (GZIPDecompressHelper) }; //============================================================================== diff --git a/Subtrees/beast/modules/beast_core/zip/beast_GZIPDecompressorInputStream.h b/Subtrees/beast/modules/beast_core/zip/beast_GZIPDecompressorInputStream.h index 8dfc79cff7..e77444a0ae 100644 --- a/Subtrees/beast/modules/beast_core/zip/beast_GZIPDecompressorInputStream.h +++ b/Subtrees/beast/modules/beast_core/zip/beast_GZIPDecompressorInputStream.h @@ -44,7 +44,9 @@ @see GZIPCompressorOutputStream */ -class BEAST_API GZIPDecompressorInputStream : public InputStream +class BEAST_API GZIPDecompressorInputStream + : public InputStream + , LeakChecked { public: //============================================================================== @@ -95,8 +97,6 @@ private: class GZIPDecompressHelper; friend class ScopedPointer ; ScopedPointer helper; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (GZIPDecompressorInputStream) }; #endif // BEAST_GZIPDECOMPRESSORINPUTSTREAM_BEASTHEADER diff --git a/Subtrees/beast/modules/beast_core/zip/beast_ZipFile.cpp b/Subtrees/beast/modules/beast_core/zip/beast_ZipFile.cpp index 66bdf071ed..7ace22e120 100644 --- a/Subtrees/beast/modules/beast_core/zip/beast_ZipFile.cpp +++ b/Subtrees/beast/modules/beast_core/zip/beast_ZipFile.cpp @@ -111,7 +111,9 @@ namespace } //============================================================================== -class ZipFile::ZipInputStream : public InputStream +class ZipFile::ZipInputStream + : public InputStream + , LeakChecked { public: ZipInputStream (ZipFile& zf, ZipFile::ZipEntryHolder& zei) @@ -208,8 +210,6 @@ private: int headerSize; InputStream* inputStream; ScopedPointer streamToDelete; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ZipInputStream) }; @@ -442,7 +442,7 @@ Result ZipFile::uncompressEntry (const int index, //============================================================================= extern unsigned long beast_crc32 (unsigned long crc, const unsigned char*, unsigned len); -class ZipFile::Builder::Item +class ZipFile::Builder::Item : LeakChecked , Uncopyable { public: Item (const File& f, const int compression, const String& storedPath) @@ -547,8 +547,6 @@ private: target.writeShort ((short) storedPathname.toUTF8().sizeInBytes() - 1); target.writeShort (0); // extra field length } - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (Item) }; //============================================================================= diff --git a/Subtrees/beast/modules/beast_core/zip/beast_ZipFile.h b/Subtrees/beast/modules/beast_core/zip/beast_ZipFile.h index eabedab4c9..d137d3fefc 100644 --- a/Subtrees/beast/modules/beast_core/zip/beast_ZipFile.h +++ b/Subtrees/beast/modules/beast_core/zip/beast_ZipFile.h @@ -42,7 +42,7 @@ This can enumerate the items in a ZIP file and can create suitable stream objects to read each one. */ -class BEAST_API ZipFile +class BEAST_API ZipFile : LeakChecked , Uncopyable { public: /** Creates a ZipFile based for a file. */ @@ -184,7 +184,7 @@ public: Currently this just stores the files with no compression.. That will be added soon! */ - class Builder + class Builder : LeakChecked , Uncopyable { public: Builder(); @@ -212,8 +212,6 @@ public: class Item; friend class OwnedArray; OwnedArray items; - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (Builder) }; private: @@ -242,8 +240,6 @@ private: #endif void init(); - - BEAST_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ZipFile) }; #endif // BEAST_ZIPFILE_BEASTHEADER diff --git a/Subtrees/hyperleveldb/.gitignore b/Subtrees/hyperleveldb/.gitignore new file mode 100644 index 0000000000..23454ca877 --- /dev/null +++ b/Subtrees/hyperleveldb/.gitignore @@ -0,0 +1,32 @@ +# wildcards +.deps +.dirstamp +*.la +.libs +*.lo +*.o +*_test +# specific files +aclocal.m4 +autom4te.cache/ +benchmark +config.guess +config.h +config.h.in +config.log +config.status +config.sub +configure +db_bench +depcomp +install-sh +leveldbutil +libhyperleveldb.pc +libtool +ltmain.sh +m4/ +Makefile +Makefile.in +Makefile.old +missing +stamp-h1 diff --git a/Subtrees/hyperleveldb/AUTHORS b/Subtrees/hyperleveldb/AUTHORS new file mode 100644 index 0000000000..6078f62e80 --- /dev/null +++ b/Subtrees/hyperleveldb/AUTHORS @@ -0,0 +1,11 @@ +# Names should be added to this file like so: +# Name or Organization + +Google Inc. + +# Initial version authors: +Jeffrey Dean +Sanjay Ghemawat + +# HyperLevelDB authors: +Robert Escriva diff --git a/Subtrees/hyperleveldb/LICENSE b/Subtrees/hyperleveldb/LICENSE new file mode 100644 index 0000000000..8e80208cd7 --- /dev/null +++ b/Subtrees/hyperleveldb/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Subtrees/hyperleveldb/Makefile.am b/Subtrees/hyperleveldb/Makefile.am new file mode 100644 index 0000000000..da8dc7e75c --- /dev/null +++ b/Subtrees/hyperleveldb/Makefile.am @@ -0,0 +1,249 @@ +## Copyright (c) 2013 +## All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted provided that the following conditions are met: +## +## * Redistributions of source code must retain the above copyright notice, +## this list of conditions and the following disclaimer. +## * Redistributions in binary form must reproduce the above copyright +## notice, this list of conditions and the following disclaimer in the +## documentation and/or other materials provided with the distribution. +## * Neither the name of nb nor the names of its contributors may be used to +## endorse or promote products derived from this software without specific +## prior written permission. +## +## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +## POSSIBILITY OF SUCH DAMAGE. + +ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS} +AM_CFLAGS = -DLEVELDB_PLATFORM_POSIX -fno-builtin-memcmp -fno-builtin-memmove +AM_CXXFLAGS = -DLEVELDB_PLATFORM_POSIX -fno-builtin-memcmp -fno-builtin-memmove +AM_MAKEFLAGS = --no-print-directory + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = libhyperleveldb.pc + +EXTRA_DIST = +EXTRA_DIST += AUTHORS +EXTRA_DIST += LICENSE +EXTRA_DIST += NEWS +EXTRA_DIST += README +EXTRA_DIST += TODO +EXTRA_DIST += port/README + +nobase_include_HEADERS = +nobase_include_HEADERS += hyperleveldb/cache.h +nobase_include_HEADERS += hyperleveldb/c.h +nobase_include_HEADERS += hyperleveldb/comparator.h +nobase_include_HEADERS += hyperleveldb/db.h +nobase_include_HEADERS += hyperleveldb/env.h +nobase_include_HEADERS += hyperleveldb/filter_policy.h +nobase_include_HEADERS += hyperleveldb/iterator.h +nobase_include_HEADERS += hyperleveldb/options.h +nobase_include_HEADERS += hyperleveldb/slice.h +nobase_include_HEADERS += hyperleveldb/status.h +nobase_include_HEADERS += hyperleveldb/table_builder.h +nobase_include_HEADERS += hyperleveldb/table.h +nobase_include_HEADERS += hyperleveldb/write_batch.h +noinst_HEADERS = +noinst_HEADERS += db/builder.h +noinst_HEADERS += db/dbformat.h +noinst_HEADERS += db/db_impl.h +noinst_HEADERS += db/db_iter.h +noinst_HEADERS += db/filename.h +noinst_HEADERS += db/log_format.h +noinst_HEADERS += db/log_reader.h +noinst_HEADERS += db/log_writer.h +noinst_HEADERS += db/memtable.h +noinst_HEADERS += db/skiplist.h +noinst_HEADERS += db/snapshot.h +noinst_HEADERS += db/table_cache.h +noinst_HEADERS += db/version_edit.h +noinst_HEADERS += db/version_set.h +noinst_HEADERS += db/write_batch_internal.h +noinst_HEADERS += helpers/memenv/memenv.h +noinst_HEADERS += port/atomic_pointer.h +noinst_HEADERS += port/port_example.h +noinst_HEADERS += port/port.h +noinst_HEADERS += port/port_posix.h +noinst_HEADERS += port/thread_annotations.h +noinst_HEADERS += port/win/stdint.h +noinst_HEADERS += table/block_builder.h +noinst_HEADERS += table/block.h +noinst_HEADERS += table/filter_block.h +noinst_HEADERS += table/format.h +noinst_HEADERS += table/iterator_wrapper.h +noinst_HEADERS += table/merger.h +noinst_HEADERS += table/two_level_iterator.h +noinst_HEADERS += util/arena.h +noinst_HEADERS += util/coding.h +noinst_HEADERS += util/crc32c.h +noinst_HEADERS += util/hash.h +noinst_HEADERS += util/histogram.h +noinst_HEADERS += util/logging.h +noinst_HEADERS += util/mutexlock.h +noinst_HEADERS += util/posix_logger.h +noinst_HEADERS += util/random.h +noinst_HEADERS += util/testharness.h +noinst_HEADERS += util/testutil.h + +lib_LTLIBRARIES = libhyperleveldb.la + +libhyperleveldb_la_SOURCES = +libhyperleveldb_la_SOURCES += db/builder.cc +libhyperleveldb_la_SOURCES += db/c.cc +libhyperleveldb_la_SOURCES += db/dbformat.cc +libhyperleveldb_la_SOURCES += db/db_impl.cc +libhyperleveldb_la_SOURCES += db/db_iter.cc +libhyperleveldb_la_SOURCES += db/filename.cc +libhyperleveldb_la_SOURCES += db/log_reader.cc +libhyperleveldb_la_SOURCES += db/log_writer.cc +libhyperleveldb_la_SOURCES += db/memtable.cc +libhyperleveldb_la_SOURCES += db/repair.cc +libhyperleveldb_la_SOURCES += db/table_cache.cc +libhyperleveldb_la_SOURCES += db/version_edit.cc +libhyperleveldb_la_SOURCES += db/version_set.cc +libhyperleveldb_la_SOURCES += db/write_batch.cc +libhyperleveldb_la_SOURCES += table/block_builder.cc +libhyperleveldb_la_SOURCES += table/block.cc +libhyperleveldb_la_SOURCES += table/filter_block.cc +libhyperleveldb_la_SOURCES += table/format.cc +libhyperleveldb_la_SOURCES += table/iterator.cc +libhyperleveldb_la_SOURCES += table/merger.cc +libhyperleveldb_la_SOURCES += table/table_builder.cc +libhyperleveldb_la_SOURCES += table/table.cc +libhyperleveldb_la_SOURCES += table/two_level_iterator.cc +libhyperleveldb_la_SOURCES += util/arena.cc +libhyperleveldb_la_SOURCES += util/bloom.cc +libhyperleveldb_la_SOURCES += util/cache.cc +libhyperleveldb_la_SOURCES += util/coding.cc +libhyperleveldb_la_SOURCES += util/comparator.cc +libhyperleveldb_la_SOURCES += util/crc32c.cc +libhyperleveldb_la_SOURCES += util/env.cc +libhyperleveldb_la_SOURCES += util/env_posix.cc +libhyperleveldb_la_SOURCES += util/filter_policy.cc +libhyperleveldb_la_SOURCES += util/hash.cc +libhyperleveldb_la_SOURCES += util/histogram.cc +libhyperleveldb_la_SOURCES += util/logging.cc +libhyperleveldb_la_SOURCES += util/options.cc +libhyperleveldb_la_SOURCES += util/status.cc +libhyperleveldb_la_SOURCES += port/port_posix.cc +libhyperleveldb_la_LDFLAGS = -pthread + +TESTUTIL = util/testutil.cc +TESTHARNESS = util/testharness.cc $(TESTUTIL) + +noinst_PROGRAMS = +noinst_PROGRAMS += db_bench +noinst_PROGRAMS += leveldbutil + +EXTRA_PROGRAMS = +EXTRA_PROGRAMS += benchmark +EXTRA_PROGRAMS += db_bench_sqlite3 +EXTRA_PROGRAMS += db_bench_tree_db + +check_PROGRAMS = +check_PROGRAMS += arena_test +check_PROGRAMS += bloom_test +check_PROGRAMS += c_test +check_PROGRAMS += cache_test +check_PROGRAMS += coding_test +check_PROGRAMS += corruption_test +check_PROGRAMS += crc32c_test +check_PROGRAMS += db_test +check_PROGRAMS += dbformat_test +check_PROGRAMS += env_test +check_PROGRAMS += filename_test +check_PROGRAMS += filter_block_test +check_PROGRAMS += log_test +check_PROGRAMS += skiplist_test +check_PROGRAMS += table_test +check_PROGRAMS += version_edit_test +check_PROGRAMS += version_set_test +check_PROGRAMS += write_batch_test +check_PROGRAMS += issue178_test + +TESTS = $(check_PROGRAMS) + +benchmark_SOURCES = benchmark.cc +benchmark_LDADD = libhyperleveldb.la -lpthread -le -lpopt -larmnod -lnumbers +benchmark_LDFLAGS = -no-install + +db_bench_SOURCES = db/db_bench.cc $(TESTUTIL) +db_bench_LDADD = libhyperleveldb.la -lpthread + +db_bench_sqlite3_SOURCES = doc/bench/db_bench_sqlite3.cc $(TESTUTIL) +db_bench_sqlite3_LDADD = -lsqlite3 + +db_bench_tree_db_SOURCES = doc/bench/db_bench_tree_db.cc $(TESTUTIL) +db_bench_tree_db_LDADD = -lkyotocabinet + +leveldbutil_SOURCES = db/leveldb_main.cc +leveldbutil_LDADD = libhyperleveldb.la -lpthread + +arena_test_SOURCES = util/arena_test.cc $(TESTHARNESS) +arena_test_LDADD = libhyperleveldb.la -lpthread + +bloom_test_SOURCES = util/bloom_test.cc $(TESTHARNESS) +bloom_test_LDADD = libhyperleveldb.la -lpthread + +c_test_SOURCES = db/c_test.c $(TESTHARNESS) +c_test_LDADD = libhyperleveldb.la -lpthread + +cache_test_SOURCES = util/cache_test.cc $(TESTHARNESS) +cache_test_LDADD = libhyperleveldb.la -lpthread + +coding_test_SOURCES = util/coding_test.cc $(TESTHARNESS) +coding_test_LDADD = libhyperleveldb.la -lpthread + +corruption_test_SOURCES = db/corruption_test.cc $(TESTHARNESS) +corruption_test_LDADD = libhyperleveldb.la -lpthread + +crc32c_test_SOURCES = util/crc32c_test.cc $(TESTHARNESS) +crc32c_test_LDADD = libhyperleveldb.la -lpthread + +db_test_SOURCES = db/db_test.cc $(TESTHARNESS) +db_test_LDADD = libhyperleveldb.la -lpthread + +dbformat_test_SOURCES = db/dbformat_test.cc $(TESTHARNESS) +dbformat_test_LDADD = libhyperleveldb.la -lpthread + +env_test_SOURCES = util/env_test.cc $(TESTHARNESS) +env_test_LDADD = libhyperleveldb.la -lpthread + +filename_test_SOURCES = db/filename_test.cc $(TESTHARNESS) +filename_test_LDADD = libhyperleveldb.la -lpthread + +filter_block_test_SOURCES = table/filter_block_test.cc $(TESTHARNESS) +filter_block_test_LDADD = libhyperleveldb.la -lpthread + +log_test_SOURCES = db/log_test.cc $(TESTHARNESS) +log_test_LDADD = libhyperleveldb.la -lpthread + +table_test_SOURCES = table/table_test.cc $(TESTHARNESS) +table_test_LDADD = libhyperleveldb.la -lpthread + +skiplist_test_SOURCES = db/skiplist_test.cc $(TESTHARNESS) +skiplist_test_LDADD = libhyperleveldb.la -lpthread + +version_edit_test_SOURCES = db/version_edit_test.cc $(TESTHARNESS) +version_edit_test_LDADD = libhyperleveldb.la -lpthread + +version_set_test_SOURCES = db/version_set_test.cc $(TESTHARNESS) +version_set_test_LDADD = libhyperleveldb.la -lpthread + +write_batch_test_SOURCES = db/write_batch_test.cc $(TESTHARNESS) +write_batch_test_LDADD = libhyperleveldb.la -lpthread + +issue178_test_SOURCES = issues/issue178_test.cc $(TESTHARNESS) +issue178_test_LDADD = libhyperleveldb.la -lpthread diff --git a/Subtrees/hyperleveldb/NEWS b/Subtrees/hyperleveldb/NEWS new file mode 100644 index 0000000000..3fd99242d7 --- /dev/null +++ b/Subtrees/hyperleveldb/NEWS @@ -0,0 +1,17 @@ +Release 1.2 2011-05-16 +---------------------- + +Fixes for larger databases (tested up to one billion 100-byte entries, +i.e., ~100GB). + +(1) Place hard limit on number of level-0 files. This fixes errors +of the form "too many open files". + +(2) Fixed memtable management. Before the fix, a heavy write burst +could cause unbounded memory usage. + +A fix for a logging bug where the reader would incorrectly complain +about corruption. + +Allow public access to WriteBatch contents so that users can easily +wrap a DB. diff --git a/Subtrees/hyperleveldb/README b/Subtrees/hyperleveldb/README new file mode 100644 index 0000000000..3618adeeed --- /dev/null +++ b/Subtrees/hyperleveldb/README @@ -0,0 +1,51 @@ +leveldb: A key-value store +Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) + +The code under this directory implements a system for maintaining a +persistent key/value store. + +See doc/index.html for more explanation. +See doc/impl.html for a brief overview of the implementation. + +The public interface is in include/*.h. Callers should not include or +rely on the details of any other header files in this package. Those +internal APIs may be changed without warning. + +Guide to header files: + +include/db.h + Main interface to the DB: Start here + +include/options.h + Control over the behavior of an entire database, and also + control over the behavior of individual reads and writes. + +include/comparator.h + Abstraction for user-specified comparison function. If you want + just bytewise comparison of keys, you can use the default comparator, + but clients can write their own comparator implementations if they + want custom ordering (e.g. to handle different character + encodings, etc.) + +include/iterator.h + Interface for iterating over data. You can get an iterator + from a DB object. + +include/write_batch.h + Interface for atomically applying multiple updates to a database. + +include/slice.h + A simple module for maintaining a pointer and a length into some + other byte array. + +include/status.h + Status is returned from many of the public interfaces and is used + to report success and various kinds of errors. + +include/env.h + Abstraction of the OS environment. A posix implementation of + this interface is in util/env_posix.cc + +include/table.h +include/table_builder.h + Lower-level modules that most clients probably won't use directly diff --git a/Subtrees/hyperleveldb/TODO b/Subtrees/hyperleveldb/TODO new file mode 100644 index 0000000000..e603c07137 --- /dev/null +++ b/Subtrees/hyperleveldb/TODO @@ -0,0 +1,14 @@ +ss +- Stats + +db +- Maybe implement DB::BulkDeleteForRange(start_key, end_key) + that would blow away files whose ranges are entirely contained + within [start_key..end_key]? For Chrome, deletion of obsolete + object stores, etc. can be done in the background anyway, so + probably not that important. +- There have been requests for MultiGet. + +After a range is completely deleted, what gets rid of the +corresponding files if we do no future changes to that range. Make +the conditions for triggering compactions fire in more situations? diff --git a/Subtrees/hyperleveldb/benchmark.cc b/Subtrees/hyperleveldb/benchmark.cc new file mode 100644 index 0000000000..656371ff70 --- /dev/null +++ b/Subtrees/hyperleveldb/benchmark.cc @@ -0,0 +1,212 @@ +// Copyright (c) 2013, Cornell University +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of HyperLevelDB nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +// C +#include + +// STL +#include +#include + +// LevelDB +#include +#include +#include + +// po6 +#include +#include + +// e +#include + +// armnod +#include + +// numbers +#include + +static long _done = 0; +static long _number = 1000000; +static long _threads = 1; + +static struct poptOption _popts[] = { + {"number", 'n', POPT_ARG_LONG, &_number, 0, + "perform N operations against the database (default: 1000000)", "N"}, + {"threads", 't', POPT_ARG_LONG, &_threads, 0, + "run the test with T concurrent threads (default: 1)", "T"}, + POPT_TABLEEND +}; + +static void +worker_thread(leveldb::DB*, + numbers::throughput_latency_logger* tll, + const armnod::argparser& k, + const armnod::argparser& v); + +int +main(int argc, const char* argv[]) +{ + armnod::argparser key_parser("key-"); + armnod::argparser value_parser("value-"); + std::vector popts; + poptOption s[] = {POPT_AUTOHELP {NULL, 0, POPT_ARG_INCLUDE_TABLE, _popts, 0, "Benchmark:", NULL}, POPT_TABLEEND}; + popts.push_back(s[0]); + popts.push_back(s[1]); + popts.push_back(key_parser.options("Key Generation:")); + popts.push_back(value_parser.options("Value Generation:")); + popts.push_back(s[2]); + poptContext poptcon; + poptcon = poptGetContext(NULL, argc, argv, &popts.front(), POPT_CONTEXT_POSIXMEHARDER); + e::guard g = e::makeguard(poptFreeContext, poptcon); g.use_variable(); + poptSetOtherOptionHelp(poptcon, "[OPTIONS]"); + + int rc; + + while ((rc = poptGetNextOpt(poptcon)) != -1) + { + switch (rc) + { + case POPT_ERROR_NOARG: + case POPT_ERROR_BADOPT: + case POPT_ERROR_BADNUMBER: + case POPT_ERROR_OVERFLOW: + std::cerr << poptStrerror(rc) << " " << poptBadOption(poptcon, 0) << std::endl; + return EXIT_FAILURE; + case POPT_ERROR_OPTSTOODEEP: + case POPT_ERROR_BADQUOTE: + case POPT_ERROR_ERRNO: + default: + std::cerr << "logic error in argument parsing" << std::endl; + return EXIT_FAILURE; + } + } + + leveldb::Options opts; + opts.create_if_missing = true; + opts.write_buffer_size = 64ULL * 1024ULL * 1024ULL; + opts.filter_policy = leveldb::NewBloomFilterPolicy(10); + leveldb::DB* db; + leveldb::Status st = leveldb::DB::Open(opts, "tmp", &db); + + if (!st.ok()) + { + std::cerr << "could not open LevelDB: " << st.ToString() << std::endl; + return EXIT_FAILURE; + } + + numbers::throughput_latency_logger tll; + + if (!tll.open("benchmark.log")) + { + std::cerr << "could not open log: " << strerror(errno) << std::endl; + return EXIT_FAILURE; + } + + typedef std::tr1::shared_ptr thread_ptr; + std::vector threads; + + for (size_t i = 0; i < _threads; ++i) + { + thread_ptr t(new po6::threads::thread(std::tr1::bind(worker_thread, db, &tll, key_parser, value_parser))); + threads.push_back(t); + t->start(); + } + + for (size_t i = 0; i < threads.size(); ++i) + { + threads[i]->join(); + } + + std::string tmp; + if (db->GetProperty("leveldb.stats", &tmp)) std::cout << tmp << std::endl; + delete db; + + if (!tll.close()) + { + std::cerr << "could not close log: " << strerror(errno) << std::endl; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +static uint64_t +get_random() +{ + po6::io::fd sysrand(open("/dev/urandom", O_RDONLY)); + + if (sysrand.get() < 0) + { + return 0xcafebabe; + } + + uint64_t ret; + + if (sysrand.read(&ret, sizeof(ret)) != sizeof(ret)) + { + return 0xdeadbeef; + } + + return ret; +} + +void +worker_thread(leveldb::DB* db, + numbers::throughput_latency_logger* tll, + const armnod::argparser& _k, + const armnod::argparser& _v) +{ + armnod::generator key(armnod::argparser(_k).config()); + armnod::generator val(armnod::argparser(_v).config()); + key.seed(get_random()); + val.seed(get_random()); + numbers::throughput_latency_logger::thread_state ts; + tll->initialize_thread(&ts); + + while (__sync_fetch_and_add(&_done, 1) < _number) + { + std::string k = key(); + std::string v = val(); + + // issue a "get" + std::string tmp; + leveldb::ReadOptions ropts; + leveldb::Status rst = db->Get(ropts, leveldb::Slice(k.data(), k.size()), &tmp); + assert(rst.ok() || rst.IsNotFound()); + + // issue a "put" + leveldb::WriteOptions wopts; + wopts.sync = false; + tll->start(&ts, 0); + leveldb::Status wst = db->Put(wopts, leveldb::Slice(k.data(), k.size()), leveldb::Slice(v.data(), v.size())); + tll->finish(&ts); + assert(wst.ok()); + } + + tll->terminate_thread(&ts); +} diff --git a/Subtrees/hyperleveldb/configure.ac b/Subtrees/hyperleveldb/configure.ac new file mode 100644 index 0000000000..744538a8d9 --- /dev/null +++ b/Subtrees/hyperleveldb/configure.ac @@ -0,0 +1,63 @@ +# Copyright (c) 2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of HyperLevelDB nor the names of its contributors may +# be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# -*- Autoconf -*- +# +# Process this file with autoconf to produce a configure script. + +AC_PREREQ([2.62]) +AC_INIT([hyperleveldb], [2.0.dev], [robert@hyperdex.org]) +AM_INIT_AUTOMAKE([foreign subdir-objects dist-bzip2]) +m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) +LT_PREREQ([2.2]) +LT_INIT +AC_CONFIG_SRCDIR([hyperleveldb/db.h]) +AC_CONFIG_HEADERS([config.h]) +AC_CONFIG_MACRO_DIR([m4]) + +# Checks for programs. +AC_PROG_CXX +AC_PROG_CC +AC_LANG(C++) + +ANAL_WARNINGS_CXX + +# Checks for libraries. + +# Checks for header files. + +# Checks for typedefs, structures, and compiler characteristics. + +# Checks for library functions. +AC_FUNC_ERROR_AT_LINE +AC_FUNC_MMAP +AC_CHECK_FUNCS([alarm clock_gettime mach_absolute_time ftruncate memmove mkdir munmap rmdir socket]) + +# Optional components + +AC_CONFIG_FILES([Makefile libhyperleveldb.pc]) +AC_OUTPUT diff --git a/Subtrees/hyperleveldb/db/builder.cc b/Subtrees/hyperleveldb/db/builder.cc new file mode 100644 index 0000000000..eb6db54e72 --- /dev/null +++ b/Subtrees/hyperleveldb/db/builder.cc @@ -0,0 +1,88 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "builder.h" + +#include "filename.h" +#include "dbformat.h" +#include "table_cache.h" +#include "version_edit.h" +#include "../hyperleveldb/db.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/iterator.h" + +namespace hyperleveldb { + +Status BuildTable(const std::string& dbname, + Env* env, + const Options& options, + TableCache* table_cache, + Iterator* iter, + FileMetaData* meta) { + Status s; + meta->file_size = 0; + iter->SeekToFirst(); + + std::string fname = TableFileName(dbname, meta->number); + if (iter->Valid()) { + WritableFile* file; + s = env->NewWritableFile(fname, &file); + if (!s.ok()) { + return s; + } + + TableBuilder* builder = new TableBuilder(options, file); + meta->smallest.DecodeFrom(iter->key()); + for (; iter->Valid(); iter->Next()) { + Slice key = iter->key(); + meta->largest.DecodeFrom(key); + builder->Add(key, iter->value()); + } + + // Finish and check for builder errors + if (s.ok()) { + s = builder->Finish(); + if (s.ok()) { + meta->file_size = builder->FileSize(); + assert(meta->file_size > 0); + } + } else { + builder->Abandon(); + } + delete builder; + + // Finish and check for file errors + if (s.ok()) { + s = file->Sync(); + } + if (s.ok()) { + s = file->Close(); + } + delete file; + file = NULL; + + if (s.ok()) { + // Verify that the table is usable + Iterator* it = table_cache->NewIterator(ReadOptions(), + meta->number, + meta->file_size); + s = it->status(); + delete it; + } + } + + // Check for input iterator errors + if (!iter->status().ok()) { + s = iter->status(); + } + + if (s.ok() && meta->file_size > 0) { + // Keep it + } else { + env->DeleteFile(fname); + } + return s; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/builder.h b/Subtrees/hyperleveldb/db/builder.h new file mode 100644 index 0000000000..545e25a719 --- /dev/null +++ b/Subtrees/hyperleveldb/db/builder.h @@ -0,0 +1,34 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_BUILDER_H_ +#define STORAGE_HYPERLEVELDB_DB_BUILDER_H_ + +#include "../hyperleveldb/status.h" + +namespace hyperleveldb { + +struct Options; +struct FileMetaData; + +class Env; +class Iterator; +class TableCache; +class VersionEdit; + +// Build a Table file from the contents of *iter. The generated file +// will be named according to meta->number. On success, the rest of +// *meta will be filled with metadata about the generated table. +// If no data is present in *iter, meta->file_size will be set to +// zero, and no Table file will be produced. +extern Status BuildTable(const std::string& dbname, + Env* env, + const Options& options, + TableCache* table_cache, + Iterator* iter, + FileMetaData* meta); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_BUILDER_H_ diff --git a/Subtrees/hyperleveldb/db/c.cc b/Subtrees/hyperleveldb/db/c.cc new file mode 100644 index 0000000000..ff8c736bb1 --- /dev/null +++ b/Subtrees/hyperleveldb/db/c.cc @@ -0,0 +1,595 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "c.h" + +#include +#include +#include "../hyperleveldb/cache.h" +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/db.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/filter_policy.h" +#include "../hyperleveldb/iterator.h" +#include "../hyperleveldb/options.h" +#include "../hyperleveldb/status.h" +#include "../hyperleveldb/write_batch.h" + +using leveldb::Cache; +using leveldb::Comparator; +using leveldb::CompressionType; +using leveldb::DB; +using leveldb::Env; +using leveldb::FileLock; +using leveldb::FilterPolicy; +using leveldb::Iterator; +using leveldb::kMajorVersion; +using leveldb::kMinorVersion; +using leveldb::Logger; +using leveldb::NewBloomFilterPolicy; +using leveldb::NewLRUCache; +using leveldb::Options; +using leveldb::RandomAccessFile; +using leveldb::Range; +using leveldb::ReadOptions; +using leveldb::SequentialFile; +using leveldb::Slice; +using leveldb::Snapshot; +using leveldb::Status; +using leveldb::WritableFile; +using leveldb::WriteBatch; +using leveldb::WriteOptions; + +extern "C" { + +struct leveldb_t { DB* rep; }; +struct leveldb_iterator_t { Iterator* rep; }; +struct leveldb_writebatch_t { WriteBatch rep; }; +struct leveldb_snapshot_t { const Snapshot* rep; }; +struct leveldb_readoptions_t { ReadOptions rep; }; +struct leveldb_writeoptions_t { WriteOptions rep; }; +struct leveldb_options_t { Options rep; }; +struct leveldb_cache_t { Cache* rep; }; +struct leveldb_seqfile_t { SequentialFile* rep; }; +struct leveldb_randomfile_t { RandomAccessFile* rep; }; +struct leveldb_writablefile_t { WritableFile* rep; }; +struct leveldb_logger_t { Logger* rep; }; +struct leveldb_filelock_t { FileLock* rep; }; + +struct leveldb_comparator_t : public Comparator { + void* state_; + void (*destructor_)(void*); + int (*compare_)( + void*, + const char* a, size_t alen, + const char* b, size_t blen); + const char* (*name_)(void*); + + virtual ~leveldb_comparator_t() { + (*destructor_)(state_); + } + + virtual int Compare(const Slice& a, const Slice& b) const { + return (*compare_)(state_, a.data(), a.size(), b.data(), b.size()); + } + + virtual const char* Name() const { + return (*name_)(state_); + } + + // No-ops since the C binding does not support key shortening methods. + virtual void FindShortestSeparator(std::string*, const Slice&) const { } + virtual void FindShortSuccessor(std::string* key) const { } +}; + +struct leveldb_filterpolicy_t : public FilterPolicy { + void* state_; + void (*destructor_)(void*); + const char* (*name_)(void*); + char* (*create_)( + void*, + const char* const* key_array, const size_t* key_length_array, + int num_keys, + size_t* filter_length); + unsigned char (*key_match_)( + void*, + const char* key, size_t length, + const char* filter, size_t filter_length); + + virtual ~leveldb_filterpolicy_t() { + (*destructor_)(state_); + } + + virtual const char* Name() const { + return (*name_)(state_); + } + + virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { + std::vector key_pointers(n); + std::vector key_sizes(n); + for (int i = 0; i < n; i++) { + key_pointers[i] = keys[i].data(); + key_sizes[i] = keys[i].size(); + } + size_t len; + char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len); + dst->append(filter, len); + free(filter); + } + + virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { + return (*key_match_)(state_, key.data(), key.size(), + filter.data(), filter.size()); + } +}; + +struct leveldb_env_t { + Env* rep; + bool is_default; +}; + +static bool SaveError(char** errptr, const Status& s) { + assert(errptr != NULL); + if (s.ok()) { + return false; + } else if (*errptr == NULL) { + *errptr = strdup(s.ToString().c_str()); + } else { + // TODO(sanjay): Merge with existing error? + free(*errptr); + *errptr = strdup(s.ToString().c_str()); + } + return true; +} + +static char* CopyString(const std::string& str) { + char* result = reinterpret_cast(malloc(sizeof(char) * str.size())); + memcpy(result, str.data(), sizeof(char) * str.size()); + return result; +} + +leveldb_t* leveldb_open( + const leveldb_options_t* options, + const char* name, + char** errptr) { + DB* db; + if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) { + return NULL; + } + leveldb_t* result = new leveldb_t; + result->rep = db; + return result; +} + +void leveldb_close(leveldb_t* db) { + delete db->rep; + delete db; +} + +void leveldb_put( + leveldb_t* db, + const leveldb_writeoptions_t* options, + const char* key, size_t keylen, + const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, + db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen))); +} + +void leveldb_delete( + leveldb_t* db, + const leveldb_writeoptions_t* options, + const char* key, size_t keylen, + char** errptr) { + SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen))); +} + + +void leveldb_write( + leveldb_t* db, + const leveldb_writeoptions_t* options, + leveldb_writebatch_t* batch, + char** errptr) { + SaveError(errptr, db->rep->Write(options->rep, &batch->rep)); +} + +char* leveldb_get( + leveldb_t* db, + const leveldb_readoptions_t* options, + const char* key, size_t keylen, + size_t* vallen, + char** errptr) { + char* result = NULL; + std::string tmp; + Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + } else { + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +leveldb_iterator_t* leveldb_create_iterator( + leveldb_t* db, + const leveldb_readoptions_t* options) { + leveldb_iterator_t* result = new leveldb_iterator_t; + result->rep = db->rep->NewIterator(options->rep); + return result; +} + +const leveldb_snapshot_t* leveldb_create_snapshot( + leveldb_t* db) { + leveldb_snapshot_t* result = new leveldb_snapshot_t; + result->rep = db->rep->GetSnapshot(); + return result; +} + +void leveldb_release_snapshot( + leveldb_t* db, + const leveldb_snapshot_t* snapshot) { + db->rep->ReleaseSnapshot(snapshot->rep); + delete snapshot; +} + +char* leveldb_property_value( + leveldb_t* db, + const char* propname) { + std::string tmp; + if (db->rep->GetProperty(Slice(propname), &tmp)) { + // We use strdup() since we expect human readable output. + return strdup(tmp.c_str()); + } else { + return NULL; + } +} + +void leveldb_approximate_sizes( + leveldb_t* db, + int num_ranges, + const char* const* range_start_key, const size_t* range_start_key_len, + const char* const* range_limit_key, const size_t* range_limit_key_len, + uint64_t* sizes) { + Range* ranges = new Range[num_ranges]; + for (int i = 0; i < num_ranges; i++) { + ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]); + ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]); + } + db->rep->GetApproximateSizes(ranges, num_ranges, sizes); + delete[] ranges; +} + +void leveldb_compact_range( + leveldb_t* db, + const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len) { + Slice a, b; + db->rep->CompactRange( + // Pass NULL Slice if corresponding "const char*" is NULL + (start_key ? (a = Slice(start_key, start_key_len), &a) : NULL), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : NULL)); +} + +void leveldb_destroy_db( + const leveldb_options_t* options, + const char* name, + char** errptr) { + SaveError(errptr, DestroyDB(name, options->rep)); +} + +void leveldb_repair_db( + const leveldb_options_t* options, + const char* name, + char** errptr) { + SaveError(errptr, RepairDB(name, options->rep)); +} + +void leveldb_iter_destroy(leveldb_iterator_t* iter) { + delete iter->rep; + delete iter; +} + +unsigned char leveldb_iter_valid(const leveldb_iterator_t* iter) { + return iter->rep->Valid(); +} + +void leveldb_iter_seek_to_first(leveldb_iterator_t* iter) { + iter->rep->SeekToFirst(); +} + +void leveldb_iter_seek_to_last(leveldb_iterator_t* iter) { + iter->rep->SeekToLast(); +} + +void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) { + iter->rep->Seek(Slice(k, klen)); +} + +void leveldb_iter_next(leveldb_iterator_t* iter) { + iter->rep->Next(); +} + +void leveldb_iter_prev(leveldb_iterator_t* iter) { + iter->rep->Prev(); +} + +const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) { + Slice s = iter->rep->key(); + *klen = s.size(); + return s.data(); +} + +const char* leveldb_iter_value(const leveldb_iterator_t* iter, size_t* vlen) { + Slice s = iter->rep->value(); + *vlen = s.size(); + return s.data(); +} + +void leveldb_iter_get_error(const leveldb_iterator_t* iter, char** errptr) { + SaveError(errptr, iter->rep->status()); +} + +leveldb_writebatch_t* leveldb_writebatch_create() { + return new leveldb_writebatch_t; +} + +void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { + delete b; +} + +void leveldb_writebatch_clear(leveldb_writebatch_t* b) { + b->rep.Clear(); +} + +void leveldb_writebatch_put( + leveldb_writebatch_t* b, + const char* key, size_t klen, + const char* val, size_t vlen) { + b->rep.Put(Slice(key, klen), Slice(val, vlen)); +} + +void leveldb_writebatch_delete( + leveldb_writebatch_t* b, + const char* key, size_t klen) { + b->rep.Delete(Slice(key, klen)); +} + +void leveldb_writebatch_iterate( + leveldb_writebatch_t* b, + void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)) { + class H : public WriteBatch::Handler { + public: + void* state_; + void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen); + void (*deleted_)(void*, const char* k, size_t klen); + virtual void Put(const Slice& key, const Slice& value) { + (*put_)(state_, key.data(), key.size(), value.data(), value.size()); + } + virtual void Delete(const Slice& key) { + (*deleted_)(state_, key.data(), key.size()); + } + }; + H handler; + handler.state_ = state; + handler.put_ = put; + handler.deleted_ = deleted; + b->rep.Iterate(&handler); +} + +leveldb_options_t* leveldb_options_create() { + return new leveldb_options_t; +} + +void leveldb_options_destroy(leveldb_options_t* options) { + delete options; +} + +void leveldb_options_set_comparator( + leveldb_options_t* opt, + leveldb_comparator_t* cmp) { + opt->rep.comparator = cmp; +} + +void leveldb_options_set_filter_policy( + leveldb_options_t* opt, + leveldb_filterpolicy_t* policy) { + opt->rep.filter_policy = policy; +} + +void leveldb_options_set_create_if_missing( + leveldb_options_t* opt, unsigned char v) { + opt->rep.create_if_missing = v; +} + +void leveldb_options_set_error_if_exists( + leveldb_options_t* opt, unsigned char v) { + opt->rep.error_if_exists = v; +} + +void leveldb_options_set_paranoid_checks( + leveldb_options_t* opt, unsigned char v) { + opt->rep.paranoid_checks = v; +} + +void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) { + opt->rep.env = (env ? env->rep : NULL); +} + +void leveldb_options_set_info_log(leveldb_options_t* opt, leveldb_logger_t* l) { + opt->rep.info_log = (l ? l->rep : NULL); +} + +void leveldb_options_set_write_buffer_size(leveldb_options_t* opt, size_t s) { + opt->rep.write_buffer_size = s; +} + +void leveldb_options_set_max_open_files(leveldb_options_t* opt, int n) { + opt->rep.max_open_files = n; +} + +void leveldb_options_set_cache(leveldb_options_t* opt, leveldb_cache_t* c) { + opt->rep.block_cache = c->rep; +} + +void leveldb_options_set_block_size(leveldb_options_t* opt, size_t s) { + opt->rep.block_size = s; +} + +void leveldb_options_set_block_restart_interval(leveldb_options_t* opt, int n) { + opt->rep.block_restart_interval = n; +} + +void leveldb_options_set_compression(leveldb_options_t* opt, int t) { + opt->rep.compression = static_cast(t); +} + +leveldb_comparator_t* leveldb_comparator_create( + void* state, + void (*destructor)(void*), + int (*compare)( + void*, + const char* a, size_t alen, + const char* b, size_t blen), + const char* (*name)(void*)) { + leveldb_comparator_t* result = new leveldb_comparator_t; + result->state_ = state; + result->destructor_ = destructor; + result->compare_ = compare; + result->name_ = name; + return result; +} + +void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { + delete cmp; +} + +leveldb_filterpolicy_t* leveldb_filterpolicy_create( + void* state, + void (*destructor)(void*), + char* (*create_filter)( + void*, + const char* const* key_array, const size_t* key_length_array, + int num_keys, + size_t* filter_length), + unsigned char (*key_may_match)( + void*, + const char* key, size_t length, + const char* filter, size_t filter_length), + const char* (*name)(void*)) { + leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t; + result->state_ = state; + result->destructor_ = destructor; + result->create_ = create_filter; + result->key_match_ = key_may_match; + result->name_ = name; + return result; +} + +void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t* filter) { + delete filter; +} + +leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) { + // Make a leveldb_filterpolicy_t, but override all of its methods so + // they delegate to a NewBloomFilterPolicy() instead of user + // supplied C functions. + struct Wrapper : public leveldb_filterpolicy_t { + const FilterPolicy* rep_; + ~Wrapper() { delete rep_; } + const char* Name() const { return rep_->Name(); } + void CreateFilter(const Slice* keys, int n, std::string* dst) const { + return rep_->CreateFilter(keys, n, dst); + } + bool KeyMayMatch(const Slice& key, const Slice& filter) const { + return rep_->KeyMayMatch(key, filter); + } + static void DoNothing(void*) { } + }; + Wrapper* wrapper = new Wrapper; + wrapper->rep_ = NewBloomFilterPolicy(bits_per_key); + wrapper->state_ = NULL; + wrapper->destructor_ = &Wrapper::DoNothing; + return wrapper; +} + +leveldb_readoptions_t* leveldb_readoptions_create() { + return new leveldb_readoptions_t; +} + +void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { + delete opt; +} + +void leveldb_readoptions_set_verify_checksums( + leveldb_readoptions_t* opt, + unsigned char v) { + opt->rep.verify_checksums = v; +} + +void leveldb_readoptions_set_fill_cache( + leveldb_readoptions_t* opt, unsigned char v) { + opt->rep.fill_cache = v; +} + +void leveldb_readoptions_set_snapshot( + leveldb_readoptions_t* opt, + const leveldb_snapshot_t* snap) { + opt->rep.snapshot = (snap ? snap->rep : NULL); +} + +leveldb_writeoptions_t* leveldb_writeoptions_create() { + return new leveldb_writeoptions_t; +} + +void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { + delete opt; +} + +void leveldb_writeoptions_set_sync( + leveldb_writeoptions_t* opt, unsigned char v) { + opt->rep.sync = v; +} + +leveldb_cache_t* leveldb_cache_create_lru(size_t capacity) { + leveldb_cache_t* c = new leveldb_cache_t; + c->rep = NewLRUCache(capacity); + return c; +} + +void leveldb_cache_destroy(leveldb_cache_t* cache) { + delete cache->rep; + delete cache; +} + +leveldb_env_t* leveldb_create_default_env() { + leveldb_env_t* result = new leveldb_env_t; + result->rep = Env::Default(); + result->is_default = true; + return result; +} + +void leveldb_env_destroy(leveldb_env_t* env) { + if (!env->is_default) delete env->rep; + delete env; +} + +void leveldb_free(void* ptr) { + free(ptr); +} + +int leveldb_major_version() { + return kMajorVersion; +} + +int leveldb_minor_version() { + return kMinorVersion; +} + +} // end extern "C" diff --git a/Subtrees/hyperleveldb/db/c_test.c b/Subtrees/hyperleveldb/db/c_test.c new file mode 100644 index 0000000000..d75f25f864 --- /dev/null +++ b/Subtrees/hyperleveldb/db/c_test.c @@ -0,0 +1,390 @@ +/* Copyright (c) 2011 The LevelDB Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. See the AUTHORS file for names of contributors. */ + +#include "c.h" + +#include +#include +#include +#include +#include +#include + +const char* phase = ""; +static char dbname[200]; + +static void StartPhase(const char* name) { + fprintf(stderr, "=== Test %s\n", name); + phase = name; +} + +static const char* GetTempDir(void) { + const char* ret = getenv("TEST_TMPDIR"); + if (ret == NULL || ret[0] == '\0') + ret = "/tmp"; + return ret; +} + +#define CheckNoError(err) \ + if ((err) != NULL) { \ + fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \ + abort(); \ + } + +#define CheckCondition(cond) \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \ + abort(); \ + } + +static void CheckEqual(const char* expected, const char* v, size_t n) { + if (expected == NULL && v == NULL) { + // ok + } else if (expected != NULL && v != NULL && n == strlen(expected) && + memcmp(expected, v, n) == 0) { + // ok + return; + } else { + fprintf(stderr, "%s: expected '%s', got '%s'\n", + phase, + (expected ? expected : "(null)"), + (v ? v : "(null")); + abort(); + } +} + +static void Free(char** ptr) { + if (*ptr) { + free(*ptr); + *ptr = NULL; + } +} + +static void CheckGet( + leveldb_t* db, + const leveldb_readoptions_t* options, + const char* key, + const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = leveldb_get(db, options, key, strlen(key), &val_len, &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckIter(leveldb_iterator_t* iter, + const char* key, const char* val) { + size_t len; + const char* str; + str = leveldb_iter_key(iter, &len); + CheckEqual(key, str, len); + str = leveldb_iter_value(iter, &len); + CheckEqual(val, str, len); +} + +// Callback from leveldb_writebatch_iterate() +static void CheckPut(void* ptr, + const char* k, size_t klen, + const char* v, size_t vlen) { + int* state = (int*) ptr; + CheckCondition(*state < 2); + switch (*state) { + case 0: + CheckEqual("bar", k, klen); + CheckEqual("b", v, vlen); + break; + case 1: + CheckEqual("box", k, klen); + CheckEqual("c", v, vlen); + break; + } + (*state)++; +} + +// Callback from leveldb_writebatch_iterate() +static void CheckDel(void* ptr, const char* k, size_t klen) { + int* state = (int*) ptr; + CheckCondition(*state == 2); + CheckEqual("bar", k, klen); + (*state)++; +} + +static void CmpDestroy(void* arg) { } + +static int CmpCompare(void* arg, const char* a, size_t alen, + const char* b, size_t blen) { + int n = (alen < blen) ? alen : blen; + int r = memcmp(a, b, n); + if (r == 0) { + if (alen < blen) r = -1; + else if (alen > blen) r = +1; + } + return r; +} + +static const char* CmpName(void* arg) { + return "foo"; +} + +// Custom filter policy +static unsigned char fake_filter_result = 1; +static void FilterDestroy(void* arg) { } +static const char* FilterName(void* arg) { + return "TestFilter"; +} +static char* FilterCreate( + void* arg, + const char* const* key_array, const size_t* key_length_array, + int num_keys, + size_t* filter_length) { + *filter_length = 4; + char* result = malloc(4); + memcpy(result, "fake", 4); + return result; +} +unsigned char FilterKeyMatch( + void* arg, + const char* key, size_t length, + const char* filter, size_t filter_length) { + CheckCondition(filter_length == 4); + CheckCondition(memcmp(filter, "fake", 4) == 0); + return fake_filter_result; +} + +int main(int argc, char** argv) { + leveldb_t* db; + leveldb_comparator_t* cmp; + leveldb_cache_t* cache; + leveldb_env_t* env; + leveldb_options_t* options; + leveldb_readoptions_t* roptions; + leveldb_writeoptions_t* woptions; + char* err = NULL; + int run = -1; + + CheckCondition(leveldb_major_version() >= 1); + CheckCondition(leveldb_minor_version() >= 1); + + snprintf(dbname, sizeof(dbname), + "%s/leveldb_c_test-%d", + GetTempDir(), + ((int) geteuid())); + + StartPhase("create_objects"); + cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); + env = leveldb_create_default_env(); + cache = leveldb_cache_create_lru(100000); + + options = leveldb_options_create(); + leveldb_options_set_comparator(options, cmp); + leveldb_options_set_error_if_exists(options, 1); + leveldb_options_set_cache(options, cache); + leveldb_options_set_env(options, env); + leveldb_options_set_info_log(options, NULL); + leveldb_options_set_write_buffer_size(options, 100000); + leveldb_options_set_paranoid_checks(options, 1); + leveldb_options_set_max_open_files(options, 10); + leveldb_options_set_block_size(options, 1024); + leveldb_options_set_block_restart_interval(options, 8); + leveldb_options_set_compression(options, leveldb_no_compression); + + roptions = leveldb_readoptions_create(); + leveldb_readoptions_set_verify_checksums(roptions, 1); + leveldb_readoptions_set_fill_cache(roptions, 0); + + woptions = leveldb_writeoptions_create(); + leveldb_writeoptions_set_sync(woptions, 1); + + StartPhase("destroy"); + leveldb_destroy_db(options, dbname, &err); + Free(&err); + + StartPhase("open_error"); + db = leveldb_open(options, dbname, &err); + CheckCondition(err != NULL); + Free(&err); + + StartPhase("leveldb_free"); + db = leveldb_open(options, dbname, &err); + CheckCondition(err != NULL); + leveldb_free(err); + err = NULL; + + StartPhase("open"); + leveldb_options_set_create_if_missing(options, 1); + db = leveldb_open(options, dbname, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", NULL); + + StartPhase("put"); + leveldb_put(db, woptions, "foo", 3, "hello", 5, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("compactall"); + leveldb_compact_range(db, NULL, 0, NULL, 0); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("compactrange"); + leveldb_compact_range(db, "a", 1, "z", 1); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("writebatch"); + { + leveldb_writebatch_t* wb = leveldb_writebatch_create(); + leveldb_writebatch_put(wb, "foo", 3, "a", 1); + leveldb_writebatch_clear(wb); + leveldb_writebatch_put(wb, "bar", 3, "b", 1); + leveldb_writebatch_put(wb, "box", 3, "c", 1); + leveldb_writebatch_delete(wb, "bar", 3); + leveldb_write(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "hello"); + CheckGet(db, roptions, "bar", NULL); + CheckGet(db, roptions, "box", "c"); + int pos = 0; + leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); + CheckCondition(pos == 3); + leveldb_writebatch_destroy(wb); + } + + StartPhase("iter"); + { + leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions); + CheckCondition(!leveldb_iter_valid(iter)); + leveldb_iter_seek_to_first(iter); + CheckCondition(leveldb_iter_valid(iter)); + CheckIter(iter, "box", "c"); + leveldb_iter_next(iter); + CheckIter(iter, "foo", "hello"); + leveldb_iter_prev(iter); + CheckIter(iter, "box", "c"); + leveldb_iter_prev(iter); + CheckCondition(!leveldb_iter_valid(iter)); + leveldb_iter_seek_to_last(iter); + CheckIter(iter, "foo", "hello"); + leveldb_iter_seek(iter, "b", 1); + CheckIter(iter, "box", "c"); + leveldb_iter_get_error(iter, &err); + CheckNoError(err); + leveldb_iter_destroy(iter); + } + + StartPhase("approximate_sizes"); + { + int i; + int n = 20000; + char keybuf[100]; + char valbuf[100]; + uint64_t sizes[2]; + const char* start[2] = { "a", "k00000000000000010000" }; + size_t start_len[2] = { 1, 21 }; + const char* limit[2] = { "k00000000000000010000", "z" }; + size_t limit_len[2] = { 21, 1 }; + leveldb_writeoptions_set_sync(woptions, 0); + for (i = 0; i < n; i++) { + snprintf(keybuf, sizeof(keybuf), "k%020d", i); + snprintf(valbuf, sizeof(valbuf), "v%020d", i); + leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf), + &err); + CheckNoError(err); + } + leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes); + CheckCondition(sizes[0] > 0); + CheckCondition(sizes[1] > 0); + } + + StartPhase("property"); + { + char* prop = leveldb_property_value(db, "nosuchprop"); + CheckCondition(prop == NULL); + prop = leveldb_property_value(db, "leveldb.stats"); + CheckCondition(prop != NULL); + Free(&prop); + } + + StartPhase("snapshot"); + { + const leveldb_snapshot_t* snap; + snap = leveldb_create_snapshot(db); + leveldb_delete(db, woptions, "foo", 3, &err); + CheckNoError(err); + leveldb_readoptions_set_snapshot(roptions, snap); + CheckGet(db, roptions, "foo", "hello"); + leveldb_readoptions_set_snapshot(roptions, NULL); + CheckGet(db, roptions, "foo", NULL); + leveldb_release_snapshot(db, snap); + } + + StartPhase("repair"); + { + leveldb_close(db); + leveldb_options_set_create_if_missing(options, 0); + leveldb_options_set_error_if_exists(options, 0); + leveldb_repair_db(options, dbname, &err); + CheckNoError(err); + db = leveldb_open(options, dbname, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", NULL); + CheckGet(db, roptions, "bar", NULL); + CheckGet(db, roptions, "box", "c"); + leveldb_options_set_create_if_missing(options, 1); + leveldb_options_set_error_if_exists(options, 1); + } + + StartPhase("filter"); + for (run = 0; run < 2; run++) { + // First run uses custom filter, second run uses bloom filter + CheckNoError(err); + leveldb_filterpolicy_t* policy; + if (run == 0) { + policy = leveldb_filterpolicy_create( + NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName); + } else { + policy = leveldb_filterpolicy_create_bloom(10); + } + + // Create new database + leveldb_close(db); + leveldb_destroy_db(options, dbname, &err); + leveldb_options_set_filter_policy(options, policy); + db = leveldb_open(options, dbname, &err); + CheckNoError(err); + leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err); + CheckNoError(err); + leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err); + CheckNoError(err); + leveldb_compact_range(db, NULL, 0, NULL, 0); + + fake_filter_result = 1; + CheckGet(db, roptions, "foo", "foovalue"); + CheckGet(db, roptions, "bar", "barvalue"); + if (phase == 0) { + // Must not find value when custom filter returns false + fake_filter_result = 0; + CheckGet(db, roptions, "foo", NULL); + CheckGet(db, roptions, "bar", NULL); + fake_filter_result = 1; + + CheckGet(db, roptions, "foo", "foovalue"); + CheckGet(db, roptions, "bar", "barvalue"); + } + leveldb_options_set_filter_policy(options, NULL); + leveldb_filterpolicy_destroy(policy); + } + + StartPhase("cleanup"); + leveldb_close(db); + leveldb_options_destroy(options); + leveldb_readoptions_destroy(roptions); + leveldb_writeoptions_destroy(woptions); + leveldb_cache_destroy(cache); + leveldb_comparator_destroy(cmp); + leveldb_env_destroy(env); + + fprintf(stderr, "PASS\n"); + return 0; +} diff --git a/Subtrees/hyperleveldb/db/corruption_test.cc b/Subtrees/hyperleveldb/db/corruption_test.cc new file mode 100644 index 0000000000..1c80f46b14 --- /dev/null +++ b/Subtrees/hyperleveldb/db/corruption_test.cc @@ -0,0 +1,360 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/db.h" + +#include +#include +#include +#include +#include "../hyperleveldb/cache.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/table.h" +#include "../hyperleveldb/write_batch.h" +#include "db_impl.h" +#include "filename.h" +#include "log_format.h" +#include "version_set.h" +#include "../util/logging.h" +#include "../util/testharness.h" +#include "../util/testutil.h" + +namespace hyperleveldb { + +static const int kValueSize = 1000; + +class CorruptionTest { + public: + test::ErrorEnv env_; + std::string dbname_; + Cache* tiny_cache_; + Options options_; + DB* db_; + + CorruptionTest() { + tiny_cache_ = NewLRUCache(100); + options_.env = &env_; + dbname_ = test::TmpDir() + "/db_test"; + DestroyDB(dbname_, options_); + + db_ = NULL; + options_.create_if_missing = true; + Reopen(); + options_.create_if_missing = false; + } + + ~CorruptionTest() { + delete db_; + DestroyDB(dbname_, Options()); + delete tiny_cache_; + } + + Status TryReopen(Options* options = NULL) { + delete db_; + db_ = NULL; + Options opt = (options ? *options : options_); + opt.env = &env_; + opt.block_cache = tiny_cache_; + return DB::Open(opt, dbname_, &db_); + } + + void Reopen(Options* options = NULL) { + ASSERT_OK(TryReopen(options)); + } + + void RepairDB() { + delete db_; + db_ = NULL; + ASSERT_OK(::leveldb::RepairDB(dbname_, options_)); + } + + void Build(int n) { + std::string key_space, value_space; + WriteBatch batch; + for (int i = 0; i < n; i++) { + //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n); + Slice key = Key(i, &key_space); + batch.Clear(); + batch.Put(key, Value(i, &value_space)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + } + } + + void Check(int min_expected, int max_expected) { + int next_expected = 0; + int missed = 0; + int bad_keys = 0; + int bad_values = 0; + int correct = 0; + std::string value_space; + Iterator* iter = db_->NewIterator(ReadOptions()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + uint64_t key; + Slice in(iter->key()); + if (!ConsumeDecimalNumber(&in, &key) || + !in.empty() || + key < next_expected) { + bad_keys++; + continue; + } + missed += (key - next_expected); + next_expected = key + 1; + if (iter->value() != Value(key, &value_space)) { + bad_values++; + } else { + correct++; + } + } + delete iter; + + fprintf(stderr, + "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n", + min_expected, max_expected, correct, bad_keys, bad_values, missed); + ASSERT_LE(min_expected, correct); + ASSERT_GE(max_expected, correct); + } + + void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) { + // Pick file to corrupt + std::vector filenames; + ASSERT_OK(env_.GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + std::string fname; + int picked_number = -1; + for (int i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && + type == filetype && + int(number) > picked_number) { // Pick latest file + fname = dbname_ + "/" + filenames[i]; + picked_number = number; + } + } + ASSERT_TRUE(!fname.empty()) << filetype; + + struct stat sbuf; + if (stat(fname.c_str(), &sbuf) != 0) { + const char* msg = strerror(errno); + ASSERT_TRUE(false) << fname << ": " << msg; + } + + if (offset < 0) { + // Relative to end of file; make it absolute + if (-offset > sbuf.st_size) { + offset = 0; + } else { + offset = sbuf.st_size + offset; + } + } + if (offset > sbuf.st_size) { + offset = sbuf.st_size; + } + if (offset + bytes_to_corrupt > sbuf.st_size) { + bytes_to_corrupt = sbuf.st_size - offset; + } + + // Do it + std::string contents; + Status s = ReadFileToString(Env::Default(), fname, &contents); + ASSERT_TRUE(s.ok()) << s.ToString(); + for (int i = 0; i < bytes_to_corrupt; i++) { + contents[i + offset] ^= 0x80; + } + s = WriteStringToFile(Env::Default(), contents, fname); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + + int Property(const std::string& name) { + std::string property; + int result; + if (db_->GetProperty(name, &property) && + sscanf(property.c_str(), "%d", &result) == 1) { + return result; + } else { + return -1; + } + } + + // Return the ith key + Slice Key(int i, std::string* storage) { + char buf[100]; + snprintf(buf, sizeof(buf), "%016d", i); + storage->assign(buf, strlen(buf)); + return Slice(*storage); + } + + // Return the value to associate with the specified key + Slice Value(int k, std::string* storage) { + Random r(k); + return test::RandomString(&r, kValueSize, storage); + } +}; + +TEST(CorruptionTest, Recovery) { + Build(100); + Check(100, 100); + Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record + Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block + Reopen(); + + // The 64 records in the first two log blocks are completely lost. + Check(36, 36); +} + +TEST(CorruptionTest, RecoverWriteError) { + env_.writable_file_error_ = true; + Status s = TryReopen(); + ASSERT_TRUE(!s.ok()); +} + +TEST(CorruptionTest, NewFileErrorDuringWrite) { + // Do enough writing to force minor compaction + env_.writable_file_error_ = true; + const int num = 3 + (Options().write_buffer_size / kValueSize); + std::string value_storage; + Status s; + for (int i = 0; s.ok() && i < num; i++) { + WriteBatch batch; + batch.Put("a", Value(100, &value_storage)); + s = db_->Write(WriteOptions(), &batch); + } + ASSERT_TRUE(!s.ok()); + ASSERT_GE(env_.num_writable_file_errors_, 1); + env_.writable_file_error_ = false; + Reopen(); +} + +TEST(CorruptionTest, TableFile) { + Build(100); + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + dbi->TEST_CompactRange(0, NULL, NULL); + dbi->TEST_CompactRange(1, NULL, NULL); + + Corrupt(kTableFile, 100, 1); + Check(99, 99); +} + +TEST(CorruptionTest, TableFileIndexData) { + Build(10000); // Enough to build multiple Tables + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + + Corrupt(kTableFile, -2000, 500); + Reopen(); + Check(5000, 9999); +} + +TEST(CorruptionTest, MissingDescriptor) { + Build(1000); + RepairDB(); + Reopen(); + Check(1000, 1000); +} + +TEST(CorruptionTest, SequenceNumberRecovery) { + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5")); + RepairDB(); + Reopen(); + std::string v; + ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("v5", v); + // Write something. If sequence number was not recovered properly, + // it will be hidden by an earlier write. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6")); + ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("v6", v); + Reopen(); + ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("v6", v); +} + +TEST(CorruptionTest, CorruptedDescriptor) { + ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello")); + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + dbi->TEST_CompactRange(0, NULL, NULL); + + Corrupt(kDescriptorFile, 0, 1000); + Status s = TryReopen(); + ASSERT_TRUE(!s.ok()); + + RepairDB(); + Reopen(); + std::string v; + ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("hello", v); +} + +TEST(CorruptionTest, CompactionInputError) { + Build(10); + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last))); + + Corrupt(kTableFile, 100, 1); + Check(9, 9); + + // Force compactions by writing lots of values + Build(10000); + Check(10000, 10000); +} + +TEST(CorruptionTest, CompactionInputErrorParanoid) { + Options options; + options.paranoid_checks = true; + options.write_buffer_size = 1048576; + Reopen(&options); + DBImpl* dbi = reinterpret_cast(db_); + + // Fill levels >= 1 so memtable compaction outputs to level 1 + for (int level = 1; level < config::kNumLevels; level++) { + dbi->Put(WriteOptions(), "", "begin"); + dbi->Put(WriteOptions(), "~", "end"); + dbi->TEST_CompactMemTable(); + } + + Build(10); + dbi->TEST_CompactMemTable(); + env_.SleepForMicroseconds(1000000); + ASSERT_EQ(1, Property("leveldb.num-files-at-level0")); + + Corrupt(kTableFile, 100, 1); + Check(9, 9); + + // Write must eventually fail because of corrupted table + Status s; + std::string tmp1, tmp2; + for (int i = 0; i < 1000000 && s.ok(); i++) { + s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2)); + } + ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db"; +} + +TEST(CorruptionTest, UnrelatedKeys) { + Build(10); + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + Corrupt(kTableFile, 100, 1); + + std::string tmp1, tmp2; + ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2))); + std::string v; + ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); + ASSERT_EQ(Value(1000, &tmp2).ToString(), v); + dbi->TEST_CompactMemTable(); + ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); + ASSERT_EQ(Value(1000, &tmp2).ToString(), v); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/db/db_bench.cc b/Subtrees/hyperleveldb/db/db_bench.cc new file mode 100644 index 0000000000..1569fa79fc --- /dev/null +++ b/Subtrees/hyperleveldb/db/db_bench.cc @@ -0,0 +1,979 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include "db_impl.h" +#include "version_set.h" +#include "../hyperleveldb/cache.h" +#include "../hyperleveldb/db.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/write_batch.h" +#include "../port/port.h" +#include "../util/crc32c.h" +#include "../util/histogram.h" +#include "../util/mutexlock.h" +#include "../util/random.h" +#include "../util/testutil.h" + +// Comma-separated list of operations to run in the specified order +// Actual benchmarks: +// fillseq -- write N values in sequential key order in async mode +// fillrandom -- write N values in random key order in async mode +// overwrite -- overwrite N values in random key order in async mode +// fillsync -- write N/100 values in random key order in sync mode +// fill100K -- write N/1000 100K values in random order in async mode +// deleteseq -- delete N keys in sequential order +// deleterandom -- delete N keys in random order +// readseq -- read N times sequentially +// readreverse -- read N times in reverse order +// readrandom -- read N times in random order +// readmissing -- read N missing keys in random order +// readhot -- read N times in random order from 1% section of DB +// seekrandom -- N random seeks +// crc32c -- repeated crc32c of 4K of data +// acquireload -- load N*1000 times +// Meta operations: +// compact -- Compact the entire DB +// stats -- Print DB stats +// sstables -- Print sstable info +// heapprofile -- Dump a heap profile (if supported by this port) +static const char* FLAGS_benchmarks = + "fillseq," + "fillsync," + "fillrandom," + "overwrite," + "readrandom," + "readrandom," // Extra run to allow previous compactions to quiesce + "readseq," + "readreverse," + "compact," + "readrandom," + "readseq," + "readreverse," + "fill100K," + "crc32c," + "snappycomp," + "snappyuncomp," + "acquireload," + ; + +// Number of key/values to place in database +static int FLAGS_num = 1000000; + +// Number of read operations to do. If negative, do FLAGS_num reads. +static int FLAGS_reads = -1; + +// Number of concurrent threads to run. +static int FLAGS_threads = 1; + +// Size of each value +static int FLAGS_value_size = 100; + +// Arrange to generate values that shrink to this fraction of +// their original size after compression +static double FLAGS_compression_ratio = 0.5; + +// Print histogram of operation timings +static bool FLAGS_histogram = false; + +// Number of bytes to buffer in memtable before compacting +// (initialized to default value by "main") +static int FLAGS_write_buffer_size = 0; + +// Number of bytes to use as a cache of uncompressed data. +// Negative means use default settings. +static int FLAGS_cache_size = -1; + +// Maximum number of files to keep open at the same time (use default if == 0) +static int FLAGS_open_files = 0; + +// Bloom filter bits per key. +// Negative means use default settings. +static int FLAGS_bloom_bits = -1; + +// If true, do not destroy the existing database. If you set this +// flag and also specify a benchmark that wants a fresh database, that +// benchmark will fail. +static bool FLAGS_use_existing_db = false; + +// Use the db with the following name. +static const char* FLAGS_db = NULL; + +namespace hyperleveldb { + +namespace { + +// Helper for quickly generating random data. +class RandomGenerator { + private: + std::string data_; + int pos_; + + public: + RandomGenerator() { + // We use a limited amount of data over and over again and ensure + // that it is larger than the compression window (32KB), and also + // large enough to serve all typical value sizes we want to write. + Random rnd(301); + std::string piece; + while (data_.size() < 1048576) { + // Add a short fragment that is as compressible as specified + // by FLAGS_compression_ratio. + test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece); + data_.append(piece); + } + pos_ = 0; + } + + Slice Generate(int len) { + if (pos_ + len > data_.size()) { + pos_ = 0; + assert(len < data_.size()); + } + pos_ += len; + return Slice(data_.data() + pos_ - len, len); + } +}; + +static Slice TrimSpace(Slice s) { + int start = 0; + while (start < s.size() && isspace(s[start])) { + start++; + } + int limit = s.size(); + while (limit > start && isspace(s[limit-1])) { + limit--; + } + return Slice(s.data() + start, limit - start); +} + +static void AppendWithSpace(std::string* str, Slice msg) { + if (msg.empty()) return; + if (!str->empty()) { + str->push_back(' '); + } + str->append(msg.data(), msg.size()); +} + +class Stats { + private: + double start_; + double finish_; + double seconds_; + int done_; + int next_report_; + int64_t bytes_; + double last_op_finish_; + Histogram hist_; + std::string message_; + + public: + Stats() { Start(); } + + void Start() { + next_report_ = 100; + last_op_finish_ = start_; + hist_.Clear(); + done_ = 0; + bytes_ = 0; + seconds_ = 0; + start_ = Env::Default()->NowMicros(); + finish_ = start_; + message_.clear(); + } + + void Merge(const Stats& other) { + hist_.Merge(other.hist_); + done_ += other.done_; + bytes_ += other.bytes_; + seconds_ += other.seconds_; + if (other.start_ < start_) start_ = other.start_; + if (other.finish_ > finish_) finish_ = other.finish_; + + // Just keep the messages from one thread + if (message_.empty()) message_ = other.message_; + } + + void Stop() { + finish_ = Env::Default()->NowMicros(); + seconds_ = (finish_ - start_) * 1e-6; + } + + void AddMessage(Slice msg) { + AppendWithSpace(&message_, msg); + } + + void FinishedSingleOp() { + if (FLAGS_histogram) { + double now = Env::Default()->NowMicros(); + double micros = now - last_op_finish_; + hist_.Add(micros); + if (micros > 20000) { + fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); + fflush(stderr); + } + last_op_finish_ = now; + } + + done_++; + if (done_ >= next_report_) { + if (next_report_ < 1000) next_report_ += 100; + else if (next_report_ < 5000) next_report_ += 500; + else if (next_report_ < 10000) next_report_ += 1000; + else if (next_report_ < 50000) next_report_ += 5000; + else if (next_report_ < 100000) next_report_ += 10000; + else if (next_report_ < 500000) next_report_ += 50000; + else next_report_ += 100000; + fprintf(stderr, "... finished %d ops%30s\r", done_, ""); + fflush(stderr); + } + } + + void AddBytes(int64_t n) { + bytes_ += n; + } + + void Report(const Slice& name) { + // Pretend at least one op was done in case we are running a benchmark + // that does not call FinishedSingleOp(). + if (done_ < 1) done_ = 1; + + std::string extra; + if (bytes_ > 0) { + // Rate is computed on actual elapsed time, not the sum of per-thread + // elapsed times. + double elapsed = (finish_ - start_) * 1e-6; + char rate[100]; + snprintf(rate, sizeof(rate), "%6.1f MB/s", + (bytes_ / 1048576.0) / elapsed); + extra = rate; + } + AppendWithSpace(&extra, message_); + + fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", + name.ToString().c_str(), + seconds_ * 1e6 / done_, + (extra.empty() ? "" : " "), + extra.c_str()); + if (FLAGS_histogram) { + fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); + } + fflush(stdout); + } +}; + +// State shared by all concurrent executions of the same benchmark. +struct SharedState { + port::Mutex mu; + port::CondVar cv; + int total; + + // Each thread goes through the following states: + // (1) initializing + // (2) waiting for others to be initialized + // (3) running + // (4) done + + int num_initialized; + int num_done; + bool start; + + SharedState() : cv(&mu) { } +}; + +// Per-thread state for concurrent executions of the same benchmark. +struct ThreadState { + int tid; // 0..n-1 when running in n threads + Random rand; // Has different seeds for different threads + Stats stats; + SharedState* shared; + + ThreadState(int index) + : tid(index), + rand(1000 + index) { + } +}; + +} // namespace + +class Benchmark { + private: + Cache* cache_; + const FilterPolicy* filter_policy_; + DB* db_; + int num_; + int value_size_; + int entries_per_batch_; + WriteOptions write_options_; + int reads_; + int heap_counter_; + + void PrintHeader() { + const int kKeySize = 16; + PrintEnvironment(); + fprintf(stdout, "Keys: %d bytes each\n", kKeySize); + fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n", + FLAGS_value_size, + static_cast(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); + fprintf(stdout, "Entries: %d\n", num_); + fprintf(stdout, "RawSize: %.1f MB (estimated)\n", + ((static_cast(kKeySize + FLAGS_value_size) * num_) + / 1048576.0)); + fprintf(stdout, "FileSize: %.1f MB (estimated)\n", + (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) + / 1048576.0)); + PrintWarnings(); + fprintf(stdout, "------------------------------------------------\n"); + } + + void PrintWarnings() { +#if defined(__GNUC__) && !defined(__OPTIMIZE__) + fprintf(stdout, + "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n" + ); +#endif +#ifndef NDEBUG + fprintf(stdout, + "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); +#endif + + // See if snappy is working by attempting to compress a compressible string + const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"; + std::string compressed; + if (!port::Snappy_Compress(text, sizeof(text), &compressed)) { + fprintf(stdout, "WARNING: Snappy compression is not enabled\n"); + } else if (compressed.size() >= sizeof(text)) { + fprintf(stdout, "WARNING: Snappy compression is not effective\n"); + } + } + + void PrintEnvironment() { + fprintf(stderr, "LevelDB: version %d.%d\n", + kMajorVersion, kMinorVersion); + +#if defined(__linux) + time_t now = time(NULL); + fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline + + FILE* cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo != NULL) { + char line[1000]; + int num_cpus = 0; + std::string cpu_type; + std::string cache_size; + while (fgets(line, sizeof(line), cpuinfo) != NULL) { + const char* sep = strchr(line, ':'); + if (sep == NULL) { + continue; + } + Slice key = TrimSpace(Slice(line, sep - 1 - line)); + Slice val = TrimSpace(Slice(sep + 1)); + if (key == "model name") { + ++num_cpus; + cpu_type = val.ToString(); + } else if (key == "cache size") { + cache_size = val.ToString(); + } + } + fclose(cpuinfo); + fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str()); + fprintf(stderr, "CPUCache: %s\n", cache_size.c_str()); + } +#endif + } + + public: + Benchmark() + : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL), + filter_policy_(FLAGS_bloom_bits >= 0 + ? NewBloomFilterPolicy(FLAGS_bloom_bits) + : NULL), + db_(NULL), + num_(FLAGS_num), + value_size_(FLAGS_value_size), + entries_per_batch_(1), + reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), + heap_counter_(0) { + std::vector files; + Env::Default()->GetChildren(FLAGS_db, &files); + for (int i = 0; i < files.size(); i++) { + if (Slice(files[i]).starts_with("heap-")) { + Env::Default()->DeleteFile(std::string(FLAGS_db) + "/" + files[i]); + } + } + if (!FLAGS_use_existing_db) { + DestroyDB(FLAGS_db, Options()); + } + } + + ~Benchmark() { + delete db_; + delete cache_; + delete filter_policy_; + } + + void Run() { + PrintHeader(); + Open(); + + const char* benchmarks = FLAGS_benchmarks; + while (benchmarks != NULL) { + const char* sep = strchr(benchmarks, ','); + Slice name; + if (sep == NULL) { + name = benchmarks; + benchmarks = NULL; + } else { + name = Slice(benchmarks, sep - benchmarks); + benchmarks = sep + 1; + } + + // Reset parameters that may be overriddden bwlow + num_ = FLAGS_num; + reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); + value_size_ = FLAGS_value_size; + entries_per_batch_ = 1; + write_options_ = WriteOptions(); + + void (Benchmark::*method)(ThreadState*) = NULL; + bool fresh_db = false; + int num_threads = FLAGS_threads; + + if (name == Slice("fillseq")) { + fresh_db = true; + method = &Benchmark::WriteSeq; + } else if (name == Slice("fillbatch")) { + fresh_db = true; + entries_per_batch_ = 1000; + method = &Benchmark::WriteSeq; + } else if (name == Slice("fillrandom")) { + fresh_db = true; + method = &Benchmark::WriteRandom; + } else if (name == Slice("overwrite")) { + fresh_db = false; + method = &Benchmark::WriteRandom; + } else if (name == Slice("fillsync")) { + fresh_db = true; + num_ /= 1000; + write_options_.sync = true; + method = &Benchmark::WriteRandom; + } else if (name == Slice("fill100K")) { + fresh_db = true; + num_ /= 1000; + value_size_ = 100 * 1000; + method = &Benchmark::WriteRandom; + } else if (name == Slice("readseq")) { + method = &Benchmark::ReadSequential; + } else if (name == Slice("readreverse")) { + method = &Benchmark::ReadReverse; + } else if (name == Slice("readrandom")) { + method = &Benchmark::ReadRandom; + } else if (name == Slice("readmissing")) { + method = &Benchmark::ReadMissing; + } else if (name == Slice("seekrandom")) { + method = &Benchmark::SeekRandom; + } else if (name == Slice("readhot")) { + method = &Benchmark::ReadHot; + } else if (name == Slice("readrandomsmall")) { + reads_ /= 1000; + method = &Benchmark::ReadRandom; + } else if (name == Slice("deleteseq")) { + method = &Benchmark::DeleteSeq; + } else if (name == Slice("deleterandom")) { + method = &Benchmark::DeleteRandom; + } else if (name == Slice("readwhilewriting")) { + num_threads++; // Add extra thread for writing + method = &Benchmark::ReadWhileWriting; + } else if (name == Slice("compact")) { + method = &Benchmark::Compact; + } else if (name == Slice("crc32c")) { + method = &Benchmark::Crc32c; + } else if (name == Slice("acquireload")) { + method = &Benchmark::AcquireLoad; + } else if (name == Slice("snappycomp")) { + method = &Benchmark::SnappyCompress; + } else if (name == Slice("snappyuncomp")) { + method = &Benchmark::SnappyUncompress; + } else if (name == Slice("heapprofile")) { + HeapProfile(); + } else if (name == Slice("stats")) { + PrintStats("leveldb.stats"); + } else if (name == Slice("sstables")) { + PrintStats("leveldb.sstables"); + } else { + if (name != Slice()) { // No error message for empty name + fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str()); + } + } + + if (fresh_db) { + if (FLAGS_use_existing_db) { + fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n", + name.ToString().c_str()); + method = NULL; + } else { + delete db_; + db_ = NULL; + DestroyDB(FLAGS_db, Options()); + Open(); + } + } + + if (method != NULL) { + RunBenchmark(num_threads, name, method); + } + } + } + + private: + struct ThreadArg { + Benchmark* bm; + SharedState* shared; + ThreadState* thread; + void (Benchmark::*method)(ThreadState*); + }; + + static void ThreadBody(void* v) { + ThreadArg* arg = reinterpret_cast(v); + SharedState* shared = arg->shared; + ThreadState* thread = arg->thread; + { + MutexLock l(&shared->mu); + shared->num_initialized++; + if (shared->num_initialized >= shared->total) { + shared->cv.SignalAll(); + } + while (!shared->start) { + shared->cv.Wait(); + } + } + + thread->stats.Start(); + (arg->bm->*(arg->method))(thread); + thread->stats.Stop(); + + { + MutexLock l(&shared->mu); + shared->num_done++; + if (shared->num_done >= shared->total) { + shared->cv.SignalAll(); + } + } + } + + void RunBenchmark(int n, Slice name, + void (Benchmark::*method)(ThreadState*)) { + SharedState shared; + shared.total = n; + shared.num_initialized = 0; + shared.num_done = 0; + shared.start = false; + + ThreadArg* arg = new ThreadArg[n]; + for (int i = 0; i < n; i++) { + arg[i].bm = this; + arg[i].method = method; + arg[i].shared = &shared; + arg[i].thread = new ThreadState(i); + arg[i].thread->shared = &shared; + Env::Default()->StartThread(ThreadBody, &arg[i]); + } + + shared.mu.Lock(); + while (shared.num_initialized < n) { + shared.cv.Wait(); + } + + shared.start = true; + shared.cv.SignalAll(); + while (shared.num_done < n) { + shared.cv.Wait(); + } + shared.mu.Unlock(); + + for (int i = 1; i < n; i++) { + arg[0].thread->stats.Merge(arg[i].thread->stats); + } + arg[0].thread->stats.Report(name); + + for (int i = 0; i < n; i++) { + delete arg[i].thread; + } + delete[] arg; + } + + void Crc32c(ThreadState* thread) { + // Checksum about 500MB of data total + const int size = 4096; + const char* label = "(4K per op)"; + std::string data(size, 'x'); + int64_t bytes = 0; + uint32_t crc = 0; + while (bytes < 500 * 1048576) { + crc = crc32c::Value(data.data(), size); + thread->stats.FinishedSingleOp(); + bytes += size; + } + // Print so result is not dead + fprintf(stderr, "... crc=0x%x\r", static_cast(crc)); + + thread->stats.AddBytes(bytes); + thread->stats.AddMessage(label); + } + + void AcquireLoad(ThreadState* thread) { + int dummy; + port::AtomicPointer ap(&dummy); + int count = 0; + void *ptr = NULL; + thread->stats.AddMessage("(each op is 1000 loads)"); + while (count < 100000) { + for (int i = 0; i < 1000; i++) { + ptr = ap.Acquire_Load(); + } + count++; + thread->stats.FinishedSingleOp(); + } + if (ptr == NULL) exit(1); // Disable unused variable warning. + } + + void SnappyCompress(ThreadState* thread) { + RandomGenerator gen; + Slice input = gen.Generate(Options().block_size); + int64_t bytes = 0; + int64_t produced = 0; + bool ok = true; + std::string compressed; + while (ok && bytes < 1024 * 1048576) { // Compress 1G + ok = port::Snappy_Compress(input.data(), input.size(), &compressed); + produced += compressed.size(); + bytes += input.size(); + thread->stats.FinishedSingleOp(); + } + + if (!ok) { + thread->stats.AddMessage("(snappy failure)"); + } else { + char buf[100]; + snprintf(buf, sizeof(buf), "(output: %.1f%%)", + (produced * 100.0) / bytes); + thread->stats.AddMessage(buf); + thread->stats.AddBytes(bytes); + } + } + + void SnappyUncompress(ThreadState* thread) { + RandomGenerator gen; + Slice input = gen.Generate(Options().block_size); + std::string compressed; + bool ok = port::Snappy_Compress(input.data(), input.size(), &compressed); + int64_t bytes = 0; + char* uncompressed = new char[input.size()]; + while (ok && bytes < 1024 * 1048576) { // Compress 1G + ok = port::Snappy_Uncompress(compressed.data(), compressed.size(), + uncompressed); + bytes += input.size(); + thread->stats.FinishedSingleOp(); + } + delete[] uncompressed; + + if (!ok) { + thread->stats.AddMessage("(snappy failure)"); + } else { + thread->stats.AddBytes(bytes); + } + } + + void Open() { + assert(db_ == NULL); + Options options; + options.create_if_missing = !FLAGS_use_existing_db; + options.block_cache = cache_; + options.write_buffer_size = FLAGS_write_buffer_size; + options.max_open_files = FLAGS_open_files; + options.filter_policy = filter_policy_; + Status s = DB::Open(options, FLAGS_db, &db_); + if (!s.ok()) { + fprintf(stderr, "open error: %s\n", s.ToString().c_str()); + exit(1); + } + } + + void WriteSeq(ThreadState* thread) { + DoWrite(thread, true); + } + + void WriteRandom(ThreadState* thread) { + DoWrite(thread, false); + } + + void DoWrite(ThreadState* thread, bool seq) { + if (num_ != FLAGS_num) { + char msg[100]; + snprintf(msg, sizeof(msg), "(%d ops)", num_); + thread->stats.AddMessage(msg); + } + + RandomGenerator gen; + WriteBatch batch; + Status s; + int64_t bytes = 0; + for (int i = 0; i < num_; i += entries_per_batch_) { + batch.Clear(); + for (int j = 0; j < entries_per_batch_; j++) { + const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); + char key[100]; + snprintf(key, sizeof(key), "%016d", k); + batch.Put(key, gen.Generate(value_size_)); + bytes += value_size_ + strlen(key); + thread->stats.FinishedSingleOp(); + } + s = db_->Write(write_options_, &batch); + if (!s.ok()) { + fprintf(stderr, "put error: %s\n", s.ToString().c_str()); + exit(1); + } + } + thread->stats.AddBytes(bytes); + } + + void ReadSequential(ThreadState* thread) { + Iterator* iter = db_->NewIterator(ReadOptions()); + int i = 0; + int64_t bytes = 0; + for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) { + bytes += iter->key().size() + iter->value().size(); + thread->stats.FinishedSingleOp(); + ++i; + } + delete iter; + thread->stats.AddBytes(bytes); + } + + void ReadReverse(ThreadState* thread) { + Iterator* iter = db_->NewIterator(ReadOptions()); + int i = 0; + int64_t bytes = 0; + for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) { + bytes += iter->key().size() + iter->value().size(); + thread->stats.FinishedSingleOp(); + ++i; + } + delete iter; + thread->stats.AddBytes(bytes); + } + + void ReadRandom(ThreadState* thread) { + ReadOptions options; + std::string value; + int found = 0; + for (int i = 0; i < reads_; i++) { + char key[100]; + const int k = thread->rand.Next() % FLAGS_num; + snprintf(key, sizeof(key), "%016d", k); + if (db_->Get(options, key, &value).ok()) { + found++; + } + thread->stats.FinishedSingleOp(); + } + char msg[100]; + snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); + thread->stats.AddMessage(msg); + } + + void ReadMissing(ThreadState* thread) { + ReadOptions options; + std::string value; + for (int i = 0; i < reads_; i++) { + char key[100]; + const int k = thread->rand.Next() % FLAGS_num; + snprintf(key, sizeof(key), "%016d.", k); + db_->Get(options, key, &value); + thread->stats.FinishedSingleOp(); + } + } + + void ReadHot(ThreadState* thread) { + ReadOptions options; + std::string value; + const int range = (FLAGS_num + 99) / 100; + for (int i = 0; i < reads_; i++) { + char key[100]; + const int k = thread->rand.Next() % range; + snprintf(key, sizeof(key), "%016d", k); + db_->Get(options, key, &value); + thread->stats.FinishedSingleOp(); + } + } + + void SeekRandom(ThreadState* thread) { + ReadOptions options; + std::string value; + int found = 0; + for (int i = 0; i < reads_; i++) { + Iterator* iter = db_->NewIterator(options); + char key[100]; + const int k = thread->rand.Next() % FLAGS_num; + snprintf(key, sizeof(key), "%016d", k); + iter->Seek(key); + if (iter->Valid() && iter->key() == key) found++; + delete iter; + thread->stats.FinishedSingleOp(); + } + char msg[100]; + snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); + thread->stats.AddMessage(msg); + } + + void DoDelete(ThreadState* thread, bool seq) { + RandomGenerator gen; + WriteBatch batch; + Status s; + for (int i = 0; i < num_; i += entries_per_batch_) { + batch.Clear(); + for (int j = 0; j < entries_per_batch_; j++) { + const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); + char key[100]; + snprintf(key, sizeof(key), "%016d", k); + batch.Delete(key); + thread->stats.FinishedSingleOp(); + } + s = db_->Write(write_options_, &batch); + if (!s.ok()) { + fprintf(stderr, "del error: %s\n", s.ToString().c_str()); + exit(1); + } + } + } + + void DeleteSeq(ThreadState* thread) { + DoDelete(thread, true); + } + + void DeleteRandom(ThreadState* thread) { + DoDelete(thread, false); + } + + void ReadWhileWriting(ThreadState* thread) { + if (thread->tid > 0) { + ReadRandom(thread); + } else { + // Special thread that keeps writing until other threads are done. + RandomGenerator gen; + while (true) { + { + MutexLock l(&thread->shared->mu); + if (thread->shared->num_done + 1 >= thread->shared->num_initialized) { + // Other threads have finished + break; + } + } + + const int k = thread->rand.Next() % FLAGS_num; + char key[100]; + snprintf(key, sizeof(key), "%016d", k); + Status s = db_->Put(write_options_, key, gen.Generate(value_size_)); + if (!s.ok()) { + fprintf(stderr, "put error: %s\n", s.ToString().c_str()); + exit(1); + } + } + + // Do not count any of the preceding work/delay in stats. + thread->stats.Start(); + } + } + + void Compact(ThreadState* thread) { + db_->CompactRange(NULL, NULL); + } + + void PrintStats(const char* key) { + std::string stats; + if (!db_->GetProperty(key, &stats)) { + stats = "(failed)"; + } + fprintf(stdout, "\n%s\n", stats.c_str()); + } + + static void WriteToFile(void* arg, const char* buf, int n) { + reinterpret_cast(arg)->Append(Slice(buf, n)); + } + + void HeapProfile() { + char fname[100]; + snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_); + WritableFile* file; + Status s = Env::Default()->NewWritableFile(fname, &file); + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + return; + } + bool ok = port::GetHeapProfile(WriteToFile, file); + delete file; + if (!ok) { + fprintf(stderr, "heap profiling not supported\n"); + Env::Default()->DeleteFile(fname); + } + } +}; + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + FLAGS_write_buffer_size = leveldb::Options().write_buffer_size; + FLAGS_open_files = leveldb::Options().max_open_files; + std::string default_db_path; + + for (int i = 1; i < argc; i++) { + double d; + int n; + char junk; + if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) { + FLAGS_benchmarks = argv[i] + strlen("--benchmarks="); + } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) { + FLAGS_compression_ratio = d; + } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_histogram = n; + } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_use_existing_db = n; + } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { + FLAGS_num = n; + } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { + FLAGS_reads = n; + } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) { + FLAGS_threads = n; + } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { + FLAGS_value_size = n; + } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) { + FLAGS_write_buffer_size = n; + } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) { + FLAGS_cache_size = n; + } else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) { + FLAGS_bloom_bits = n; + } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) { + FLAGS_open_files = n; + } else if (strncmp(argv[i], "--db=", 5) == 0) { + FLAGS_db = argv[i] + 5; + } else { + fprintf(stderr, "Invalid flag '%s'\n", argv[i]); + exit(1); + } + } + + // Choose a location for the test database if none given with --db= + if (FLAGS_db == NULL) { + leveldb::Env::Default()->GetTestDirectory(&default_db_path); + default_db_path += "/dbbench"; + FLAGS_db = default_db_path.c_str(); + } + + leveldb::Benchmark benchmark; + benchmark.Run(); + return 0; +} diff --git a/Subtrees/hyperleveldb/db/db_impl.cc b/Subtrees/hyperleveldb/db/db_impl.cc new file mode 100644 index 0000000000..613cba295c --- /dev/null +++ b/Subtrees/hyperleveldb/db/db_impl.cc @@ -0,0 +1,1616 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db_impl.h" + +#include +#include +#include +#include +#include +#include +#include "builder.h" +#include "db_iter.h" +#include "dbformat.h" +#include "filename.h" +#include "log_reader.h" +#include "log_writer.h" +#include "memtable.h" +#include "table_cache.h" +#include "version_set.h" +#include "write_batch_internal.h" +#include "../hyperleveldb/db.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/status.h" +#include "../hyperleveldb/table.h" +#include "../hyperleveldb/table_builder.h" +#include "../port/port.h" +#include "../table/block.h" +#include "../table/merger.h" +#include "../table/two_level_iterator.h" +#include "../util/coding.h" +#include "../util/logging.h" +#include "../util/mutexlock.h" + +namespace hyperleveldb { + +const int kNumNonTableCacheFiles = 10; + +struct DBImpl::CompactionState { + Compaction* const compaction; + + // Sequence numbers < smallest_snapshot are not significant since we + // will never have to service a snapshot below smallest_snapshot. + // Therefore if we have seen a sequence number S <= smallest_snapshot, + // we can drop all entries for the same key with sequence numbers < S. + SequenceNumber smallest_snapshot; + + // Files produced by compaction + struct Output { + uint64_t number; + uint64_t file_size; + InternalKey smallest, largest; + }; + std::vector outputs; + + // State kept for output being generated + WritableFile* outfile; + TableBuilder* builder; + + uint64_t total_bytes; + + Output* current_output() { return &outputs[outputs.size()-1]; } + + explicit CompactionState(Compaction* c) + : compaction(c), + outfile(NULL), + builder(NULL), + total_bytes(0) { + } +}; + +// Fix user-supplied options to be reasonable +template +static void ClipToRange(T* ptr, V minvalue, V maxvalue) { + if (static_cast(*ptr) > maxvalue) *ptr = maxvalue; + if (static_cast(*ptr) < minvalue) *ptr = minvalue; +} +Options SanitizeOptions(const std::string& dbname, + const InternalKeyComparator* icmp, + const InternalFilterPolicy* ipolicy, + const Options& src) { + Options result = src; + result.comparator = icmp; + result.filter_policy = (src.filter_policy != NULL) ? ipolicy : NULL; + ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000); + ClipToRange(&result.write_buffer_size, 64<<10, 1<<30); + ClipToRange(&result.block_size, 1<<10, 4<<20); + if (result.info_log == NULL) { + // Open a log file in the same directory as the db + src.env->CreateDir(dbname); // In case it does not exist + src.env->RenameFile(InfoLogFileName(dbname), OldInfoLogFileName(dbname)); + Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log); + if (!s.ok()) { + // No place suitable for logging + result.info_log = NULL; + } + } + if (result.block_cache == NULL) { + result.block_cache = NewLRUCache(8 << 20); + } + return result; +} + +DBImpl::DBImpl(const Options& options, const std::string& dbname) + : env_(options.env), + internal_comparator_(options.comparator), + internal_filter_policy_(options.filter_policy), + options_(SanitizeOptions( + dbname, &internal_comparator_, &internal_filter_policy_, options)), + owns_info_log_(options_.info_log != options.info_log), + owns_cache_(options_.block_cache != options.block_cache), + dbname_(dbname), + db_lock_(NULL), + shutting_down_(NULL), + mem_(new MemTable(internal_comparator_)), + imm_(NULL), + logfile_(NULL), + logfile_number_(0), + log_(NULL), + writers_lower_(0), + writers_upper_(0), + bg_fg_cv_(&mutex_), + allow_background_activity_(false), + num_bg_threads_(0), + bg_compaction_cv_(&mutex_), + bg_memtable_cv_(&mutex_), + bg_optimistic_trip_(false), + bg_optimistic_cv_(&mutex_), + bg_log_cv_(&mutex_), + bg_log_occupied_(false), + manual_compaction_(NULL), + consecutive_compaction_errors_(0) { + mutex_.Lock(); + mem_->Ref(); + has_imm_.Release_Store(NULL); + env_->StartThread(&DBImpl::CompactMemTableWrapper, this); + env_->StartThread(&DBImpl::CompactOptimisticWrapper, this); + env_->StartThread(&DBImpl::CompactLevelWrapper, this); + num_bg_threads_ = 3; + + // Reserve ten files or so for other uses and give the rest to TableCache. + const int table_cache_size = options.max_open_files - kNumNonTableCacheFiles; + table_cache_ = new TableCache(dbname_, &options_, table_cache_size); + versions_ = new VersionSet(dbname_, &options_, table_cache_, + &internal_comparator_); + + for (int i = 0; i < config::kNumLevels; ++i) { + levels_locked_[i] = false; + } + mutex_.Unlock(); +} + +DBImpl::~DBImpl() { + // Wait for background work to finish + mutex_.Lock(); + shutting_down_.Release_Store(this); // Any non-NULL value is ok + bg_optimistic_cv_.SignalAll(); + bg_compaction_cv_.SignalAll(); + bg_memtable_cv_.SignalAll(); + while (num_bg_threads_ > 0) { + bg_fg_cv_.Wait(); + } + mutex_.Unlock(); + + if (db_lock_ != NULL) { + env_->UnlockFile(db_lock_); + } + + delete versions_; + if (mem_ != NULL) mem_->Unref(); + if (imm_ != NULL) imm_->Unref(); + delete log_; + delete logfile_; + delete table_cache_; + + if (owns_info_log_) { + delete options_.info_log; + } + if (owns_cache_) { + delete options_.block_cache; + } +} + +Status DBImpl::NewDB() { + VersionEdit new_db; + new_db.SetComparatorName(user_comparator()->Name()); + new_db.SetLogNumber(0); + new_db.SetNextFile(2); + new_db.SetLastSequence(0); + + const std::string manifest = DescriptorFileName(dbname_, 1); + WritableFile* file; + Status s = env_->NewWritableFile(manifest, &file); + if (!s.ok()) { + return s; + } + { + log::Writer log(file); + std::string record; + new_db.EncodeTo(&record); + s = log.AddRecord(record); + if (s.ok()) { + s = file->Close(); + } + } + delete file; + if (s.ok()) { + // Make "CURRENT" file that points to the new manifest file. + s = SetCurrentFile(env_, dbname_, 1); + } else { + env_->DeleteFile(manifest); + } + return s; +} + +void DBImpl::MaybeIgnoreError(Status* s) const { + if (s->ok() || options_.paranoid_checks) { + // No change needed + } else { + Log(options_.info_log, "Ignoring error %s", s->ToString().c_str()); + *s = Status::OK(); + } +} + +void DBImpl::DeleteObsoleteFiles() { + // Make a set of all of the live files + std::set live = pending_outputs_; + versions_->AddLiveFiles(&live); + + std::vector filenames; + env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type)) { + bool keep = true; + switch (type) { + case kLogFile: + keep = ((number >= versions_->LogNumber()) || + (number == versions_->PrevLogNumber())); + break; + case kDescriptorFile: + // Keep my manifest file, and any newer incarnations' + // (in case there is a race that allows other incarnations) + keep = (number >= versions_->ManifestFileNumber()); + break; + case kTableFile: + keep = (live.find(number) != live.end()); + break; + case kTempFile: + // Any temp files that are currently being written to must + // be recorded in pending_outputs_, which is inserted into "live" + keep = (live.find(number) != live.end()); + break; + case kCurrentFile: + case kDBLockFile: + case kInfoLogFile: + keep = true; + break; + } + + if (!keep) { + if (type == kTableFile) { + table_cache_->Evict(number); + } + Log(options_.info_log, "Delete type=%d #%lld\n", + int(type), + static_cast(number)); + env_->DeleteFile(dbname_ + "/" + filenames[i]); + } + } + } +} + +Status DBImpl::Recover(VersionEdit* edit) { + mutex_.AssertHeld(); + + // Ignore error from CreateDir since the creation of the DB is + // committed only when the descriptor is created, and this directory + // may already exist from a previous failed creation attempt. + env_->CreateDir(dbname_); + assert(db_lock_ == NULL); + Status s = env_->LockFile(LockFileName(dbname_), &db_lock_); + if (!s.ok()) { + return s; + } + + if (!env_->FileExists(CurrentFileName(dbname_))) { + if (options_.create_if_missing) { + s = NewDB(); + if (!s.ok()) { + return s; + } + } else { + return Status::InvalidArgument( + dbname_, "does not exist (create_if_missing is false)"); + } + } else { + if (options_.error_if_exists) { + return Status::InvalidArgument( + dbname_, "exists (error_if_exists is true)"); + } + } + + s = versions_->Recover(); + if (s.ok()) { + SequenceNumber max_sequence(0); + + // Recover from all newer log files than the ones named in the + // descriptor (new log files may have been added by the previous + // incarnation without registering them in the descriptor). + // + // Note that PrevLogNumber() is no longer used, but we pay + // attention to it in case we are recovering a database + // produced by an older version of leveldb. + const uint64_t min_log = versions_->LogNumber(); + const uint64_t prev_log = versions_->PrevLogNumber(); + std::vector filenames; + s = env_->GetChildren(dbname_, &filenames); + if (!s.ok()) { + return s; + } + std::set expected; + versions_->AddLiveFiles(&expected); + uint64_t number; + FileType type; + std::vector logs; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type)) { + expected.erase(number); + if (type == kLogFile && ((number >= min_log) || (number == prev_log))) + logs.push_back(number); + } + } + if (!expected.empty()) { + char buf[50]; + snprintf(buf, sizeof(buf), "%d missing files; e.g.", + static_cast(expected.size())); + return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); + } + + // Recover in the order in which the logs were generated + std::sort(logs.begin(), logs.end()); + for (size_t i = 0; i < logs.size(); i++) { + s = RecoverLogFile(logs[i], edit, &max_sequence); + + // The previous incarnation may not have written any MANIFEST + // records after allocating this log number. So we manually + // update the file number allocation counter in VersionSet. + versions_->MarkFileNumberUsed(logs[i]); + } + + if (s.ok()) { + if (versions_->LastSequence() < max_sequence) { + versions_->SetLastSequence(max_sequence); + } + } + } + + return s; +} + +Status DBImpl::RecoverLogFile(uint64_t log_number, + VersionEdit* edit, + SequenceNumber* max_sequence) { + struct LogReporter : public log::Reader::Reporter { + Env* env; + Logger* info_log; + const char* fname; + Status* status; // NULL if options_.paranoid_checks==false + virtual void Corruption(size_t bytes, const Status& s) { + Log(info_log, "%s%s: dropping %d bytes; %s", + (this->status == NULL ? "(ignoring error) " : ""), + fname, static_cast(bytes), s.ToString().c_str()); + if (this->status != NULL && this->status->ok()) *this->status = s; + } + }; + + mutex_.AssertHeld(); + + // Open the log file + std::string fname = LogFileName(dbname_, log_number); + SequentialFile* file; + Status status = env_->NewSequentialFile(fname, &file); + if (!status.ok()) { + MaybeIgnoreError(&status); + return status; + } + + // Create the log reader. + LogReporter reporter; + reporter.env = env_; + reporter.info_log = options_.info_log; + reporter.fname = fname.c_str(); + reporter.status = (options_.paranoid_checks ? &status : NULL); + // We intentially make log::Reader do checksumming even if + // paranoid_checks==false so that corruptions cause entire commits + // to be skipped instead of propagating bad information (like overly + // large sequence numbers). + log::Reader reader(file, &reporter, true/*checksum*/, + 0/*initial_offset*/); + Log(options_.info_log, "Recovering log #%llu", + (unsigned long long) log_number); + + // Read all the records and add to a memtable + std::string scratch; + Slice record; + WriteBatch batch; + MemTable* mem = NULL; + while (reader.ReadRecord(&record, &scratch) && + status.ok()) { + if (record.size() < 12) { + reporter.Corruption( + record.size(), Status::Corruption("log record too small")); + continue; + } + WriteBatchInternal::SetContents(&batch, record); + + if (mem == NULL) { + mem = new MemTable(internal_comparator_); + mem->Ref(); + } + status = WriteBatchInternal::InsertInto(&batch, mem); + MaybeIgnoreError(&status); + if (!status.ok()) { + break; + } + const SequenceNumber last_seq = + WriteBatchInternal::Sequence(&batch) + + WriteBatchInternal::Count(&batch) - 1; + if (last_seq > *max_sequence) { + *max_sequence = last_seq; + } + + if (mem->ApproximateMemoryUsage() > options_.write_buffer_size) { + status = WriteLevel0Table(mem, edit, NULL, NULL); + if (!status.ok()) { + // Reflect errors immediately so that conditions like full + // file-systems cause the DB::Open() to fail. + break; + } + mem->Unref(); + mem = NULL; + } + } + + if (status.ok() && mem != NULL) { + status = WriteLevel0Table(mem, edit, NULL, NULL); + // Reflect errors immediately so that conditions like full + // file-systems cause the DB::Open() to fail. + } + + if (mem != NULL) mem->Unref(); + delete file; + return status; +} + +Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit, + Version* base, uint64_t* number) { + mutex_.AssertHeld(); + const uint64_t start_micros = env_->NowMicros(); + FileMetaData meta; + meta.number = versions_->NewFileNumber(); + if (number) { + *number = meta.number; + } + pending_outputs_.insert(meta.number); + Iterator* iter = mem->NewIterator(); + Log(options_.info_log, "Level-0 table #%llu: started", + (unsigned long long) meta.number); + + Status s; + { + mutex_.Unlock(); + s = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta); + mutex_.Lock(); + } + + Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s", + (unsigned long long) meta.number, + (unsigned long long) meta.file_size, + s.ToString().c_str()); + delete iter; + + // Note that if file_size is zero, the file has been deleted and + // should not be added to the manifest. + int level = 0; + if (s.ok() && meta.file_size > 0) { + const Slice min_user_key = meta.smallest.user_key(); + const Slice max_user_key = meta.largest.user_key(); + if (base != NULL) { + level = base->PickLevelForMemTableOutput(min_user_key, max_user_key); + while (level > 0 && levels_locked_[level]) { + --level; + } + } + edit->AddFile(level, meta.number, meta.file_size, + meta.smallest, meta.largest); + } + + CompactionStats stats; + stats.micros = env_->NowMicros() - start_micros; + stats.bytes_written = meta.file_size; + stats_[level].Add(stats); + return s; +} + +void DBImpl::CompactMemTableThread() { + MutexLock l(&mutex_); + while (!shutting_down_.Acquire_Load() && !allow_background_activity_) { + bg_memtable_cv_.Wait(); + } + while (!shutting_down_.Acquire_Load()) { + while (!shutting_down_.Acquire_Load() && imm_ == NULL) { + bg_memtable_cv_.Wait(); + } + if (shutting_down_.Acquire_Load()) { + break; + } + + // Save the contents of the memtable as a new Table + VersionEdit edit; + Version* base = versions_->current(); + base->Ref(); + uint64_t number; + Status s = WriteLevel0Table(imm_, &edit, base, &number); + base->Unref(); base = NULL; + + if (s.ok() && shutting_down_.Acquire_Load()) { + s = Status::IOError("Deleting DB during memtable compaction"); + } + + // Replace immutable memtable with the generated Table + if (s.ok()) { + edit.SetPrevLogNumber(0); + edit.SetLogNumber(logfile_number_); // Earlier logs no longer needed + s = versions_->LogAndApply(&edit, &mutex_, &bg_log_cv_, &bg_log_occupied_); + } + + pending_outputs_.erase(number); + + if (s.ok()) { + // Commit to the new state + imm_->Unref(); + imm_ = NULL; + has_imm_.Release_Store(NULL); + bg_fg_cv_.SignalAll(); + bg_compaction_cv_.Signal(); + DeleteObsoleteFiles(); + } + + if (!shutting_down_.Acquire_Load() && !s.ok()) { + // Wait a little bit before retrying background compaction in + // case this is an environmental problem and we do not want to + // chew up resources for failed compactions for the duration of + // the problem. + bg_fg_cv_.SignalAll(); // In case a waiter can proceed despite the error + Log(options_.info_log, "Waiting after memtable compaction error: %s", + s.ToString().c_str()); + mutex_.Unlock(); + env_->SleepForMicroseconds(1000000); + mutex_.Lock(); + } + + assert(config::kL0_SlowdownWritesTrigger > 0); + if (versions_->NumLevelFiles(0) >= config::kL0_SlowdownWritesTrigger - 1) { + bg_optimistic_trip_ = true; + bg_optimistic_cv_.Signal(); + } + } + Log(options_.info_log, "cleaning up CompactMemTableThread"); + num_bg_threads_ -= 1; + bg_fg_cv_.SignalAll(); +} + +void DBImpl::CompactRange(const Slice* begin, const Slice* end) { + int max_level_with_files = 1; + { + MutexLock l(&mutex_); + Version* base = versions_->current(); + for (int level = 1; level < config::kNumLevels; level++) { + if (base->OverlapInLevel(level, begin, end)) { + max_level_with_files = level; + } + } + } + TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap + for (int level = 0; level < max_level_with_files; level++) { + TEST_CompactRange(level, begin, end); + } +} + +void DBImpl::TEST_CompactRange(int level, const Slice* begin,const Slice* end) { + assert(level >= 0); + assert(level + 1 < config::kNumLevels); + + InternalKey begin_storage, end_storage; + + ManualCompaction manual; + manual.level = level; + manual.done = false; + if (begin == NULL) { + manual.begin = NULL; + } else { + begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek); + manual.begin = &begin_storage; + } + if (end == NULL) { + manual.end = NULL; + } else { + end_storage = InternalKey(*end, 0, static_cast(0)); + manual.end = &end_storage; + } + + MutexLock l(&mutex_); + while (!manual.done) { + while (manual_compaction_ != NULL) { + bg_fg_cv_.Wait(); + } + manual_compaction_ = &manual; + bg_compaction_cv_.Signal(); + bg_memtable_cv_.Signal(); + while (manual_compaction_ == &manual) { + bg_fg_cv_.Wait(); + } + } +} + +Status DBImpl::TEST_CompactMemTable() { + // NULL batch means just wait for earlier writes to be done + Status s = Write(WriteOptions(), NULL); + if (s.ok()) { + // Wait until the compaction completes + MutexLock l(&mutex_); + while (imm_ != NULL && bg_error_.ok()) { + bg_fg_cv_.Wait(); + } + if (imm_ != NULL) { + s = bg_error_; + } + } + return s; +} + +void DBImpl::CompactLevelThread() { + MutexLock l(&mutex_); + while (!shutting_down_.Acquire_Load() && !allow_background_activity_) { + bg_compaction_cv_.Wait(); + } + while (!shutting_down_.Acquire_Load()) { + while (!shutting_down_.Acquire_Load() && + manual_compaction_ == NULL && + !versions_->NeedsCompaction(levels_locked_)) { + bg_compaction_cv_.Wait(); + } + if (shutting_down_.Acquire_Load()) { + break; + } + + assert(manual_compaction_ == NULL || num_bg_threads_ == 3); + Status s = BackgroundCompaction(); + bg_fg_cv_.SignalAll(); // before the backoff In case a waiter + // can proceed despite the error + + if (s.ok()) { + // Success + consecutive_compaction_errors_ = 0; + } else if (shutting_down_.Acquire_Load()) { + // Error most likely due to shutdown; do not wait + } else { + // Wait a little bit before retrying background compaction in + // case this is an environmental problem and we do not want to + // chew up resources for failed compactions for the duration of + // the problem. + Log(options_.info_log, "Waiting after background compaction error: %s", + s.ToString().c_str()); + mutex_.Unlock(); + ++consecutive_compaction_errors_; + int seconds_to_sleep = 1; + for (int i = 0; i < 3 && i < consecutive_compaction_errors_ - 1; ++i) { + seconds_to_sleep *= 2; + } + env_->SleepForMicroseconds(seconds_to_sleep * 1000000); + mutex_.Lock(); + } + } + Log(options_.info_log, "cleaning up CompactLevelThread"); + num_bg_threads_ -= 1; + bg_fg_cv_.SignalAll(); +} + +Status DBImpl::BackgroundCompaction() { + mutex_.AssertHeld(); + Compaction* c = NULL; + bool is_manual = (manual_compaction_ != NULL); + InternalKey manual_end; + if (is_manual) { + ManualCompaction* m = manual_compaction_; + c = versions_->CompactRange(m->level, m->begin, m->end); + m->done = (c == NULL); + if (c != NULL) { + manual_end = c->input(0, c->num_input_files(0) - 1)->largest; + } + Log(options_.info_log, + "Manual compaction at level-%d from %s .. %s; will stop at %s\n", + m->level, + (m->begin ? m->begin->DebugString().c_str() : "(begin)"), + (m->end ? m->end->DebugString().c_str() : "(end)"), + (m->done ? "(end)" : manual_end.DebugString().c_str())); + } else { + int level = versions_->PickCompactionLevel(levels_locked_); + if (level != config::kNumLevels) { + c = versions_->PickCompaction(level); + } + if (c) { + assert(!levels_locked_[c->level() + 0]); + assert(!levels_locked_[c->level() + 1]); + levels_locked_[c->level() + 0] = true; + levels_locked_[c->level() + 1] = true; + } + } + + Status status; + + if (c == NULL) { + // Nothing to do + } else if (!is_manual && c->IsTrivialMove() && c->level() > 0) { + // Move file to next level + for (size_t i = 0; i < c->num_input_files(0); ++i) { + FileMetaData* f = c->input(0, i); + c->edit()->DeleteFile(c->level(), f->number); + c->edit()->AddFile(c->level() + 1, f->number, f->file_size, + f->smallest, f->largest); + } + status = versions_->LogAndApply(c->edit(), &mutex_, &bg_log_cv_, &bg_log_occupied_); + VersionSet::LevelSummaryStorage tmp; + for (size_t i = 0; i < c->num_input_files(0); ++i) { + FileMetaData* f = c->input(0, i); + Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n", + static_cast(f->number), + c->level() + 1, + static_cast(f->file_size), + status.ToString().c_str(), + versions_->LevelSummary(&tmp)); + } + } else { + CompactionState* compact = new CompactionState(c); + status = DoCompactionWork(compact); + CleanupCompaction(compact); + c->ReleaseInputs(); + DeleteObsoleteFiles(); + } + + if (c) { + levels_locked_[c->level() + 0] = false; + levels_locked_[c->level() + 1] = false; + delete c; + } + + if (status.ok()) { + // Done + } else if (shutting_down_.Acquire_Load()) { + // Ignore compaction errors found during shutting down + } else { + Log(options_.info_log, + "Compaction error: %s", status.ToString().c_str()); + if (options_.paranoid_checks && bg_error_.ok()) { + bg_error_ = status; + } + } + + if (is_manual) { + ManualCompaction* m = manual_compaction_; + if (!status.ok()) { + m->done = true; + } + if (!m->done) { + // We only compacted part of the requested range. Update *m + // to the range that is left to be compacted. + m->tmp_storage = manual_end; + m->begin = &m->tmp_storage; + } + manual_compaction_ = NULL; + } + return status; +} + +void DBImpl::CompactOptimisticThread() { + MutexLock l(&mutex_); + while (!shutting_down_.Acquire_Load() && !allow_background_activity_) { + bg_optimistic_cv_.Wait(); + } + while (!shutting_down_.Acquire_Load()) { + while (!shutting_down_.Acquire_Load() && !bg_optimistic_trip_) { + bg_optimistic_cv_.Wait(); + } + if (shutting_down_.Acquire_Load()) { + break; + } + bg_optimistic_trip_ = false; + Status s = OptimisticCompaction(); + + if (!shutting_down_.Acquire_Load() && !s.ok()) { + // Wait a little bit before retrying background compaction in + // case this is an environmental problem and we do not want to + // chew up resources for failed compactions for the duration of + // the problem. + Log(options_.info_log, "Waiting after optimistic compaction error: %s", + s.ToString().c_str()); + mutex_.Unlock(); + env_->SleepForMicroseconds(1000000); + mutex_.Lock(); + } + } + Log(options_.info_log, "cleaning up OptimisticCompactThread"); + num_bg_threads_ -= 1; + bg_fg_cv_.SignalAll(); +} + +Status DBImpl::OptimisticCompaction() { + mutex_.AssertHeld(); + Log(options_.info_log, "Optimistic compaction started"); + bool did_compaction = true; + uint64_t iters = 0; + while (did_compaction) { + ++iters; + did_compaction = false; + Compaction* c = NULL; + for (size_t level = 1; level + 1 < config::kNumLevels; ++level) { + if (levels_locked_[level] || levels_locked_[level + 1]) { + continue; + } + Compaction* tmp = versions_->PickCompaction(level); + if (tmp && tmp->IsTrivialMove()) { + if (c) { + delete c; + } + c = tmp; + break; + } else if (c && tmp && c->ratio() < tmp->ratio()) { + delete c; + c = tmp; + } else if (!c) { + c = tmp; + } else { + delete tmp; + } + } + if (!c) { + continue; + } + if (!c->IsTrivialMove() && c->ratio() < .90) { + delete c; + continue; + } + assert(!levels_locked_[c->level() + 0]); + assert(!levels_locked_[c->level() + 1]); + levels_locked_[c->level() + 0] = true; + levels_locked_[c->level() + 1] = true; + + did_compaction = true; + Status status; + + if (c->IsTrivialMove() && c->level() > 0) { + // Move file to next level + for (size_t i = 0; i < c->num_input_files(0); ++i) { + FileMetaData* f = c->input(0, i); + c->edit()->DeleteFile(c->level(), f->number); + c->edit()->AddFile(c->level() + 1, f->number, f->file_size, + f->smallest, f->largest); + } + status = versions_->LogAndApply(c->edit(), &mutex_, &bg_log_cv_, &bg_log_occupied_); + VersionSet::LevelSummaryStorage tmp; + for (size_t i = 0; i < c->num_input_files(0); ++i) { + FileMetaData* f = c->input(0, i); + Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n", + static_cast(f->number), + c->level() + 1, + static_cast(f->file_size), + status.ToString().c_str(), + versions_->LevelSummary(&tmp)); + } + } else { + CompactionState* compact = new CompactionState(c); + status = DoCompactionWork(compact); + CleanupCompaction(compact); + c->ReleaseInputs(); + DeleteObsoleteFiles(); + } + + levels_locked_[c->level() + 0] = false; + levels_locked_[c->level() + 1] = false; + delete c; + + if (status.ok()) { + // Done + } else if (shutting_down_.Acquire_Load()) { + // Ignore compaction errors found during shutting down + break; + } else { + Log(options_.info_log, + "Compaction error: %s", status.ToString().c_str()); + if (options_.paranoid_checks && bg_error_.ok()) { + bg_error_ = status; + } + break; + } + } + Log(options_.info_log, "Optimistic compaction ended after %ld iterations", iters); + return Status::OK(); +} + +void DBImpl::CleanupCompaction(CompactionState* compact) { + mutex_.AssertHeld(); + if (compact->builder != NULL) { + // May happen if we get a shutdown call in the middle of compaction + compact->builder->Abandon(); + delete compact->builder; + } else { + assert(compact->outfile == NULL); + } + delete compact->outfile; + for (size_t i = 0; i < compact->outputs.size(); i++) { + const CompactionState::Output& out = compact->outputs[i]; + pending_outputs_.erase(out.number); + } + delete compact; +} + +Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) { + assert(compact != NULL); + assert(compact->builder == NULL); + uint64_t file_number; + { + mutex_.Lock(); + file_number = versions_->NewFileNumber(); + pending_outputs_.insert(file_number); + CompactionState::Output out; + out.number = file_number; + out.smallest.Clear(); + out.largest.Clear(); + compact->outputs.push_back(out); + mutex_.Unlock(); + } + + // Make the output file + std::string fname = TableFileName(dbname_, file_number); + Status s = env_->NewWritableFile(fname, &compact->outfile); + if (s.ok()) { + compact->builder = new TableBuilder(options_, compact->outfile); + } + return s; +} + +Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, + Iterator* input) { + assert(compact != NULL); + assert(compact->outfile != NULL); + assert(compact->builder != NULL); + + const uint64_t output_number = compact->current_output()->number; + assert(output_number != 0); + + // Check for iterator errors + Status s = input->status(); + const uint64_t current_entries = compact->builder->NumEntries(); + if (s.ok()) { + s = compact->builder->Finish(); + } else { + compact->builder->Abandon(); + } + const uint64_t current_bytes = compact->builder->FileSize(); + compact->current_output()->file_size = current_bytes; + compact->total_bytes += current_bytes; + delete compact->builder; + compact->builder = NULL; + + // Finish and check for file errors + if (s.ok()) { + s = compact->outfile->Sync(); + } + if (s.ok()) { + s = compact->outfile->Close(); + } + delete compact->outfile; + compact->outfile = NULL; + + if (s.ok() && current_entries > 0) { + // Verify that the table is usable + Iterator* iter = table_cache_->NewIterator(ReadOptions(), + output_number, + current_bytes); + s = iter->status(); + delete iter; + if (s.ok()) { + Log(options_.info_log, + "Generated table #%llu: %lld keys, %lld bytes", + (unsigned long long) output_number, + (unsigned long long) current_entries, + (unsigned long long) current_bytes); + } + } + return s; +} + + +Status DBImpl::InstallCompactionResults(CompactionState* compact) { + mutex_.AssertHeld(); + Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes", + compact->compaction->num_input_files(0), + compact->compaction->level(), + compact->compaction->num_input_files(1), + compact->compaction->level() + 1, + static_cast(compact->total_bytes)); + + // Add compaction outputs + compact->compaction->AddInputDeletions(compact->compaction->edit()); + const int level = compact->compaction->level(); + for (size_t i = 0; i < compact->outputs.size(); i++) { + const CompactionState::Output& out = compact->outputs[i]; + compact->compaction->edit()->AddFile( + level + 1, + out.number, out.file_size, out.smallest, out.largest); + } + return versions_->LogAndApply(compact->compaction->edit(), &mutex_, &bg_log_cv_, &bg_log_occupied_); +} + +Status DBImpl::DoCompactionWork(CompactionState* compact) { + const uint64_t start_micros = env_->NowMicros(); + int64_t imm_micros = 0; // Micros spent doing imm_ compactions + + Log(options_.info_log, "Compacting %d@%d + %d@%d files", + compact->compaction->num_input_files(0), + compact->compaction->level(), + compact->compaction->num_input_files(1), + compact->compaction->level() + 1); + + assert(versions_->NumLevelFiles(compact->compaction->level()) > 0); + assert(compact->builder == NULL); + assert(compact->outfile == NULL); + if (snapshots_.empty()) { + compact->smallest_snapshot = versions_->LastSequence(); + } else { + compact->smallest_snapshot = snapshots_.oldest()->number_; + } + + // Release mutex while we're actually doing the compaction work + mutex_.Unlock(); + + Iterator* input = versions_->MakeInputIterator(compact->compaction); + input->SeekToFirst(); + Status status; + ParsedInternalKey ikey; + std::string current_user_key; + bool has_current_user_key = false; + SequenceNumber last_sequence_for_key = kMaxSequenceNumber; + for (; input->Valid() && !shutting_down_.Acquire_Load(); ) { + Slice key = input->key(); + // Handle key/value, add to state, etc. + bool drop = false; + if (!ParseInternalKey(key, &ikey)) { + // Do not hide error keys + current_user_key.clear(); + has_current_user_key = false; + last_sequence_for_key = kMaxSequenceNumber; + } else { + if (!has_current_user_key || + user_comparator()->Compare(ikey.user_key, + Slice(current_user_key)) != 0) { + // First occurrence of this user key + current_user_key.assign(ikey.user_key.data(), ikey.user_key.size()); + has_current_user_key = true; + last_sequence_for_key = kMaxSequenceNumber; + } + + if (last_sequence_for_key <= compact->smallest_snapshot) { + // Hidden by an newer entry for same user key + drop = true; // (A) + } else if (ikey.type == kTypeDeletion && + ikey.sequence <= compact->smallest_snapshot && + compact->compaction->IsBaseLevelForKey(ikey.user_key)) { + // For this user key: + // (1) there is no data in higher levels + // (2) data in lower levels will have larger sequence numbers + // (3) data in layers that are being compacted here and have + // smaller sequence numbers will be dropped in the next + // few iterations of this loop (by rule (A) above). + // Therefore this deletion marker is obsolete and can be dropped. + drop = true; + } + + last_sequence_for_key = ikey.sequence; + } + + if (!drop) { + // Open output file if necessary + if (compact->builder == NULL) { + status = OpenCompactionOutputFile(compact); + if (!status.ok()) { + break; + } + } + if (compact->builder->NumEntries() == 0) { + compact->current_output()->smallest.DecodeFrom(key); + } + compact->current_output()->largest.DecodeFrom(key); + compact->builder->Add(key, input->value()); + + // Close output file if it is big enough + if (compact->builder->FileSize() >= + compact->compaction->MaxOutputFileSize()) { + status = FinishCompactionOutputFile(compact, input); + if (!status.ok()) { + break; + } + } + } + + input->Next(); + } + + if (status.ok() && shutting_down_.Acquire_Load()) { + status = Status::IOError("Deleting DB during compaction"); + } + if (status.ok() && compact->builder != NULL) { + status = FinishCompactionOutputFile(compact, input); + } + if (status.ok()) { + status = input->status(); + } + delete input; + input = NULL; + + CompactionStats stats; + stats.micros = env_->NowMicros() - start_micros - imm_micros; + for (int which = 0; which < 2; which++) { + for (int i = 0; i < compact->compaction->num_input_files(which); i++) { + stats.bytes_read += compact->compaction->input(which, i)->file_size; + } + } + for (size_t i = 0; i < compact->outputs.size(); i++) { + stats.bytes_written += compact->outputs[i].file_size; + } + + mutex_.Lock(); + stats_[compact->compaction->level() + 1].Add(stats); + + if (status.ok()) { + status = InstallCompactionResults(compact); + } + VersionSet::LevelSummaryStorage tmp; + Log(options_.info_log, + "compacted to: %s", versions_->LevelSummary(&tmp)); + return status; +} + +namespace { +struct IterState { + port::Mutex* mu; + Version* version; + MemTable* mem; + MemTable* imm; +}; + +static void CleanupIteratorState(void* arg1, void* arg2) { + IterState* state = reinterpret_cast(arg1); + state->mu->Lock(); + state->mem->Unref(); + if (state->imm != NULL) state->imm->Unref(); + state->version->Unref(); + state->mu->Unlock(); + delete state; +} +} // namespace + +Iterator* DBImpl::NewInternalIterator(const ReadOptions& options, + SequenceNumber* latest_snapshot) { + IterState* cleanup = new IterState; + mutex_.Lock(); + *latest_snapshot = versions_->LastSequence(); + + // Collect together all needed child iterators + std::vector list; + list.push_back(mem_->NewIterator()); + mem_->Ref(); + if (imm_ != NULL) { + list.push_back(imm_->NewIterator()); + imm_->Ref(); + } + versions_->current()->AddIterators(options, &list); + Iterator* internal_iter = + NewMergingIterator(&internal_comparator_, &list[0], list.size()); + versions_->current()->Ref(); + + cleanup->mu = &mutex_; + cleanup->mem = mem_; + cleanup->imm = imm_; + cleanup->version = versions_->current(); + internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, NULL); + + mutex_.Unlock(); + return internal_iter; +} + +Iterator* DBImpl::TEST_NewInternalIterator() { + SequenceNumber ignored; + return NewInternalIterator(ReadOptions(), &ignored); +} + +int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() { + MutexLock l(&mutex_); + return versions_->MaxNextLevelOverlappingBytes(); +} + +Status DBImpl::Get(const ReadOptions& options, + const Slice& key, + std::string* value) { + Status s; + MutexLock l(&mutex_); + SequenceNumber snapshot; + if (options.snapshot != NULL) { + snapshot = reinterpret_cast(options.snapshot)->number_; + } else { + snapshot = versions_->LastSequence(); + } + + MemTable* mem = mem_; + MemTable* imm = imm_; + Version* current = versions_->current(); + mem->Ref(); + if (imm != NULL) imm->Ref(); + current->Ref(); + + bool have_stat_update = false; + Version::GetStats stats; + + // Unlock while reading from files and memtables + { + mutex_.Unlock(); + // First look in the memtable, then in the immutable memtable (if any). + LookupKey lkey(key, snapshot); + if (mem->Get(lkey, value, &s)) { + // Done + } else if (imm != NULL && imm->Get(lkey, value, &s)) { + // Done + } else { + s = current->Get(options, lkey, value, &stats); + have_stat_update = true; + } + mutex_.Lock(); + } + + mem->Unref(); + if (imm != NULL) imm->Unref(); + current->Unref(); + return s; +} + +Iterator* DBImpl::NewIterator(const ReadOptions& options) { + SequenceNumber latest_snapshot; + Iterator* internal_iter = NewInternalIterator(options, &latest_snapshot); + return NewDBIterator( + &dbname_, env_, user_comparator(), internal_iter, + (options.snapshot != NULL + ? reinterpret_cast(options.snapshot)->number_ + : latest_snapshot)); +} + +const Snapshot* DBImpl::GetSnapshot() { + MutexLock l(&mutex_); + return snapshots_.New(versions_->LastSequence()); +} + +void DBImpl::ReleaseSnapshot(const Snapshot* s) { + MutexLock l(&mutex_); + snapshots_.Delete(reinterpret_cast(s)); +} + +// Convenience methods +Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val) { + return DB::Put(o, key, val); +} + +Status DBImpl::Delete(const WriteOptions& options, const Slice& key) { + return DB::Delete(options, key); +} + +// Information kept for every waiting writer +struct DBImpl::Writer { + port::Mutex mtx; + port::CondVar cv; + bool linked; + Writer* next; + uint64_t start_sequence; + uint64_t end_sequence; + WritableFile* logfile; + log::Writer* log; + MemTable* mem; + WritableFile* old_logfile; + log::Writer* old_log; + + explicit Writer() + : mtx(), + cv(&mtx), + linked(false), + next(NULL), + start_sequence(0), + end_sequence(0), + logfile(NULL), + log(NULL), + mem(NULL), + old_logfile(NULL), + old_log(NULL) { + } + ~Writer() throw () { + } +}; + +Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) { + Writer w; + Status s; + s = SequenceWriteBegin(&w, updates); + + if (s.ok() && updates != NULL) { // NULL batch is for compactions + WriteBatchInternal::SetSequence(updates, w.start_sequence); + + // Add to log and apply to memtable. We do this without holding the lock + // because both the log and the memtable are safe for concurrent access. + // The synchronization with readers occurs with SequenceWriteEnd. + s = w.log->AddRecord(WriteBatchInternal::Contents(updates)); + if (s.ok()) { + s = WriteBatchInternal::InsertInto(updates, w.mem); + } + } + + if (s.ok() && options.sync) { + s = w.logfile->Sync(); + } + + SequenceWriteEnd(&w); + return s; +} + +Status DBImpl::SequenceWriteBegin(Writer* w, WriteBatch* updates) { + Status s; + MutexLock l(&mutex_); + bool force = updates == NULL; + bool allow_delay = !force; + w->old_log = NULL; + w->old_logfile = NULL; + + while (true) { + if (!bg_error_.ok()) { + // Yield previous error + s = bg_error_; + break; + } else if (!force && + (mem_->ApproximateMemoryUsage() <= options_.write_buffer_size)) { + // There is room in current memtable + // Note that this is a sloppy check. We can overfill a memtable by the + // amount of concurrently written data. + break; + } else if (imm_ != NULL) { + // We have filled up the current memtable, but the previous + // one is still being compacted, so we wait. + bg_compaction_cv_.Signal(); + bg_memtable_cv_.Signal(); + bg_fg_cv_.Wait(); + } else { + // Attempt to switch to a new memtable and trigger compaction of old + assert(versions_->PrevLogNumber() == 0); + uint64_t new_log_number = versions_->NewFileNumber(); + WritableFile* lfile = NULL; + s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile); + if (!s.ok()) { + // Avoid chewing through file number space in a tight loop. + versions_->ReuseFileNumber(new_log_number); + break; + } + w->old_log = log_; + w->old_logfile = logfile_; + logfile_ = lfile; + logfile_number_ = new_log_number; + log_ = new log::Writer(lfile); + imm_ = mem_; + has_imm_.Release_Store(imm_); + mem_ = new MemTable(internal_comparator_); + mem_->Ref(); + force = false; // Do not force another compaction if have room + break; + } + } + + if (s.ok()) { + w->linked = true; + w->next = NULL; + uint64_t diff = updates ? WriteBatchInternal::Count(updates) : 0; + uint64_t ticket = __sync_add_and_fetch(&writers_upper_, 1 + diff); + w->start_sequence = ticket - diff; + w->end_sequence = ticket; + w->logfile = logfile_; + w->log = log_; + w->mem = mem_; + w->mem->Ref(); + } + + return s; +} + +void DBImpl::SequenceWriteEnd(Writer* w) { + if (!w->linked) { + return; + } + + // wait until we are next + while (__sync_fetch_and_add(&writers_lower_, 0) < w->start_sequence) + ; + + // swizzle state to make ours visible + { + MutexLock l(&mutex_); + versions_->SetLastSequence(w->end_sequence); + } + + // signal the next writer + __sync_fetch_and_add(&writers_lower_, 1 + w->end_sequence - w->start_sequence); + + // must do in order: log, logfile + if (w->old_log) { + assert(w->old_logfile); + delete w->old_log; + delete w->old_logfile; + bg_memtable_cv_.Signal(); + } + + // safe because Unref is synchronized internally + if (w->mem) { + w->mem->Unref(); + } +} + +bool DBImpl::GetProperty(const Slice& property, std::string* value) { + value->clear(); + + MutexLock l(&mutex_); + Slice in = property; + Slice prefix("leveldb."); + if (!in.starts_with(prefix)) return false; + in.remove_prefix(prefix.size()); + + if (in.starts_with("num-files-at-level")) { + in.remove_prefix(strlen("num-files-at-level")); + uint64_t level; + bool ok = ConsumeDecimalNumber(&in, &level) && in.empty(); + if (!ok || level >= config::kNumLevels) { + return false; + } else { + char buf[100]; + snprintf(buf, sizeof(buf), "%d", + versions_->NumLevelFiles(static_cast(level))); + *value = buf; + return true; + } + } else if (in == "stats") { + char buf[200]; + snprintf(buf, sizeof(buf), + " Compactions\n" + "Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n" + "--------------------------------------------------\n" + ); + value->append(buf); + for (int level = 0; level < config::kNumLevels; level++) { + int files = versions_->NumLevelFiles(level); + if (stats_[level].micros > 0 || files > 0) { + snprintf( + buf, sizeof(buf), + "%3d %8d %8.0f %9.0f %8.0f %9.0f\n", + level, + files, + versions_->NumLevelBytes(level) / 1048576.0, + stats_[level].micros / 1e6, + stats_[level].bytes_read / 1048576.0, + stats_[level].bytes_written / 1048576.0); + value->append(buf); + } + } + return true; + } else if (in == "sstables") { + *value = versions_->current()->DebugString(); + return true; + } + + return false; +} + +void DBImpl::GetApproximateSizes( + const Range* range, int n, + uint64_t* sizes) { + // TODO(opt): better implementation + Version* v; + { + MutexLock l(&mutex_); + versions_->current()->Ref(); + v = versions_->current(); + } + + for (int i = 0; i < n; i++) { + // Convert user_key into a corresponding internal key. + InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek); + uint64_t start = versions_->ApproximateOffsetOf(v, k1); + uint64_t limit = versions_->ApproximateOffsetOf(v, k2); + sizes[i] = (limit >= start ? limit - start : 0); + } + + { + MutexLock l(&mutex_); + v->Unref(); + } +} + +// Default implementations of convenience methods that subclasses of DB +// can call if they wish +Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) { + WriteBatch batch; + batch.Put(key, value); + return Write(opt, &batch); +} + +Status DB::Delete(const WriteOptions& opt, const Slice& key) { + WriteBatch batch; + batch.Delete(key); + return Write(opt, &batch); +} + +DB::~DB() { } + +Status DB::Open(const Options& options, const std::string& dbname, + DB** dbptr) { + *dbptr = NULL; + + DBImpl* impl = new DBImpl(options, dbname); + impl->mutex_.Lock(); + VersionEdit edit; + Status s = impl->Recover(&edit); // Handles create_if_missing, error_if_exists + if (s.ok()) { + uint64_t new_log_number = impl->versions_->NewFileNumber(); + WritableFile* lfile; + s = options.env->NewWritableFile(LogFileName(dbname, new_log_number), + &lfile); + if (s.ok()) { + edit.SetLogNumber(new_log_number); + impl->logfile_ = lfile; + impl->logfile_number_ = new_log_number; + impl->log_ = new log::Writer(lfile); + s = impl->versions_->LogAndApply(&edit, &impl->mutex_, &impl->bg_log_cv_, &impl->bg_log_occupied_); + } + if (s.ok()) { + impl->DeleteObsoleteFiles(); + impl->bg_optimistic_cv_.Signal(); + impl->bg_compaction_cv_.Signal(); + impl->bg_memtable_cv_.Signal(); + } + } + impl->pending_outputs_.clear(); + impl->allow_background_activity_ = true; + impl->bg_optimistic_cv_.SignalAll(); + impl->bg_compaction_cv_.SignalAll(); + impl->bg_memtable_cv_.SignalAll(); + impl->mutex_.Unlock(); + if (s.ok()) { + *dbptr = impl; + } else { + delete impl; + } + impl->writers_upper_ = impl->versions_->LastSequence(); + impl->writers_lower_ = impl->writers_upper_ + 1; + return s; +} + +Snapshot::~Snapshot() { +} + +Status DestroyDB(const std::string& dbname, const Options& options) { + Env* env = options.env; + std::vector filenames; + // Ignore error in case directory does not exist + env->GetChildren(dbname, &filenames); + if (filenames.empty()) { + return Status::OK(); + } + + FileLock* lock; + const std::string lockname = LockFileName(dbname); + Status result = env->LockFile(lockname, &lock); + if (result.ok()) { + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && + type != kDBLockFile) { // Lock file will be deleted at end + Status del = env->DeleteFile(dbname + "/" + filenames[i]); + if (result.ok() && !del.ok()) { + result = del; + } + } + } + env->UnlockFile(lock); // Ignore error since state is already gone + env->DeleteFile(lockname); + env->DeleteDir(dbname); // Ignore error in case dir contains other files + } + return result; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/db_impl.h b/Subtrees/hyperleveldb/db/db_impl.h new file mode 100644 index 0000000000..49eb9b1f27 --- /dev/null +++ b/Subtrees/hyperleveldb/db/db_impl.h @@ -0,0 +1,223 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_DB_IMPL_H_ +#define STORAGE_HYPERLEVELDB_DB_DB_IMPL_H_ + +#include +#include +#include "dbformat.h" +#include "log_writer.h" +#include "snapshot.h" +#include "../hyperleveldb/db.h" +#include "../hyperleveldb/env.h" +#include "../port/port.h" +#include "../port/thread_annotations.h" + +namespace hyperleveldb { + +class MemTable; +class TableCache; +class Version; +class VersionEdit; +class VersionSet; + +class DBImpl : public DB { + public: + DBImpl(const Options& options, const std::string& dbname); + virtual ~DBImpl(); + + // Implementations of the DB interface + virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value); + virtual Status Delete(const WriteOptions&, const Slice& key); + virtual Status Write(const WriteOptions& options, WriteBatch* updates); + virtual Status Get(const ReadOptions& options, + const Slice& key, + std::string* value); + virtual Iterator* NewIterator(const ReadOptions&); + virtual const Snapshot* GetSnapshot(); + virtual void ReleaseSnapshot(const Snapshot* snapshot); + virtual bool GetProperty(const Slice& property, std::string* value); + virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes); + virtual void CompactRange(const Slice* begin, const Slice* end); + + // Extra methods (for testing) that are not in the public DB interface + + // Compact any files in the named level that overlap [*begin,*end] + void TEST_CompactRange(int level, const Slice* begin, const Slice* end); + + // Force current memtable contents to be compacted. + Status TEST_CompactMemTable(); + + // Return an internal iterator over the current state of the database. + // The keys of this iterator are internal keys (see format.h). + // The returned iterator should be deleted when no longer needed. + Iterator* TEST_NewInternalIterator(); + + // Return the maximum overlapping data (in bytes) at next level for any + // file at a level >= 1. + int64_t TEST_MaxNextLevelOverlappingBytes(); + + private: + friend class DB; + struct CompactionState; + struct Writer; + + Iterator* NewInternalIterator(const ReadOptions&, + SequenceNumber* latest_snapshot); + + Status NewDB(); + + // Recover the descriptor from persistent storage. May do a significant + // amount of work to recover recently logged updates. Any changes to + // be made to the descriptor are added to *edit. + Status Recover(VersionEdit* edit) EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + void MaybeIgnoreError(Status* s) const; + + // Delete any unneeded files and stale in-memory entries. + void DeleteObsoleteFiles(); + + // A background thread to compact the in-memory write buffer to disk. + // Switches to a new log-file/memtable and writes a new descriptor iff + // successful. + static void CompactMemTableWrapper(void* db) + { reinterpret_cast(db)->CompactMemTableThread(); } + void CompactMemTableThread(); + + Status RecoverLogFile(uint64_t log_number, + VersionEdit* edit, + SequenceNumber* max_sequence) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base, uint64_t* number) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + Status SequenceWriteBegin(Writer* w, WriteBatch* updates) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void SequenceWriteEnd(Writer* w) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + static void CompactLevelWrapper(void* db) + { reinterpret_cast(db)->CompactLevelThread(); } + void CompactLevelThread(); + Status BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + static void CompactOptimisticWrapper(void* db) + { reinterpret_cast(db)->CompactOptimisticThread(); } + void CompactOptimisticThread(); + Status OptimisticCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + void CleanupCompaction(CompactionState* compact) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Status DoCompactionWork(CompactionState* compact) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Status OpenCompactionOutputFile(CompactionState* compact); + Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input); + Status InstallCompactionResults(CompactionState* compact) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Constant after construction + Env* const env_; + const InternalKeyComparator internal_comparator_; + const InternalFilterPolicy internal_filter_policy_; + const Options options_; // options_.comparator == &internal_comparator_ + bool owns_info_log_; + bool owns_cache_; + const std::string dbname_; + + // table_cache_ provides its own synchronization + TableCache* table_cache_; + + // Lock over the persistent DB state. Non-NULL iff successfully acquired. + FileLock* db_lock_; + + // State below is protected by mutex_ + port::Mutex mutex_; + port::AtomicPointer shutting_down_; + MemTable* mem_; + MemTable* imm_; // Memtable being compacted + port::AtomicPointer has_imm_; // So bg thread can detect non-NULL imm_ + WritableFile* logfile_; + uint64_t logfile_number_; + log::Writer* log_; + + // Synchronize writers + uint64_t __attribute__ ((aligned (8))) writers_lower_; + uint64_t __attribute__ ((aligned (8))) writers_upper_; + + SnapshotList snapshots_; + + // Set of table files to protect from deletion because they are + // part of ongoing compactions. + std::set pending_outputs_; + + bool allow_background_activity_; + bool levels_locked_[config::kNumLevels]; + int num_bg_threads_; + // Tell the foreground that background has done something of note + port::CondVar bg_fg_cv_; + // Communicate with compaction background thread + port::CondVar bg_compaction_cv_; + // Communicate with memtable->L0 background thread + port::CondVar bg_memtable_cv_; + // Communicate with the optimistic background thread + bool bg_optimistic_trip_; + port::CondVar bg_optimistic_cv_; + // Mutual exlusion protecting the LogAndApply func + port::CondVar bg_log_cv_; + bool bg_log_occupied_; + + // Information for a manual compaction + struct ManualCompaction { + int level; + bool done; + const InternalKey* begin; // NULL means beginning of key range + const InternalKey* end; // NULL means end of key range + InternalKey tmp_storage; // Used to keep track of compaction progress + }; + ManualCompaction* manual_compaction_; + + VersionSet* versions_; + + // Have we encountered a background error in paranoid mode? + Status bg_error_; + int consecutive_compaction_errors_; + + // Per level compaction stats. stats_[level] stores the stats for + // compactions that produced data for the specified "level". + struct CompactionStats { + int64_t micros; + int64_t bytes_read; + int64_t bytes_written; + + CompactionStats() : micros(0), bytes_read(0), bytes_written(0) { } + + void Add(const CompactionStats& c) { + this->micros += c.micros; + this->bytes_read += c.bytes_read; + this->bytes_written += c.bytes_written; + } + }; + CompactionStats stats_[config::kNumLevels]; + + // No copying allowed + DBImpl(const DBImpl&); + void operator=(const DBImpl&); + + const Comparator* user_comparator() const { + return internal_comparator_.user_comparator(); + } +}; + +// Sanitize db options. The caller should delete result.info_log if +// it is not equal to src.info_log. +extern Options SanitizeOptions(const std::string& db, + const InternalKeyComparator* icmp, + const InternalFilterPolicy* ipolicy, + const Options& src); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_DB_IMPL_H_ diff --git a/Subtrees/hyperleveldb/db/db_iter.cc b/Subtrees/hyperleveldb/db/db_iter.cc new file mode 100644 index 0000000000..3816e185d8 --- /dev/null +++ b/Subtrees/hyperleveldb/db/db_iter.cc @@ -0,0 +1,299 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db_iter.h" + +#include "filename.h" +#include "dbformat.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/iterator.h" +#include "../port/port.h" +#include "../util/logging.h" +#include "../util/mutexlock.h" + +namespace hyperleveldb { + +#if 0 +static void DumpInternalIter(Iterator* iter) { + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey k; + if (!ParseInternalKey(iter->key(), &k)) { + fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str()); + } else { + fprintf(stderr, "@ '%s'\n", k.DebugString().c_str()); + } + } +} +#endif + +namespace { + +// Memtables and sstables that make the DB representation contain +// (userkey,seq,type) => uservalue entries. DBIter +// combines multiple entries for the same userkey found in the DB +// representation into a single entry while accounting for sequence +// numbers, deletion markers, overwrites, etc. +class DBIter: public Iterator { + public: + // Which direction is the iterator currently moving? + // (1) When moving forward, the internal iterator is positioned at + // the exact entry that yields this->key(), this->value() + // (2) When moving backwards, the internal iterator is positioned + // just before all entries whose user key == this->key(). + enum Direction { + kForward, + kReverse + }; + + DBIter(const std::string* dbname, Env* env, + const Comparator* cmp, Iterator* iter, SequenceNumber s) + : dbname_(dbname), + env_(env), + user_comparator_(cmp), + iter_(iter), + sequence_(s), + direction_(kForward), + valid_(false) { + } + virtual ~DBIter() { + delete iter_; + } + virtual bool Valid() const { return valid_; } + virtual Slice key() const { + assert(valid_); + return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_; + } + virtual Slice value() const { + assert(valid_); + return (direction_ == kForward) ? iter_->value() : saved_value_; + } + virtual Status status() const { + if (status_.ok()) { + return iter_->status(); + } else { + return status_; + } + } + + virtual void Next(); + virtual void Prev(); + virtual void Seek(const Slice& target); + virtual void SeekToFirst(); + virtual void SeekToLast(); + + private: + void FindNextUserEntry(bool skipping, std::string* skip); + void FindPrevUserEntry(); + bool ParseKey(ParsedInternalKey* key); + + inline void SaveKey(const Slice& k, std::string* dst) { + dst->assign(k.data(), k.size()); + } + + inline void ClearSavedValue() { + if (saved_value_.capacity() > 1048576) { + std::string empty; + swap(empty, saved_value_); + } else { + saved_value_.clear(); + } + } + + const std::string* const dbname_; + Env* const env_; + const Comparator* const user_comparator_; + Iterator* const iter_; + SequenceNumber const sequence_; + + Status status_; + std::string saved_key_; // == current key when direction_==kReverse + std::string saved_value_; // == current raw value when direction_==kReverse + Direction direction_; + bool valid_; + + // No copying allowed + DBIter(const DBIter&); + void operator=(const DBIter&); +}; + +inline bool DBIter::ParseKey(ParsedInternalKey* ikey) { + if (!ParseInternalKey(iter_->key(), ikey)) { + status_ = Status::Corruption("corrupted internal key in DBIter"); + return false; + } else { + return true; + } +} + +void DBIter::Next() { + assert(valid_); + + if (direction_ == kReverse) { // Switch directions? + direction_ = kForward; + // iter_ is pointing just before the entries for this->key(), + // so advance into the range of entries for this->key() and then + // use the normal skipping code below. + if (!iter_->Valid()) { + iter_->SeekToFirst(); + } else { + iter_->Next(); + } + if (!iter_->Valid()) { + valid_ = false; + saved_key_.clear(); + return; + } + } + + // Temporarily use saved_key_ as storage for key to skip. + std::string* skip = &saved_key_; + SaveKey(ExtractUserKey(iter_->key()), skip); + FindNextUserEntry(true, skip); +} + +void DBIter::FindNextUserEntry(bool skipping, std::string* skip) { + // Loop until we hit an acceptable entry to yield + assert(iter_->Valid()); + assert(direction_ == kForward); + do { + ParsedInternalKey ikey; + if (ParseKey(&ikey) && ikey.sequence <= sequence_) { + switch (ikey.type) { + case kTypeDeletion: + // Arrange to skip all upcoming entries for this key since + // they are hidden by this deletion. + SaveKey(ikey.user_key, skip); + skipping = true; + break; + case kTypeValue: + if (skipping && + user_comparator_->Compare(ikey.user_key, *skip) <= 0) { + // Entry hidden + } else { + valid_ = true; + saved_key_.clear(); + return; + } + break; + } + } + iter_->Next(); + } while (iter_->Valid()); + saved_key_.clear(); + valid_ = false; +} + +void DBIter::Prev() { + assert(valid_); + + if (direction_ == kForward) { // Switch directions? + // iter_ is pointing at the current entry. Scan backwards until + // the key changes so we can use the normal reverse scanning code. + assert(iter_->Valid()); // Otherwise valid_ would have been false + SaveKey(ExtractUserKey(iter_->key()), &saved_key_); + while (true) { + iter_->Prev(); + if (!iter_->Valid()) { + valid_ = false; + saved_key_.clear(); + ClearSavedValue(); + return; + } + if (user_comparator_->Compare(ExtractUserKey(iter_->key()), + saved_key_) < 0) { + break; + } + } + direction_ = kReverse; + } + + FindPrevUserEntry(); +} + +void DBIter::FindPrevUserEntry() { + assert(direction_ == kReverse); + + ValueType value_type = kTypeDeletion; + if (iter_->Valid()) { + do { + ParsedInternalKey ikey; + if (ParseKey(&ikey) && ikey.sequence <= sequence_) { + if ((value_type != kTypeDeletion) && + user_comparator_->Compare(ikey.user_key, saved_key_) < 0) { + // We encountered a non-deleted value in entries for previous keys, + break; + } + value_type = ikey.type; + if (value_type == kTypeDeletion) { + saved_key_.clear(); + ClearSavedValue(); + } else { + Slice raw_value = iter_->value(); + if (saved_value_.capacity() > raw_value.size() + 1048576) { + std::string empty; + swap(empty, saved_value_); + } + SaveKey(ExtractUserKey(iter_->key()), &saved_key_); + saved_value_.assign(raw_value.data(), raw_value.size()); + } + } + iter_->Prev(); + } while (iter_->Valid()); + } + + if (value_type == kTypeDeletion) { + // End + valid_ = false; + saved_key_.clear(); + ClearSavedValue(); + direction_ = kForward; + } else { + valid_ = true; + } +} + +void DBIter::Seek(const Slice& target) { + direction_ = kForward; + ClearSavedValue(); + saved_key_.clear(); + AppendInternalKey( + &saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek)); + iter_->Seek(saved_key_); + if (iter_->Valid()) { + FindNextUserEntry(false, &saved_key_ /* temporary storage */); + } else { + valid_ = false; + } +} + +void DBIter::SeekToFirst() { + direction_ = kForward; + ClearSavedValue(); + iter_->SeekToFirst(); + if (iter_->Valid()) { + FindNextUserEntry(false, &saved_key_ /* temporary storage */); + } else { + valid_ = false; + } +} + +void DBIter::SeekToLast() { + direction_ = kReverse; + ClearSavedValue(); + iter_->SeekToLast(); + FindPrevUserEntry(); +} + +} // anonymous namespace + +Iterator* NewDBIterator( + const std::string* dbname, + Env* env, + const Comparator* user_key_comparator, + Iterator* internal_iter, + const SequenceNumber& sequence) { + return new DBIter(dbname, env, user_key_comparator, internal_iter, sequence); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/db_iter.h b/Subtrees/hyperleveldb/db/db_iter.h new file mode 100644 index 0000000000..5a0a64d321 --- /dev/null +++ b/Subtrees/hyperleveldb/db/db_iter.h @@ -0,0 +1,26 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_DB_ITER_H_ +#define STORAGE_HYPERLEVELDB_DB_DB_ITER_H_ + +#include +#include "../hyperleveldb/db.h" +#include "dbformat.h" + +namespace hyperleveldb { + +// Return a new iterator that converts internal keys (yielded by +// "*internal_iter") that were live at the specified "sequence" number +// into appropriate user keys. +extern Iterator* NewDBIterator( + const std::string* dbname, + Env* env, + const Comparator* user_key_comparator, + Iterator* internal_iter, + const SequenceNumber& sequence); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_DB_ITER_H_ diff --git a/Subtrees/hyperleveldb/db/db_test.cc b/Subtrees/hyperleveldb/db/db_test.cc new file mode 100644 index 0000000000..7e8b4a69c8 --- /dev/null +++ b/Subtrees/hyperleveldb/db/db_test.cc @@ -0,0 +1,2064 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/db.h" +#include "../hyperleveldb/filter_policy.h" +#include "db_impl.h" +#include "filename.h" +#include "version_set.h" +#include "write_batch_internal.h" +#include "../hyperleveldb/cache.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/table.h" +#include "../util/hash.h" +#include "../util/logging.h" +#include "../util/mutexlock.h" +#include "../util/testharness.h" +#include "../util/testutil.h" + +namespace hyperleveldb { + +static std::string RandomString(Random* rnd, int len) { + std::string r; + test::RandomString(rnd, len, &r); + return r; +} + +namespace { +class AtomicCounter { + private: + port::Mutex mu_; + int count_; + public: + AtomicCounter() : count_(0) { } + void Increment() { + IncrementBy(1); + } + void IncrementBy(int count) { + MutexLock l(&mu_); + count_ += count; + } + int Read() { + MutexLock l(&mu_); + return count_; + } + void Reset() { + MutexLock l(&mu_); + count_ = 0; + } +}; + +void DelayMilliseconds(int millis) { + Env::Default()->SleepForMicroseconds(millis * 1000); +} +} + +// Special Env used to delay background operations +class SpecialEnv : public EnvWrapper { + public: + // sstable Sync() calls are blocked while this pointer is non-NULL. + port::AtomicPointer delay_sstable_sync_; + + // Simulate no-space errors while this pointer is non-NULL. + port::AtomicPointer no_space_; + + // Simulate non-writable file system while this pointer is non-NULL + port::AtomicPointer non_writable_; + + // Force sync of manifest files to fail while this pointer is non-NULL + port::AtomicPointer manifest_sync_error_; + + // Force write to manifest files to fail while this pointer is non-NULL + port::AtomicPointer manifest_write_error_; + + bool count_random_reads_; + AtomicCounter random_read_counter_; + + AtomicCounter sleep_counter_; + AtomicCounter sleep_time_counter_; + + explicit SpecialEnv(Env* base) : EnvWrapper(base) { + delay_sstable_sync_.Release_Store(NULL); + no_space_.Release_Store(NULL); + non_writable_.Release_Store(NULL); + count_random_reads_ = false; + manifest_sync_error_.Release_Store(NULL); + manifest_write_error_.Release_Store(NULL); + } + + Status NewWritableFile(const std::string& f, WritableFile** r) { + class SSTableFile : public WritableFile { + private: + SpecialEnv* env_; + WritableFile* base_; + + public: + SSTableFile(SpecialEnv* env, WritableFile* base) + : env_(env), + base_(base) { + } + ~SSTableFile() { delete base_; } + Status WriteAt(uint64_t offset, const Slice& data) { + if (env_->no_space_.Acquire_Load() != NULL) { + // Drop writes on the floor + return Status::OK(); + } else { + return base_->WriteAt(offset, data); + } + } + Status Append(const Slice& data) { + if (env_->no_space_.Acquire_Load() != NULL) { + // Drop writes on the floor + return Status::OK(); + } else { + return base_->Append(data); + } + } + Status Close() { return base_->Close(); } + Status Sync() { + while (env_->delay_sstable_sync_.Acquire_Load() != NULL) { + DelayMilliseconds(100); + } + return base_->Sync(); + } + }; + class ManifestFile : public WritableFile { + private: + SpecialEnv* env_; + WritableFile* base_; + public: + ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { } + ~ManifestFile() { delete base_; } + Status WriteAt(uint64_t offset, const Slice& data) { + if (env_->manifest_write_error_.Acquire_Load() != NULL) { + return Status::IOError("simulated writer error"); + } else { + return base_->WriteAt(offset, data); + } + } + Status Append(const Slice& data) { + if (env_->manifest_write_error_.Acquire_Load() != NULL) { + return Status::IOError("simulated writer error"); + } else { + return base_->Append(data); + } + } + Status Close() { return base_->Close(); } + Status Sync() { + if (env_->manifest_sync_error_.Acquire_Load() != NULL) { + return Status::IOError("simulated sync error"); + } else { + return base_->Sync(); + } + } + }; + + if (non_writable_.Acquire_Load() != NULL) { + return Status::IOError("simulated write error"); + } + + Status s = target()->NewWritableFile(f, r); + if (s.ok()) { + if (strstr(f.c_str(), ".sst") != NULL) { + *r = new SSTableFile(this, *r); + } else if (strstr(f.c_str(), "MANIFEST") != NULL) { + *r = new ManifestFile(this, *r); + } + } + return s; + } + + Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) { + class CountingFile : public RandomAccessFile { + private: + RandomAccessFile* target_; + AtomicCounter* counter_; + public: + CountingFile(RandomAccessFile* target, AtomicCounter* counter) + : target_(target), counter_(counter) { + } + virtual ~CountingFile() { delete target_; } + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const { + counter_->Increment(); + return target_->Read(offset, n, result, scratch); + } + }; + + Status s = target()->NewRandomAccessFile(f, r); + if (s.ok() && count_random_reads_) { + *r = new CountingFile(*r, &random_read_counter_); + } + return s; + } + + virtual void SleepForMicroseconds(int micros) { + sleep_counter_.Increment(); + sleep_time_counter_.IncrementBy(micros); + } + +}; + +class DBTest { + private: + const FilterPolicy* filter_policy_; + + // Sequence of option configurations to try + enum OptionConfig { + kDefault, + kFilter, + kUncompressed, + kEnd + }; + int option_config_; + + public: + std::string dbname_; + SpecialEnv* env_; + DB* db_; + + Options last_options_; + + DBTest() : option_config_(kDefault), + env_(new SpecialEnv(Env::Default())) { + filter_policy_ = NewBloomFilterPolicy(10); + dbname_ = test::TmpDir() + "/db_test"; + DestroyDB(dbname_, Options()); + db_ = NULL; + Reopen(); + } + + ~DBTest() { + delete db_; + DestroyDB(dbname_, Options()); + delete env_; + delete filter_policy_; + } + + // Switch to a fresh database with the next option configuration to + // test. Return false if there are no more configurations to test. + bool ChangeOptions() { + option_config_++; + if (option_config_ >= kEnd) { + return false; + } else { + DestroyAndReopen(); + return true; + } + } + + // Return the current option configuration. + Options CurrentOptions() { + Options options; + switch (option_config_) { + case kFilter: + options.filter_policy = filter_policy_; + break; + case kUncompressed: + options.compression = kNoCompression; + break; + default: + break; + } + return options; + } + + DBImpl* dbfull() { + return reinterpret_cast(db_); + } + + void Reopen(Options* options = NULL) { + ASSERT_OK(TryReopen(options)); + } + + void Close() { + delete db_; + db_ = NULL; + } + + void DestroyAndReopen(Options* options = NULL) { + delete db_; + db_ = NULL; + DestroyDB(dbname_, Options()); + ASSERT_OK(TryReopen(options)); + } + + Status TryReopen(Options* options) { + delete db_; + db_ = NULL; + Options opts; + if (options != NULL) { + opts = *options; + } else { + opts = CurrentOptions(); + opts.create_if_missing = true; + } + last_options_ = opts; + + return DB::Open(opts, dbname_, &db_); + } + + Status Put(const std::string& k, const std::string& v) { + return db_->Put(WriteOptions(), k, v); + } + + Status Delete(const std::string& k) { + return db_->Delete(WriteOptions(), k); + } + + std::string Get(const std::string& k, const Snapshot* snapshot = NULL) { + ReadOptions options; + options.snapshot = snapshot; + std::string result; + Status s = db_->Get(options, k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + // Return a string that contains all key,value pairs in order, + // formatted like "(k1->v1)(k2->v2)". + std::string Contents() { + std::vector forward; + std::string result; + Iterator* iter = db_->NewIterator(ReadOptions()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + std::string s = IterStatus(iter); + result.push_back('('); + result.append(s); + result.push_back(')'); + forward.push_back(s); + } + + // Check reverse iteration results are the reverse of forward results + int matched = 0; + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + ASSERT_LT(matched, forward.size()); + ASSERT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]); + matched++; + } + ASSERT_EQ(matched, forward.size()); + + delete iter; + return result; + } + + std::string AllEntriesFor(const Slice& user_key) { + Iterator* iter = dbfull()->TEST_NewInternalIterator(); + InternalKey target(user_key, kMaxSequenceNumber, kTypeValue); + iter->Seek(target.Encode()); + std::string result; + if (!iter->status().ok()) { + result = iter->status().ToString(); + } else { + result = "[ "; + bool first = true; + while (iter->Valid()) { + ParsedInternalKey ikey; + if (!ParseInternalKey(iter->key(), &ikey)) { + result += "CORRUPTED"; + } else { + if (last_options_.comparator->Compare(ikey.user_key, user_key) != 0) { + break; + } + if (!first) { + result += ", "; + } + first = false; + switch (ikey.type) { + case kTypeValue: + result += iter->value().ToString(); + break; + case kTypeDeletion: + result += "DEL"; + break; + } + } + iter->Next(); + } + if (!first) { + result += " "; + } + result += "]"; + } + delete iter; + return result; + } + + int NumTableFilesAtLevel(int level) { + std::string property; + ASSERT_TRUE( + db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level), + &property)); + return atoi(property.c_str()); + } + + int TotalTableFiles() { + int result = 0; + for (int level = 0; level < config::kNumLevels; level++) { + result += NumTableFilesAtLevel(level); + } + return result; + } + + // Return spread of files per level + std::string FilesPerLevel() { + std::string result; + int last_non_zero_offset = 0; + for (int level = 0; level < config::kNumLevels; level++) { + int f = NumTableFilesAtLevel(level); + char buf[100]; + snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); + result += buf; + if (f > 0) { + last_non_zero_offset = result.size(); + } + } + result.resize(last_non_zero_offset); + return result; + } + + int CountFiles() { + std::vector files; + env_->GetChildren(dbname_, &files); + return static_cast(files.size()); + } + + uint64_t Size(const Slice& start, const Slice& limit) { + Range r(start, limit); + uint64_t size; + db_->GetApproximateSizes(&r, 1, &size); + return size; + } + + void Compact(const Slice& start, const Slice& limit) { + db_->CompactRange(&start, &limit); + } + + // Do n memtable compactions, each of which produces an sstable + // covering the range [small,large]. + void MakeTables(int n, const std::string& small, const std::string& large) { + for (int i = 0; i < n; i++) { + Put(small, "begin"); + Put(large, "end"); + dbfull()->TEST_CompactMemTable(); + } + } + + // Prevent pushing of new sstables into deeper levels by adding + // tables that cover a specified range to all levels. + void FillLevels(const std::string& smallest, const std::string& largest) { + MakeTables(config::kNumLevels, smallest, largest); + } + + void DumpFileCounts(const char* label) { + fprintf(stderr, "---\n%s:\n", label); + fprintf(stderr, "maxoverlap: %lld\n", + static_cast( + dbfull()->TEST_MaxNextLevelOverlappingBytes())); + for (int level = 0; level < config::kNumLevels; level++) { + int num = NumTableFilesAtLevel(level); + if (num > 0) { + fprintf(stderr, " level %3d : %d files\n", level, num); + } + } + } + + std::string DumpSSTableList() { + std::string property; + db_->GetProperty("leveldb.sstables", &property); + return property; + } + + std::string IterStatus(Iterator* iter) { + std::string result; + if (iter->Valid()) { + result = iter->key().ToString() + "->" + iter->value().ToString(); + } else { + result = "(invalid)"; + } + return result; + } + + bool DeleteAnSSTFile() { + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) { + ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number))); + return true; + } + } + return false; + } +}; + +TEST(DBTest, Empty) { + do { + ASSERT_TRUE(db_ != NULL); + ASSERT_EQ("NOT_FOUND", Get("foo")); + } while (ChangeOptions()); +} + +TEST(DBTest, ReadWrite) { + do { + ASSERT_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + ASSERT_OK(Put("bar", "v2")); + ASSERT_OK(Put("foo", "v3")); + ASSERT_EQ("v3", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + } while (ChangeOptions()); +} + +TEST(DBTest, PutDeleteGet) { + do { + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2")); + ASSERT_EQ("v2", Get("foo")); + ASSERT_OK(db_->Delete(WriteOptions(), "foo")); + ASSERT_EQ("NOT_FOUND", Get("foo")); + } while (ChangeOptions()); +} + +TEST(DBTest, GetFromImmutableLayer) { + do { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + Reopen(&options); + + ASSERT_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + + env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls + Put("k1", std::string(100000, 'x')); // Fill memtable + Put("k2", std::string(100000, 'y')); // Trigger compaction + ASSERT_EQ("v1", Get("foo")); + env_->delay_sstable_sync_.Release_Store(NULL); // Release sync calls + } while (ChangeOptions()); +} + +TEST(DBTest, GetFromVersions) { + do { + ASSERT_OK(Put("foo", "v1")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v1", Get("foo")); + } while (ChangeOptions()); +} + +TEST(DBTest, GetSnapshot) { + do { + // Try with both a short key and a long key + for (int i = 0; i < 2; i++) { + std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x'); + ASSERT_OK(Put(key, "v1")); + const Snapshot* s1 = db_->GetSnapshot(); + ASSERT_OK(Put(key, "v2")); + ASSERT_EQ("v2", Get(key)); + ASSERT_EQ("v1", Get(key, s1)); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v2", Get(key)); + ASSERT_EQ("v1", Get(key, s1)); + db_->ReleaseSnapshot(s1); + } + } while (ChangeOptions()); +} + +TEST(DBTest, GetLevel0Ordering) { + do { + // Check that we process level-0 files in correct order. The code + // below generates two level-0 files where the earlier one comes + // before the later one in the level-0 file list since the earlier + // one has a smaller "smallest" key. + ASSERT_OK(Put("bar", "b")); + ASSERT_OK(Put("foo", "v1")); + dbfull()->TEST_CompactMemTable(); + ASSERT_OK(Put("foo", "v2")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v2", Get("foo")); + } while (ChangeOptions()); +} + +TEST(DBTest, GetOrderedByLevels) { + do { + ASSERT_OK(Put("foo", "v1")); + Compact("a", "z"); + ASSERT_EQ("v1", Get("foo")); + ASSERT_OK(Put("foo", "v2")); + ASSERT_EQ("v2", Get("foo")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v2", Get("foo")); + } while (ChangeOptions()); +} + +TEST(DBTest, GetPicksCorrectFile) { + do { + // Arrange to have multiple files in a non-level-0 level. + ASSERT_OK(Put("a", "va")); + Compact("a", "b"); + ASSERT_OK(Put("x", "vx")); + Compact("x", "y"); + ASSERT_OK(Put("f", "vf")); + Compact("f", "g"); + ASSERT_EQ("va", Get("a")); + ASSERT_EQ("vf", Get("f")); + ASSERT_EQ("vx", Get("x")); + } while (ChangeOptions()); +} + +TEST(DBTest, IterEmpty) { + Iterator* iter = db_->NewIterator(ReadOptions()); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek("foo"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; +} + +TEST(DBTest, IterSingle) { + ASSERT_OK(Put("a", "va")); + Iterator* iter = db_->NewIterator(ReadOptions()); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek(""); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek("a"); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek("b"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; +} + +TEST(DBTest, IterMulti) { + ASSERT_OK(Put("a", "va")); + ASSERT_OK(Put("b", "vb")); + ASSERT_OK(Put("c", "vc")); + Iterator* iter = db_->NewIterator(ReadOptions()); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek(""); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("a"); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("ax"); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Seek("b"); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Seek("z"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + // Switch from reverse to forward + iter->SeekToLast(); + iter->Prev(); + iter->Prev(); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + + // Switch from forward to reverse + iter->SeekToFirst(); + iter->Next(); + iter->Next(); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + + // Make sure iter stays at snapshot + ASSERT_OK(Put("a", "va2")); + ASSERT_OK(Put("a2", "va3")); + ASSERT_OK(Put("b", "vb2")); + ASSERT_OK(Put("c", "vc2")); + ASSERT_OK(Delete("b")); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; +} + +TEST(DBTest, IterSmallAndLargeMix) { + ASSERT_OK(Put("a", "va")); + ASSERT_OK(Put("b", std::string(100000, 'b'))); + ASSERT_OK(Put("c", "vc")); + ASSERT_OK(Put("d", std::string(100000, 'd'))); + ASSERT_OK(Put("e", std::string(100000, 'e'))); + + Iterator* iter = db_->NewIterator(ReadOptions()); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b')); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd')); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e')); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e')); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd')); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b')); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; +} + +TEST(DBTest, IterMultiWithDelete) { + do { + ASSERT_OK(Put("a", "va")); + ASSERT_OK(Put("b", "vb")); + ASSERT_OK(Put("c", "vc")); + ASSERT_OK(Delete("b")); + ASSERT_EQ("NOT_FOUND", Get("b")); + + Iterator* iter = db_->NewIterator(ReadOptions()); + iter->Seek("c"); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + delete iter; + } while (ChangeOptions()); +} + +TEST(DBTest, Recover) { + do { + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put("baz", "v5")); + + Reopen(); + ASSERT_EQ("v1", Get("foo")); + + ASSERT_EQ("v1", Get("foo")); + ASSERT_EQ("v5", Get("baz")); + ASSERT_OK(Put("bar", "v2")); + ASSERT_OK(Put("foo", "v3")); + + Reopen(); + ASSERT_EQ("v3", Get("foo")); + ASSERT_OK(Put("foo", "v4")); + ASSERT_EQ("v4", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + ASSERT_EQ("v5", Get("baz")); + } while (ChangeOptions()); +} + +TEST(DBTest, RecoveryWithEmptyLog) { + do { + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put("foo", "v2")); + Reopen(); + Reopen(); + ASSERT_OK(Put("foo", "v3")); + Reopen(); + ASSERT_EQ("v3", Get("foo")); + } while (ChangeOptions()); +} + +// Check that writes done during a memtable compaction are recovered +// if the database is shutdown during the memtable compaction. +TEST(DBTest, RecoverDuringMemtableCompaction) { + do { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 1000000; + Reopen(&options); + + // Trigger a long memtable compaction and reopen the database during it + ASSERT_OK(Put("foo", "v1")); // Goes to 1st log file + ASSERT_OK(Put("big1", std::string(10000000, 'x'))); // Fills memtable + ASSERT_OK(Put("big2", std::string(1000, 'y'))); // Triggers compaction + ASSERT_OK(Put("bar", "v2")); // Goes to new log file + + Reopen(&options); + ASSERT_EQ("v1", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + ASSERT_EQ(std::string(10000000, 'x'), Get("big1")); + ASSERT_EQ(std::string(1000, 'y'), Get("big2")); + } while (ChangeOptions()); +} + +static std::string Key(int i) { + char buf[100]; + snprintf(buf, sizeof(buf), "key%06d", i); + return std::string(buf); +} + +TEST(DBTest, MinorCompactionsHappen) { + Options options = CurrentOptions(); + options.write_buffer_size = 10000; + Reopen(&options); + + const int N = 500; + + int starting_num_tables = TotalTableFiles(); + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(i), Key(i) + std::string(1000, 'v'))); + } + int ending_num_tables = TotalTableFiles(); + ASSERT_GT(ending_num_tables, starting_num_tables); + + for (int i = 0; i < N; i++) { + ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i))); + } + + Reopen(); + + for (int i = 0; i < N; i++) { + ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i))); + } +} + +TEST(DBTest, RecoverWithLargeLog) { + { + Options options = CurrentOptions(); + Reopen(&options); + ASSERT_OK(Put("big1", std::string(200000, '1'))); + ASSERT_OK(Put("big2", std::string(200000, '2'))); + ASSERT_OK(Put("small3", std::string(10, '3'))); + ASSERT_OK(Put("small4", std::string(10, '4'))); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + } + + // Make sure that if we re-open with a small write buffer size that + // we flush table files in the middle of a large log file. + Options options = CurrentOptions(); + options.write_buffer_size = 100000; + Reopen(&options); + ASSERT_EQ(NumTableFilesAtLevel(0), 3); + ASSERT_EQ(std::string(200000, '1'), Get("big1")); + ASSERT_EQ(std::string(200000, '2'), Get("big2")); + ASSERT_EQ(std::string(10, '3'), Get("small3")); + ASSERT_EQ(std::string(10, '4'), Get("small4")); + ASSERT_GT(NumTableFilesAtLevel(0), 1); +} + +TEST(DBTest, CompactionsGenerateMultipleFiles) { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; // Large write buffer + Reopen(&options); + + Random rnd(301); + + // Write 32MB (320 values, each 100K) + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + std::vector values; + for (int i = 0; i < 320; i++) { + values.push_back(RandomString(&rnd, 100000)); + ASSERT_OK(Put(Key(i), values[i])); + } + + // Reopening moves updates to level-0 + Reopen(&options); + dbfull()->TEST_CompactRange(0, NULL, NULL); + + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_GT(NumTableFilesAtLevel(1), 1); + for (int i = 0; i < 320; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } +} + +TEST(DBTest, RepeatedWritesToSameKey) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + Reopen(&options); + + // We must have at most one file per level except for level-0, + // which may have up to kL0_StopWritesTrigger files. + const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger; + + Random rnd(301); + std::string value = RandomString(&rnd, 2 * options.write_buffer_size); + for (int i = 0; i < 5 * kMaxFiles; i++) { + Put("key", value); + ASSERT_LE(TotalTableFiles(), kMaxFiles); + fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles()); + } +} + +TEST(DBTest, SparseMerge) { + Options options = CurrentOptions(); + options.compression = kNoCompression; + Reopen(&options); + + FillLevels("A", "Z"); + + // Suppose there is: + // small amount of data with prefix A + // large amount of data with prefix B + // small amount of data with prefix C + // and that recent updates have made small changes to all three prefixes. + // Check that we do not do a compaction that merges all of B in one shot. + const std::string value(1000, 'x'); + Put("A", "va"); + // Write approximately 100MB of "B" values + for (int i = 0; i < 100000; i++) { + char key[100]; + snprintf(key, sizeof(key), "B%010d", i); + Put(key, value); + } + Put("C", "vc"); + dbfull()->TEST_CompactMemTable(); + dbfull()->TEST_CompactRange(0, NULL, NULL); + + // Make sparse update + Put("A", "va2"); + Put("B100", "bvalue2"); + Put("C", "vc2"); + dbfull()->TEST_CompactMemTable(); + + // this test used to test whether or not compactions would push as high as + // possible. + // Hint: we don't do that anymore. +} + +static bool Between(uint64_t val, uint64_t low, uint64_t high) { + bool result = (val >= low) && (val <= high); + if (!result) { + fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", + (unsigned long long)(val), + (unsigned long long)(low), + (unsigned long long)(high)); + } + return result; +} + +TEST(DBTest, ApproximateSizes) { + do { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; // Large write buffer + options.compression = kNoCompression; + DestroyAndReopen(); + + ASSERT_TRUE(Between(Size("", "xyz"), 0, 0)); + Reopen(&options); + ASSERT_TRUE(Between(Size("", "xyz"), 0, 0)); + + // Write 8MB (80 values, each 100K) + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + const int N = 80; + static const int S1 = 100000; + static const int S2 = 105000; // Allow some expansion from metadata + Random rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(i), RandomString(&rnd, S1))); + } + + // 0 because GetApproximateSizes() does not account for memtable space + ASSERT_TRUE(Between(Size("", Key(50)), 0, 0)); + + // Check sizes across recovery by reopening a few times + for (int run = 0; run < 3; run++) { + Reopen(&options); + + for (int compact_start = 0; compact_start < N; compact_start += 10) { + for (int i = 0; i < N; i += 10) { + ASSERT_TRUE(Between(Size("", Key(i)), S1*i, S2*i)); + ASSERT_TRUE(Between(Size("", Key(i)+".suffix"), S1*(i+1), S2*(i+1))); + ASSERT_TRUE(Between(Size(Key(i), Key(i+10)), S1*10, S2*10)); + } + ASSERT_TRUE(Between(Size("", Key(50)), S1*50, S2*50)); + ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), S1*50, S2*50)); + + std::string cstart_str = Key(compact_start); + std::string cend_str = Key(compact_start + 9); + Slice cstart = cstart_str; + Slice cend = cend_str; + dbfull()->TEST_CompactRange(0, &cstart, &cend); + } + + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_GT(NumTableFilesAtLevel(1), 0); + } + } while (ChangeOptions()); +} + +TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) { + do { + Options options = CurrentOptions(); + options.compression = kNoCompression; + Reopen(); + + Random rnd(301); + std::string big1 = RandomString(&rnd, 100000); + ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000))); + ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000))); + ASSERT_OK(Put(Key(2), big1)); + ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000))); + ASSERT_OK(Put(Key(4), big1)); + ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000))); + ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000))); + ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000))); + + // Check sizes across recovery by reopening a few times + for (int run = 0; run < 3; run++) { + Reopen(&options); + + ASSERT_TRUE(Between(Size("", Key(0)), 0, 0)); + ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000)); + ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000)); + ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000)); + ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000)); + ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000)); + ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000)); + ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000)); + ASSERT_TRUE(Between(Size("", Key(8)), 550000, 560000)); + + ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000)); + + dbfull()->TEST_CompactRange(0, NULL, NULL); + } + } while (ChangeOptions()); +} + +TEST(DBTest, IteratorPinsRef) { + Put("foo", "hello"); + + // Get iterator that will yield the current contents of the DB. + Iterator* iter = db_->NewIterator(ReadOptions()); + + // Write to force compactions + Put("foo", "newvalue1"); + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values + } + Put("foo", "newvalue2"); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + ASSERT_EQ("hello", iter->value().ToString()); + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + delete iter; +} + +TEST(DBTest, Snapshot) { + do { + Put("foo", "v1"); + const Snapshot* s1 = db_->GetSnapshot(); + Put("foo", "v2"); + const Snapshot* s2 = db_->GetSnapshot(); + Put("foo", "v3"); + const Snapshot* s3 = db_->GetSnapshot(); + + Put("foo", "v4"); + ASSERT_EQ("v1", Get("foo", s1)); + ASSERT_EQ("v2", Get("foo", s2)); + ASSERT_EQ("v3", Get("foo", s3)); + ASSERT_EQ("v4", Get("foo")); + + db_->ReleaseSnapshot(s3); + ASSERT_EQ("v1", Get("foo", s1)); + ASSERT_EQ("v2", Get("foo", s2)); + ASSERT_EQ("v4", Get("foo")); + + db_->ReleaseSnapshot(s1); + ASSERT_EQ("v2", Get("foo", s2)); + ASSERT_EQ("v4", Get("foo")); + + db_->ReleaseSnapshot(s2); + ASSERT_EQ("v4", Get("foo")); + } while (ChangeOptions()); +} + +TEST(DBTest, HiddenValuesAreRemoved) { + do { + Random rnd(301); + FillLevels("a", "z"); + + std::string big = RandomString(&rnd, 50000); + Put("foo", big); + Put("pastfoo", "v"); + const Snapshot* snapshot = db_->GetSnapshot(); + Put("foo", "tiny"); + Put("pastfoo2", "v2"); // Advance sequence number one more + + ASSERT_OK(dbfull()->TEST_CompactMemTable()); + ASSERT_GT(NumTableFilesAtLevel(0), 0); + + ASSERT_EQ(big, Get("foo", snapshot)); + ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000)); + db_->ReleaseSnapshot(snapshot); + ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]"); + Slice x("x"); + dbfull()->TEST_CompactRange(0, NULL, &x); + ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]"); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_GE(NumTableFilesAtLevel(1), 1); + dbfull()->TEST_CompactRange(1, NULL, &x); + ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]"); + + ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000)); + } while (ChangeOptions()); +} + +TEST(DBTest, DeletionMarkers1) { + Put("foo", "v1"); + ASSERT_OK(dbfull()->TEST_CompactMemTable()); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level + + // Place a table at level last-1 to prevent merging with preceding mutation + Put("a", "begin"); + Put("z", "end"); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ(NumTableFilesAtLevel(last), 1); + ASSERT_EQ(NumTableFilesAtLevel(last-1), 1); + + Delete("foo"); + Put("foo", "v2"); + ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]"); + ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 + ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]"); + Slice z("z"); + dbfull()->TEST_CompactRange(last-2, NULL, &z); + // DEL eliminated, but v1 remains because we aren't compacting that level + // (DEL can be eliminated because v2 hides v1). + ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]"); + dbfull()->TEST_CompactRange(last-1, NULL, NULL); + // Merging last-1 w/ last, so we are the base level for "foo", so + // DEL is removed. (as is v1). + ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]"); +} + +TEST(DBTest, DeletionMarkers2) { + Put("foo", "v1"); + ASSERT_OK(dbfull()->TEST_CompactMemTable()); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level + + // Place a table at level last-1 to prevent merging with preceding mutation + Put("a", "begin"); + Put("z", "end"); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ(NumTableFilesAtLevel(last), 1); + ASSERT_EQ(NumTableFilesAtLevel(last-1), 1); + + Delete("foo"); + ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); + ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 + ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); + dbfull()->TEST_CompactRange(last-2, NULL, NULL); + // DEL kept: "last" file overlaps + ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); + dbfull()->TEST_CompactRange(last-1, NULL, NULL); + // Merging last-1 w/ last, so we are the base level for "foo", so + // DEL is removed. (as is v1). + ASSERT_EQ(AllEntriesFor("foo"), "[ ]"); +} + +TEST(DBTest, OverlapInLevel0) { + do { + ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config"; + + // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0. + ASSERT_OK(Put("100", "v100")); + ASSERT_OK(Put("999", "v999")); + dbfull()->TEST_CompactMemTable(); + ASSERT_OK(Delete("100")); + ASSERT_OK(Delete("999")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("0,1,1", FilesPerLevel()); + + // Make files spanning the following ranges in level-0: + // files[0] 200 .. 900 + // files[1] 300 .. 500 + // Note that files are sorted by smallest key. + ASSERT_OK(Put("300", "v300")); + ASSERT_OK(Put("500", "v500")); + dbfull()->TEST_CompactMemTable(); + ASSERT_OK(Put("200", "v200")); + ASSERT_OK(Put("600", "v600")); + ASSERT_OK(Put("900", "v900")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("2,1,1", FilesPerLevel()); + + // Compact away the placeholder files we created initially + dbfull()->TEST_CompactRange(1, NULL, NULL); + dbfull()->TEST_CompactRange(2, NULL, NULL); + ASSERT_EQ("2", FilesPerLevel()); + + // Do a memtable compaction. Before bug-fix, the compaction would + // not detect the overlap with level-0 files and would incorrectly place + // the deletion in a deeper level. + ASSERT_OK(Delete("600")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("3", FilesPerLevel()); + ASSERT_EQ("NOT_FOUND", Get("600")); + } while (ChangeOptions()); +} + +TEST(DBTest, L0_CompactionBug_Issue44_a) { + Reopen(); + ASSERT_OK(Put("b", "v")); + Reopen(); + ASSERT_OK(Delete("b")); + ASSERT_OK(Delete("a")); + Reopen(); + ASSERT_OK(Delete("a")); + Reopen(); + ASSERT_OK(Put("a", "v")); + Reopen(); + Reopen(); + ASSERT_EQ("(a->v)", Contents()); + DelayMilliseconds(1000); // Wait for compaction to finish + ASSERT_EQ("(a->v)", Contents()); +} + +TEST(DBTest, L0_CompactionBug_Issue44_b) { + Reopen(); + Put("",""); + Reopen(); + Delete("e"); + Put("",""); + Reopen(); + Put("c", "cv"); + Reopen(); + Put("",""); + Reopen(); + Put("",""); + DelayMilliseconds(1000); // Wait for compaction to finish + Reopen(); + Put("d","dv"); + Reopen(); + Put("",""); + Reopen(); + Delete("d"); + Delete("b"); + Reopen(); + ASSERT_EQ("(->)(c->cv)", Contents()); + DelayMilliseconds(1000); // Wait for compaction to finish + ASSERT_EQ("(->)(c->cv)", Contents()); +} + +TEST(DBTest, ComparatorCheck) { + class NewComparator : public Comparator { + public: + virtual const char* Name() const { return "leveldb.NewComparator"; } + virtual int Compare(const Slice& a, const Slice& b) const { + return BytewiseComparator()->Compare(a, b); + } + virtual void FindShortestSeparator(std::string* s, const Slice& l) const { + BytewiseComparator()->FindShortestSeparator(s, l); + } + virtual void FindShortSuccessor(std::string* key) const { + BytewiseComparator()->FindShortSuccessor(key); + } + }; + NewComparator cmp; + Options new_options = CurrentOptions(); + new_options.comparator = &cmp; + Status s = TryReopen(&new_options); + ASSERT_TRUE(!s.ok()); + ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos) + << s.ToString(); +} + +TEST(DBTest, CustomComparator) { + class NumberComparator : public Comparator { + public: + virtual const char* Name() const { return "test.NumberComparator"; } + virtual int Compare(const Slice& a, const Slice& b) const { + return ToNumber(a) - ToNumber(b); + } + virtual void FindShortestSeparator(std::string* s, const Slice& l) const { + ToNumber(*s); // Check format + ToNumber(l); // Check format + } + virtual void FindShortSuccessor(std::string* key) const { + ToNumber(*key); // Check format + } + private: + static int ToNumber(const Slice& x) { + // Check that there are no extra characters. + ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size()-1] == ']') + << EscapeString(x); + int val; + char ignored; + ASSERT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1) + << EscapeString(x); + return val; + } + }; + NumberComparator cmp; + Options new_options = CurrentOptions(); + new_options.create_if_missing = true; + new_options.comparator = &cmp; + new_options.filter_policy = NULL; // Cannot use bloom filters + new_options.write_buffer_size = 1000; // Compact more often + DestroyAndReopen(&new_options); + ASSERT_OK(Put("[10]", "ten")); + ASSERT_OK(Put("[0x14]", "twenty")); + for (int i = 0; i < 2; i++) { + ASSERT_EQ("ten", Get("[10]")); + ASSERT_EQ("ten", Get("[0xa]")); + ASSERT_EQ("twenty", Get("[20]")); + ASSERT_EQ("twenty", Get("[0x14]")); + ASSERT_EQ("NOT_FOUND", Get("[15]")); + ASSERT_EQ("NOT_FOUND", Get("[0xf]")); + Compact("[0]", "[9999]"); + } + + for (int run = 0; run < 2; run++) { + for (int i = 0; i < 1000; i++) { + char buf[100]; + snprintf(buf, sizeof(buf), "[%d]", i*10); + ASSERT_OK(Put(buf, buf)); + } + Compact("[0]", "[1000000]"); + } +} + +TEST(DBTest, ManualCompaction) { + ASSERT_EQ(config::kMaxMemCompactLevel, 2) + << "Need to update this test to match kMaxMemCompactLevel"; + + MakeTables(3, "p", "q"); + ASSERT_EQ("1,1,1", FilesPerLevel()); + + // Compaction range falls before files + Compact("", "c"); + ASSERT_EQ("1,1,1", FilesPerLevel()); + + // Compaction range falls after files + Compact("r", "z"); + ASSERT_EQ("1,1,1", FilesPerLevel()); + + // Compaction range overlaps files + Compact("p1", "p9"); + ASSERT_EQ("0,0,1", FilesPerLevel()); + + // Populate a different range + MakeTables(3, "c", "e"); + ASSERT_EQ("1,1,2", FilesPerLevel()); + + // Compact just the new range + Compact("b", "f"); + ASSERT_EQ("0,0,2", FilesPerLevel()); + + // Compact all + MakeTables(1, "a", "z"); + ASSERT_EQ("0,1,2", FilesPerLevel()); + db_->CompactRange(NULL, NULL); + ASSERT_EQ("0,0,1", FilesPerLevel()); +} + +TEST(DBTest, DBOpen_Options) { + std::string dbname = test::TmpDir() + "/db_options_test"; + DestroyDB(dbname, Options()); + + // Does not exist, and create_if_missing == false: error + DB* db = NULL; + Options opts; + opts.create_if_missing = false; + Status s = DB::Open(opts, dbname, &db); + ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != NULL); + ASSERT_TRUE(db == NULL); + + // Does not exist, and create_if_missing == true: OK + opts.create_if_missing = true; + s = DB::Open(opts, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != NULL); + + delete db; + db = NULL; + + // Does exist, and error_if_exists == true: error + opts.create_if_missing = false; + opts.error_if_exists = true; + s = DB::Open(opts, dbname, &db); + ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != NULL); + ASSERT_TRUE(db == NULL); + + // Does exist, and error_if_exists == false: OK + opts.create_if_missing = true; + opts.error_if_exists = false; + s = DB::Open(opts, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != NULL); + + delete db; + db = NULL; +} + +TEST(DBTest, Locking) { + DB* db2 = NULL; + Status s = DB::Open(CurrentOptions(), dbname_, &db2); + ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db"; +} + +// Check that number of files does not grow when we are out of space +TEST(DBTest, NoSpace) { + Options options = CurrentOptions(); + options.env = env_; + Reopen(&options); + + ASSERT_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + Compact("a", "z"); + const int num_files = CountFiles(); + env_->no_space_.Release_Store(env_); // Force out-of-space errors + env_->sleep_counter_.Reset(); + for (int i = 0; i < 5; i++) { + for (int level = 0; level < config::kNumLevels-1; level++) { + dbfull()->TEST_CompactRange(level, NULL, NULL); + } + } + env_->no_space_.Release_Store(NULL); + ASSERT_LT(CountFiles(), num_files + 3); + + // Check that compaction attempts slept after errors + ASSERT_GE(env_->sleep_counter_.Read(), 5); +} + +TEST(DBTest, ExponentialBackoff) { + Options options = CurrentOptions(); + options.env = env_; + Reopen(&options); + + ASSERT_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + Compact("a", "z"); + env_->non_writable_.Release_Store(env_); // Force errors for new files + env_->sleep_counter_.Reset(); + env_->sleep_time_counter_.Reset(); + for (int i = 0; i < 5; i++) { + dbfull()->TEST_CompactRange(2, NULL, NULL); + } + env_->non_writable_.Release_Store(NULL); + + // Wait for compaction to finish + DelayMilliseconds(1000); + + ASSERT_GE(env_->sleep_counter_.Read(), 5); + ASSERT_LT(env_->sleep_counter_.Read(), 10); + ASSERT_GE(env_->sleep_time_counter_.Read(), 10e6); +} + +TEST(DBTest, NonWritableFileSystem) { + Options options = CurrentOptions(); + options.write_buffer_size = 1000; + options.env = env_; + Reopen(&options); + ASSERT_OK(Put("foo", "v1")); + env_->non_writable_.Release_Store(env_); // Force errors for new files + std::string big(100000, 'x'); + int errors = 0; + for (int i = 0; i < 20; i++) { + fprintf(stderr, "iter %d; errors %d\n", i, errors); + if (!Put("foo", big).ok()) { + errors++; + DelayMilliseconds(100); + } + } + ASSERT_GT(errors, 0); + env_->non_writable_.Release_Store(NULL); +} + +TEST(DBTest, ManifestWriteError) { + // Test for the following problem: + // (a) Compaction produces file F + // (b) Log record containing F is written to MANIFEST file, but Sync() fails + // (c) GC deletes F + // (d) After reopening DB, reads fail since deleted F is named in log record + + // We iterate twice. In the second iteration, everything is the + // same except the log record never makes it to the MANIFEST file. + for (int iter = 0; iter < 2; iter++) { + port::AtomicPointer* error_type = (iter == 0) + ? &env_->manifest_sync_error_ + : &env_->manifest_write_error_; + + // Insert foo=>bar mapping + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + DestroyAndReopen(&options); + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Memtable compaction (will succeed) + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level + + // Merging compaction (will fail) + error_type->Release_Store(env_); + dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail + ASSERT_EQ("bar", Get("foo")); + + // Recovery: should not lose data + error_type->Release_Store(NULL); + Reopen(&options); + ASSERT_EQ("bar", Get("foo")); + } +} + +TEST(DBTest, MissingSSTFile) { + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Dump the memtable to disk. + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + + Close(); + ASSERT_TRUE(DeleteAnSSTFile()); + Options options = CurrentOptions(); + options.paranoid_checks = true; + Status s = TryReopen(&options); + ASSERT_TRUE(!s.ok()); + ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) + << s.ToString(); +} + +TEST(DBTest, FilesDeletedAfterCompaction) { + ASSERT_OK(Put("foo", "v2")); + Compact("a", "z"); + const int num_files = CountFiles(); + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put("foo", "v2")); + Compact("a", "z"); + } + ASSERT_EQ(CountFiles(), num_files); +} + +TEST(DBTest, BloomFilter) { + env_->count_random_reads_ = true; + Options options = CurrentOptions(); + options.env = env_; + options.block_cache = NewLRUCache(0); // Prevent cache hits + options.filter_policy = NewBloomFilterPolicy(10); + Reopen(&options); + + // Populate multiple layers + const int N = 10000; + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(i), Key(i))); + } + Compact("a", "z"); + for (int i = 0; i < N; i += 100) { + ASSERT_OK(Put(Key(i), Key(i))); + } + dbfull()->TEST_CompactMemTable(); + + // Prevent auto compactions triggered by seeks + env_->delay_sstable_sync_.Release_Store(env_); + + // Lookup present keys. Should rarely read from small sstable. + env_->random_read_counter_.Reset(); + for (int i = 0; i < N; i++) { + ASSERT_EQ(Key(i), Get(Key(i))); + } + int reads = env_->random_read_counter_.Read(); + fprintf(stderr, "%d present => %d reads\n", N, reads); + ASSERT_GE(reads, N); + ASSERT_LE(reads, N + 2*N/100); + + // Lookup present keys. Should rarely read from either sstable. + env_->random_read_counter_.Reset(); + for (int i = 0; i < N; i++) { + ASSERT_EQ("NOT_FOUND", Get(Key(i) + ".missing")); + } + reads = env_->random_read_counter_.Read(); + fprintf(stderr, "%d missing => %d reads\n", N, reads); + ASSERT_LE(reads, 3*N/100); + + env_->delay_sstable_sync_.Release_Store(NULL); + Close(); + delete options.block_cache; + delete options.filter_policy; +} + +// Multi-threaded test: +namespace { + +static const int kNumThreads = 4; +static const int kTestSeconds = 10; +static const int kNumKeys = 1000; + +struct MTState { + DBTest* test; + port::AtomicPointer stop; + port::AtomicPointer counter[kNumThreads]; + port::AtomicPointer thread_done[kNumThreads]; +}; + +struct MTThread { + MTState* state; + int id; +}; + +static void MTThreadBody(void* arg) { + MTThread* t = reinterpret_cast(arg); + int id = t->id; + DB* db = t->state->test->db_; + uintptr_t counter = 0; + fprintf(stderr, "... starting thread %d\n", id); + Random rnd(1000 + id); + std::string value; + char valbuf[1500]; + while (t->state->stop.Acquire_Load() == NULL) { + t->state->counter[id].Release_Store(reinterpret_cast(counter)); + + int key = rnd.Uniform(kNumKeys); + char keybuf[20]; + snprintf(keybuf, sizeof(keybuf), "%016d", key); + + if (rnd.OneIn(2)) { + // Write values of the form . + // We add some padding for force compactions. + snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d", + key, id, static_cast(counter)); + ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf))); + } else { + // Read a value and verify that it matches the pattern written above. + Status s = db->Get(ReadOptions(), Slice(keybuf), &value); + if (s.IsNotFound()) { + // Key has not yet been written + } else { + // Check that the writer thread counter is >= the counter in the value + ASSERT_OK(s); + int k, w, c; + ASSERT_EQ(3, sscanf(value.c_str(), "%d.%d.%d", &k, &w, &c)) << value; + ASSERT_EQ(k, key); + ASSERT_GE(w, 0); + ASSERT_LT(w, kNumThreads); + ASSERT_LE(c, reinterpret_cast( + t->state->counter[w].Acquire_Load())); + } + } + counter++; + } + t->state->thread_done[id].Release_Store(t); + fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter)); +} + +} // namespace + +TEST(DBTest, MultiThreaded) { + do { + // Initialize state + MTState mt; + mt.test = this; + mt.stop.Release_Store(0); + for (int id = 0; id < kNumThreads; id++) { + mt.counter[id].Release_Store(0); + mt.thread_done[id].Release_Store(0); + } + + // Start threads + MTThread thread[kNumThreads]; + for (int id = 0; id < kNumThreads; id++) { + thread[id].state = &mt; + thread[id].id = id; + env_->StartThread(MTThreadBody, &thread[id]); + } + + // Let them run for a while + DelayMilliseconds(kTestSeconds * 1000); + + // Stop the threads and wait for them to finish + mt.stop.Release_Store(&mt); + for (int id = 0; id < kNumThreads; id++) { + while (mt.thread_done[id].Acquire_Load() == NULL) { + DelayMilliseconds(100); + } + } + } while (ChangeOptions()); +} + +namespace { +typedef std::map KVMap; +} + +class ModelDB: public DB { + public: + class ModelSnapshot : public Snapshot { + public: + KVMap map_; + }; + + explicit ModelDB(const Options& options): options_(options) { } + ~ModelDB() { } + virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) { + return DB::Put(o, k, v); + } + virtual Status Delete(const WriteOptions& o, const Slice& key) { + return DB::Delete(o, key); + } + virtual Status Get(const ReadOptions& options, + const Slice& key, std::string* value) { + assert(false); // Not implemented + return Status::NotFound(key); + } + virtual Iterator* NewIterator(const ReadOptions& options) { + if (options.snapshot == NULL) { + KVMap* saved = new KVMap; + *saved = map_; + return new ModelIter(saved, true); + } else { + const KVMap* snapshot_state = + &(reinterpret_cast(options.snapshot)->map_); + return new ModelIter(snapshot_state, false); + } + } + virtual const Snapshot* GetSnapshot() { + ModelSnapshot* snapshot = new ModelSnapshot; + snapshot->map_ = map_; + return snapshot; + } + + virtual void ReleaseSnapshot(const Snapshot* snapshot) { + delete reinterpret_cast(snapshot); + } + virtual Status Write(const WriteOptions& options, WriteBatch* batch) { + class Handler : public WriteBatch::Handler { + public: + KVMap* map_; + virtual void Put(const Slice& key, const Slice& value) { + (*map_)[key.ToString()] = value.ToString(); + } + virtual void Delete(const Slice& key) { + map_->erase(key.ToString()); + } + }; + Handler handler; + handler.map_ = &map_; + return batch->Iterate(&handler); + } + + virtual bool GetProperty(const Slice& property, std::string* value) { + return false; + } + virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) { + for (int i = 0; i < n; i++) { + sizes[i] = 0; + } + } + virtual void CompactRange(const Slice* start, const Slice* end) { + } + + private: + class ModelIter: public Iterator { + public: + ModelIter(const KVMap* map, bool owned) + : map_(map), owned_(owned), iter_(map_->end()) { + } + ~ModelIter() { + if (owned_) delete map_; + } + virtual bool Valid() const { return iter_ != map_->end(); } + virtual void SeekToFirst() { iter_ = map_->begin(); } + virtual void SeekToLast() { + if (map_->empty()) { + iter_ = map_->end(); + } else { + iter_ = map_->find(map_->rbegin()->first); + } + } + virtual void Seek(const Slice& k) { + iter_ = map_->lower_bound(k.ToString()); + } + virtual void Next() { ++iter_; } + virtual void Prev() { --iter_; } + virtual Slice key() const { return iter_->first; } + virtual Slice value() const { return iter_->second; } + virtual Status status() const { return Status::OK(); } + private: + const KVMap* const map_; + const bool owned_; // Do we own map_ + KVMap::const_iterator iter_; + }; + const Options options_; + KVMap map_; +}; + +static std::string RandomKey(Random* rnd) { + int len = (rnd->OneIn(3) + ? 1 // Short sometimes to encourage collisions + : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10))); + return test::RandomKey(rnd, len); +} + +static bool CompareIterators(int step, + DB* model, + DB* db, + const Snapshot* model_snap, + const Snapshot* db_snap) { + ReadOptions options; + options.snapshot = model_snap; + Iterator* miter = model->NewIterator(options); + options.snapshot = db_snap; + Iterator* dbiter = db->NewIterator(options); + bool ok = true; + int count = 0; + for (miter->SeekToFirst(), dbiter->SeekToFirst(); + ok && miter->Valid() && dbiter->Valid(); + miter->Next(), dbiter->Next()) { + count++; + if (miter->key().compare(dbiter->key()) != 0) { + fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n", + step, + EscapeString(miter->key()).c_str(), + EscapeString(dbiter->key()).c_str()); + ok = false; + break; + } + + if (miter->value().compare(dbiter->value()) != 0) { + fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n", + step, + EscapeString(miter->key()).c_str(), + EscapeString(miter->value()).c_str(), + EscapeString(miter->value()).c_str()); + ok = false; + } + } + + if (ok) { + if (miter->Valid() != dbiter->Valid()) { + fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n", + step, miter->Valid(), dbiter->Valid()); + ok = false; + } + } + fprintf(stderr, "%d entries compared: ok=%d\n", count, ok); + delete miter; + delete dbiter; + return ok; +} + +TEST(DBTest, Randomized) { + Random rnd(test::RandomSeed()); + do { + ModelDB model(CurrentOptions()); + const int N = 10000; + const Snapshot* model_snap = NULL; + const Snapshot* db_snap = NULL; + std::string k, v; + for (int step = 0; step < N; step++) { + if (step % 100 == 0) { + fprintf(stderr, "Step %d of %d\n", step, N); + } + // TODO(sanjay): Test Get() works + int p = rnd.Uniform(100); + if (p < 45) { // Put + k = RandomKey(&rnd); + v = RandomString(&rnd, + rnd.OneIn(20) + ? 100 + rnd.Uniform(100) + : rnd.Uniform(8)); + ASSERT_OK(model.Put(WriteOptions(), k, v)); + ASSERT_OK(db_->Put(WriteOptions(), k, v)); + + } else if (p < 90) { // Delete + k = RandomKey(&rnd); + ASSERT_OK(model.Delete(WriteOptions(), k)); + ASSERT_OK(db_->Delete(WriteOptions(), k)); + + + } else { // Multi-element batch + WriteBatch b; + const int num = rnd.Uniform(8); + for (int i = 0; i < num; i++) { + if (i == 0 || !rnd.OneIn(10)) { + k = RandomKey(&rnd); + } else { + // Periodically re-use the same key from the previous iter, so + // we have multiple entries in the write batch for the same key + } + if (rnd.OneIn(2)) { + v = RandomString(&rnd, rnd.Uniform(10)); + b.Put(k, v); + } else { + b.Delete(k); + } + } + ASSERT_OK(model.Write(WriteOptions(), &b)); + ASSERT_OK(db_->Write(WriteOptions(), &b)); + } + + if ((step % 100) == 0) { + ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL)); + ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap)); + // Save a snapshot from each DB this time that we'll use next + // time we compare things, to make sure the current state is + // preserved with the snapshot + if (model_snap != NULL) model.ReleaseSnapshot(model_snap); + if (db_snap != NULL) db_->ReleaseSnapshot(db_snap); + + Reopen(); + ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL)); + + model_snap = model.GetSnapshot(); + db_snap = db_->GetSnapshot(); + } + } + if (model_snap != NULL) model.ReleaseSnapshot(model_snap); + if (db_snap != NULL) db_->ReleaseSnapshot(db_snap); + } while (ChangeOptions()); +} + +std::string MakeKey(unsigned int num) { + char buf[30]; + snprintf(buf, sizeof(buf), "%016u", num); + return std::string(buf); +} + +void BM_LogAndApply(int iters, int num_base_files) { + std::string dbname = test::TmpDir() + "/leveldb_test_benchmark"; + DestroyDB(dbname, Options()); + + DB* db = NULL; + Options opts; + opts.create_if_missing = true; + Status s = DB::Open(opts, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != NULL); + + delete db; + db = NULL; + + Env* env = Env::Default(); + + port::Mutex mu; + port::CondVar cv(&mu); + bool wt; + MutexLock l(&mu); + + InternalKeyComparator cmp(BytewiseComparator()); + Options options; + VersionSet vset(dbname, &options, NULL, &cmp); + ASSERT_OK(vset.Recover()); + VersionEdit vbase; + uint64_t fnum = 1; + for (int i = 0; i < num_base_files; i++) { + InternalKey start(MakeKey(2*fnum), 1, kTypeValue); + InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion); + vbase.AddFile(2, fnum++, 1 /* file size */, start, limit); + } + ASSERT_OK(vset.LogAndApply(&vbase, &mu, &cv, &wt)); + + uint64_t start_micros = env->NowMicros(); + + for (int i = 0; i < iters; i++) { + VersionEdit vedit; + vedit.DeleteFile(2, fnum); + InternalKey start(MakeKey(2*fnum), 1, kTypeValue); + InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion); + vedit.AddFile(2, fnum++, 1 /* file size */, start, limit); + vset.LogAndApply(&vedit, &mu, &cv, &wt); + } + uint64_t stop_micros = env->NowMicros(); + unsigned int us = stop_micros - start_micros; + char buf[16]; + snprintf(buf, sizeof(buf), "%d", num_base_files); + fprintf(stderr, + "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n", + buf, iters, us, ((float)us) / iters); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + if (argc > 1 && std::string(argv[1]) == "--benchmark") { + leveldb::BM_LogAndApply(1000, 1); + leveldb::BM_LogAndApply(1000, 100); + leveldb::BM_LogAndApply(1000, 10000); + leveldb::BM_LogAndApply(100, 100000); + return 0; + } + + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/db/dbformat.cc b/Subtrees/hyperleveldb/db/dbformat.cc new file mode 100644 index 0000000000..10cbab7a49 --- /dev/null +++ b/Subtrees/hyperleveldb/db/dbformat.cc @@ -0,0 +1,140 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include "dbformat.h" +#include "../port/port.h" +#include "../util/coding.h" + +namespace hyperleveldb { + +static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) { + assert(seq <= kMaxSequenceNumber); + assert(t <= kValueTypeForSeek); + return (seq << 8) | t; +} + +void AppendInternalKey(std::string* result, const ParsedInternalKey& key) { + result->append(key.user_key.data(), key.user_key.size()); + PutFixed64(result, PackSequenceAndType(key.sequence, key.type)); +} + +std::string ParsedInternalKey::DebugString() const { + char buf[50]; + snprintf(buf, sizeof(buf), "' @ %llu : %d", + (unsigned long long) sequence, + int(type)); + std::string result = "'"; + result += EscapeString(user_key.ToString()); + result += buf; + return result; +} + +std::string InternalKey::DebugString() const { + std::string result; + ParsedInternalKey parsed; + if (ParseInternalKey(rep_, &parsed)) { + result = parsed.DebugString(); + } else { + result = "(bad)"; + result.append(EscapeString(rep_)); + } + return result; +} + +const char* InternalKeyComparator::Name() const { + return "leveldb.InternalKeyComparator"; +} + +int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const { + // Order by: + // increasing user key (according to user-supplied comparator) + // decreasing sequence number + // decreasing type (though sequence# should be enough to disambiguate) + int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey)); + if (r == 0) { + const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8); + const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8); + if (anum > bnum) { + r = -1; + } else if (anum < bnum) { + r = +1; + } + } + return r; +} + +void InternalKeyComparator::FindShortestSeparator( + std::string* start, + const Slice& limit) const { + // Attempt to shorten the user portion of the key + Slice user_start = ExtractUserKey(*start); + Slice user_limit = ExtractUserKey(limit); + std::string tmp(user_start.data(), user_start.size()); + user_comparator_->FindShortestSeparator(&tmp, user_limit); + if (tmp.size() < user_start.size() && + user_comparator_->Compare(user_start, tmp) < 0) { + // User key has become shorter physically, but larger logically. + // Tack on the earliest possible number to the shortened user key. + PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); + assert(this->Compare(*start, tmp) < 0); + assert(this->Compare(tmp, limit) < 0); + start->swap(tmp); + } +} + +void InternalKeyComparator::FindShortSuccessor(std::string* key) const { + Slice user_key = ExtractUserKey(*key); + std::string tmp(user_key.data(), user_key.size()); + user_comparator_->FindShortSuccessor(&tmp); + if (tmp.size() < user_key.size() && + user_comparator_->Compare(user_key, tmp) < 0) { + // User key has become shorter physically, but larger logically. + // Tack on the earliest possible number to the shortened user key. + PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); + assert(this->Compare(*key, tmp) < 0); + key->swap(tmp); + } +} + +const char* InternalFilterPolicy::Name() const { + return user_policy_->Name(); +} + +void InternalFilterPolicy::CreateFilter(const Slice* keys, int n, + std::string* dst) const { + // We rely on the fact that the code in table.cc does not mind us + // adjusting keys[]. + Slice* mkey = const_cast(keys); + for (int i = 0; i < n; i++) { + mkey[i] = ExtractUserKey(keys[i]); + // TODO(sanjay): Suppress dups? + } + user_policy_->CreateFilter(keys, n, dst); +} + +bool InternalFilterPolicy::KeyMayMatch(const Slice& key, const Slice& f) const { + return user_policy_->KeyMayMatch(ExtractUserKey(key), f); +} + +LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) { + size_t usize = user_key.size(); + size_t needed = usize + 13; // A conservative estimate + char* dst; + if (needed <= sizeof(space_)) { + dst = space_; + } else { + dst = new char[needed]; + } + start_ = dst; + dst = EncodeVarint32(dst, usize + 8); + kstart_ = dst; + memcpy(dst, user_key.data(), usize); + dst += usize; + EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek)); + dst += 8; + end_ = dst; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/dbformat.h b/Subtrees/hyperleveldb/db/dbformat.h new file mode 100644 index 0000000000..1571c5e072 --- /dev/null +++ b/Subtrees/hyperleveldb/db/dbformat.h @@ -0,0 +1,229 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_FORMAT_H_ +#define STORAGE_HYPERLEVELDB_DB_FORMAT_H_ + +#include +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/db.h" +#include "../hyperleveldb/filter_policy.h" +#include "../hyperleveldb/slice.h" +#include "../hyperleveldb/table_builder.h" +#include "../util/coding.h" +#include "../util/logging.h" + +namespace hyperleveldb { + +// Grouping of constants. We may want to make some of these +// parameters set via options. +namespace config { +static const int kNumLevels = 7; + +// Level-0 compaction is started when we hit this many files. +static const int kL0_CompactionTrigger = 4; + +// Soft limit on number of level-0 files. We could slow down writes at this +// point, but don't. +static const int kL0_SlowdownWritesTrigger = 8; + +// Maximum number of level-0 files. We could stop writes at this point, but +// don't. +static const int kL0_StopWritesTrigger = 12; + +// Maximum level to which a new compacted memtable is pushed if it +// does not create overlap. We try to push to level 2 to avoid the +// relatively expensive level 0=>1 compactions and to avoid some +// expensive manifest file operations. We do not push all the way to +// the largest level since that can generate a lot of wasted disk +// space if the same key space is being repeatedly overwritten. +static const int kMaxMemCompactLevel = 2; + +} // namespace config + +class InternalKey; + +// Value types encoded as the last component of internal keys. +// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk +// data structures. +enum ValueType { + kTypeDeletion = 0x0, + kTypeValue = 0x1 +}; +// kValueTypeForSeek defines the ValueType that should be passed when +// constructing a ParsedInternalKey object for seeking to a particular +// sequence number (since we sort sequence numbers in decreasing order +// and the value type is embedded as the low 8 bits in the sequence +// number in internal keys, we need to use the highest-numbered +// ValueType, not the lowest). +static const ValueType kValueTypeForSeek = kTypeValue; + +typedef uint64_t SequenceNumber; + +// We leave eight bits empty at the bottom so a type and sequence# +// can be packed together into 64-bits. +static const SequenceNumber kMaxSequenceNumber = + ((0x1ull << 56) - 1); + +struct ParsedInternalKey { + Slice user_key; + SequenceNumber sequence; + ValueType type; + + ParsedInternalKey() { } // Intentionally left uninitialized (for speed) + ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t) + : user_key(u), sequence(seq), type(t) { } + std::string DebugString() const; +}; + +// Return the length of the encoding of "key". +inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) { + return key.user_key.size() + 8; +} + +// Append the serialization of "key" to *result. +extern void AppendInternalKey(std::string* result, + const ParsedInternalKey& key); + +// Attempt to parse an internal key from "internal_key". On success, +// stores the parsed data in "*result", and returns true. +// +// On error, returns false, leaves "*result" in an undefined state. +extern bool ParseInternalKey(const Slice& internal_key, + ParsedInternalKey* result); + +// Returns the user key portion of an internal key. +inline Slice ExtractUserKey(const Slice& internal_key) { + assert(internal_key.size() >= 8); + return Slice(internal_key.data(), internal_key.size() - 8); +} + +inline ValueType ExtractValueType(const Slice& internal_key) { + assert(internal_key.size() >= 8); + const size_t n = internal_key.size(); + uint64_t num = DecodeFixed64(internal_key.data() + n - 8); + unsigned char c = num & 0xff; + return static_cast(c); +} + +// A comparator for internal keys that uses a specified comparator for +// the user key portion and breaks ties by decreasing sequence number. +class InternalKeyComparator : public Comparator { + private: + const Comparator* user_comparator_; + public: + explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) { } + virtual const char* Name() const; + virtual int Compare(const Slice& a, const Slice& b) const; + virtual void FindShortestSeparator( + std::string* start, + const Slice& limit) const; + virtual void FindShortSuccessor(std::string* key) const; + + const Comparator* user_comparator() const { return user_comparator_; } + + int Compare(const InternalKey& a, const InternalKey& b) const; +}; + +// Filter policy wrapper that converts from internal keys to user keys +class InternalFilterPolicy : public FilterPolicy { + private: + const FilterPolicy* const user_policy_; + public: + explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) { } + virtual const char* Name() const; + virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const; + virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const; +}; + +// Modules in this directory should keep internal keys wrapped inside +// the following class instead of plain strings so that we do not +// incorrectly use string comparisons instead of an InternalKeyComparator. +class InternalKey { + private: + std::string rep_; + public: + InternalKey() { } // Leave rep_ as empty to indicate it is invalid + InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) { + AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t)); + } + + void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); } + Slice Encode() const { + assert(!rep_.empty()); + return rep_; + } + + Slice user_key() const { return ExtractUserKey(rep_); } + + void SetFrom(const ParsedInternalKey& p) { + rep_.clear(); + AppendInternalKey(&rep_, p); + } + + void Clear() { rep_.clear(); } + + std::string DebugString() const; +}; + +inline int InternalKeyComparator::Compare( + const InternalKey& a, const InternalKey& b) const { + return Compare(a.Encode(), b.Encode()); +} + +inline bool ParseInternalKey(const Slice& internal_key, + ParsedInternalKey* result) { + const size_t n = internal_key.size(); + if (n < 8) return false; + uint64_t num = DecodeFixed64(internal_key.data() + n - 8); + unsigned char c = num & 0xff; + result->sequence = num >> 8; + result->type = static_cast(c); + result->user_key = Slice(internal_key.data(), n - 8); + return (c <= static_cast(kTypeValue)); +} + +// A helper class useful for DBImpl::Get() +class LookupKey { + public: + // Initialize *this for looking up user_key at a snapshot with + // the specified sequence number. + LookupKey(const Slice& user_key, SequenceNumber sequence); + + ~LookupKey(); + + // Return a key suitable for lookup in a MemTable. + Slice memtable_key() const { return Slice(start_, end_ - start_); } + + // Return an internal key (suitable for passing to an internal iterator) + Slice internal_key() const { return Slice(kstart_, end_ - kstart_); } + + // Return the user key + Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); } + + private: + // We construct a char array of the form: + // klength varint32 <-- start_ + // userkey char[klength] <-- kstart_ + // tag uint64 + // <-- end_ + // The array is a suitable MemTable key. + // The suffix starting with "userkey" can be used as an InternalKey. + const char* start_; + const char* kstart_; + const char* end_; + char space_[200]; // Avoid allocation for short keys + + // No copying allowed + LookupKey(const LookupKey&); + void operator=(const LookupKey&); +}; + +inline LookupKey::~LookupKey() { + if (start_ != space_) delete[] start_; +} + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_FORMAT_H_ diff --git a/Subtrees/hyperleveldb/db/dbformat_test.cc b/Subtrees/hyperleveldb/db/dbformat_test.cc new file mode 100644 index 0000000000..10d3b632d0 --- /dev/null +++ b/Subtrees/hyperleveldb/db/dbformat_test.cc @@ -0,0 +1,112 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "dbformat.h" +#include "../util/logging.h" +#include "../util/testharness.h" + +namespace hyperleveldb { + +static std::string IKey(const std::string& user_key, + uint64_t seq, + ValueType vt) { + std::string encoded; + AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); + return encoded; +} + +static std::string Shorten(const std::string& s, const std::string& l) { + std::string result = s; + InternalKeyComparator(BytewiseComparator()).FindShortestSeparator(&result, l); + return result; +} + +static std::string ShortSuccessor(const std::string& s) { + std::string result = s; + InternalKeyComparator(BytewiseComparator()).FindShortSuccessor(&result); + return result; +} + +static void TestKey(const std::string& key, + uint64_t seq, + ValueType vt) { + std::string encoded = IKey(key, seq, vt); + + Slice in(encoded); + ParsedInternalKey decoded("", 0, kTypeValue); + + ASSERT_TRUE(ParseInternalKey(in, &decoded)); + ASSERT_EQ(key, decoded.user_key.ToString()); + ASSERT_EQ(seq, decoded.sequence); + ASSERT_EQ(vt, decoded.type); + + ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded)); +} + +class FormatTest { }; + +TEST(FormatTest, InternalKey_EncodeDecode) { + const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" }; + const uint64_t seq[] = { + 1, 2, 3, + (1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1, + (1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1, + (1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1 + }; + for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { + for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { + TestKey(keys[k], seq[s], kTypeValue); + TestKey("hello", 1, kTypeDeletion); + } + } +} + +TEST(FormatTest, InternalKeyShortSeparator) { + // When user keys are same + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), + IKey("foo", 99, kTypeValue))); + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), + IKey("foo", 101, kTypeValue))); + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), + IKey("foo", 100, kTypeValue))); + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), + IKey("foo", 100, kTypeDeletion))); + + // When user keys are misordered + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), + IKey("bar", 99, kTypeValue))); + + // When user keys are different, but correctly ordered + ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("foo", 100, kTypeValue), + IKey("hello", 200, kTypeValue))); + + // When start user key is prefix of limit user key + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), + IKey("foobar", 200, kTypeValue))); + + // When limit user key is prefix of start user key + ASSERT_EQ(IKey("foobar", 100, kTypeValue), + Shorten(IKey("foobar", 100, kTypeValue), + IKey("foo", 200, kTypeValue))); +} + +TEST(FormatTest, InternalKeyShortestSuccessor) { + ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), + ShortSuccessor(IKey("foo", 100, kTypeValue))); + ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue), + ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/db/filename.cc b/Subtrees/hyperleveldb/db/filename.cc new file mode 100644 index 0000000000..3f8ead9480 --- /dev/null +++ b/Subtrees/hyperleveldb/db/filename.cc @@ -0,0 +1,139 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include "filename.h" +#include "dbformat.h" +#include "../hyperleveldb/env.h" +#include "../util/logging.h" + +namespace hyperleveldb { + +// A utility routine: write "data" to the named file and Sync() it. +extern Status WriteStringToFileSync(Env* env, const Slice& data, + const std::string& fname); + +static std::string MakeFileName(const std::string& name, uint64_t number, + const char* suffix) { + char buf[100]; + snprintf(buf, sizeof(buf), "/%06llu.%s", + static_cast(number), + suffix); + return name + buf; +} + +std::string LogFileName(const std::string& name, uint64_t number) { + assert(number > 0); + return MakeFileName(name, number, "log"); +} + +std::string TableFileName(const std::string& name, uint64_t number) { + assert(number > 0); + return MakeFileName(name, number, "sst"); +} + +std::string DescriptorFileName(const std::string& dbname, uint64_t number) { + assert(number > 0); + char buf[100]; + snprintf(buf, sizeof(buf), "/MANIFEST-%06llu", + static_cast(number)); + return dbname + buf; +} + +std::string CurrentFileName(const std::string& dbname) { + return dbname + "/CURRENT"; +} + +std::string LockFileName(const std::string& dbname) { + return dbname + "/LOCK"; +} + +std::string TempFileName(const std::string& dbname, uint64_t number) { + assert(number > 0); + return MakeFileName(dbname, number, "dbtmp"); +} + +std::string InfoLogFileName(const std::string& dbname) { + return dbname + "/LOG"; +} + +// Return the name of the old info log file for "dbname". +std::string OldInfoLogFileName(const std::string& dbname) { + return dbname + "/LOG.old"; +} + + +// Owned filenames have the form: +// dbname/CURRENT +// dbname/LOCK +// dbname/LOG +// dbname/LOG.old +// dbname/MANIFEST-[0-9]+ +// dbname/[0-9]+.(log|sst) +bool ParseFileName(const std::string& fname, + uint64_t* number, + FileType* type) { + Slice rest(fname); + if (rest == "CURRENT") { + *number = 0; + *type = kCurrentFile; + } else if (rest == "LOCK") { + *number = 0; + *type = kDBLockFile; + } else if (rest == "LOG" || rest == "LOG.old") { + *number = 0; + *type = kInfoLogFile; + } else if (rest.starts_with("MANIFEST-")) { + rest.remove_prefix(strlen("MANIFEST-")); + uint64_t num; + if (!ConsumeDecimalNumber(&rest, &num)) { + return false; + } + if (!rest.empty()) { + return false; + } + *type = kDescriptorFile; + *number = num; + } else { + // Avoid strtoull() to keep filename format independent of the + // current locale + uint64_t num; + if (!ConsumeDecimalNumber(&rest, &num)) { + return false; + } + Slice suffix = rest; + if (suffix == Slice(".log")) { + *type = kLogFile; + } else if (suffix == Slice(".sst")) { + *type = kTableFile; + } else if (suffix == Slice(".dbtmp")) { + *type = kTempFile; + } else { + return false; + } + *number = num; + } + return true; +} + +Status SetCurrentFile(Env* env, const std::string& dbname, + uint64_t descriptor_number) { + // Remove leading "dbname/" and add newline to manifest file name + std::string manifest = DescriptorFileName(dbname, descriptor_number); + Slice contents = manifest; + assert(contents.starts_with(dbname + "/")); + contents.remove_prefix(dbname.size() + 1); + std::string tmp = TempFileName(dbname, descriptor_number); + Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp); + if (s.ok()) { + s = env->RenameFile(tmp, CurrentFileName(dbname)); + } + if (!s.ok()) { + env->DeleteFile(tmp); + } + return s; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/filename.h b/Subtrees/hyperleveldb/db/filename.h new file mode 100644 index 0000000000..f787cc8202 --- /dev/null +++ b/Subtrees/hyperleveldb/db/filename.h @@ -0,0 +1,80 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// File names used by DB code + +#ifndef STORAGE_HYPERLEVELDB_DB_FILENAME_H_ +#define STORAGE_HYPERLEVELDB_DB_FILENAME_H_ + +#include +#include +#include "../hyperleveldb/slice.h" +#include "../hyperleveldb/status.h" +#include "../port/port.h" + +namespace hyperleveldb { + +class Env; + +enum FileType { + kLogFile, + kDBLockFile, + kTableFile, + kDescriptorFile, + kCurrentFile, + kTempFile, + kInfoLogFile // Either the current one, or an old one +}; + +// Return the name of the log file with the specified number +// in the db named by "dbname". The result will be prefixed with +// "dbname". +extern std::string LogFileName(const std::string& dbname, uint64_t number); + +// Return the name of the sstable with the specified number +// in the db named by "dbname". The result will be prefixed with +// "dbname". +extern std::string TableFileName(const std::string& dbname, uint64_t number); + +// Return the name of the descriptor file for the db named by +// "dbname" and the specified incarnation number. The result will be +// prefixed with "dbname". +extern std::string DescriptorFileName(const std::string& dbname, + uint64_t number); + +// Return the name of the current file. This file contains the name +// of the current manifest file. The result will be prefixed with +// "dbname". +extern std::string CurrentFileName(const std::string& dbname); + +// Return the name of the lock file for the db named by +// "dbname". The result will be prefixed with "dbname". +extern std::string LockFileName(const std::string& dbname); + +// Return the name of a temporary file owned by the db named "dbname". +// The result will be prefixed with "dbname". +extern std::string TempFileName(const std::string& dbname, uint64_t number); + +// Return the name of the info log file for "dbname". +extern std::string InfoLogFileName(const std::string& dbname); + +// Return the name of the old info log file for "dbname". +extern std::string OldInfoLogFileName(const std::string& dbname); + +// If filename is a leveldb file, store the type of the file in *type. +// The number encoded in the filename is stored in *number. If the +// filename was successfully parsed, returns true. Else return false. +extern bool ParseFileName(const std::string& filename, + uint64_t* number, + FileType* type); + +// Make the CURRENT file point to the descriptor file with the +// specified number. +extern Status SetCurrentFile(Env* env, const std::string& dbname, + uint64_t descriptor_number); + + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_FILENAME_H_ diff --git a/Subtrees/hyperleveldb/db/filename_test.cc b/Subtrees/hyperleveldb/db/filename_test.cc new file mode 100644 index 0000000000..90d9bbb827 --- /dev/null +++ b/Subtrees/hyperleveldb/db/filename_test.cc @@ -0,0 +1,122 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "filename.h" + +#include "dbformat.h" +#include "../port/port.h" +#include "../util/logging.h" +#include "../util/testharness.h" + +namespace hyperleveldb { + +class FileNameTest { }; + +TEST(FileNameTest, Parse) { + Slice db; + FileType type; + uint64_t number; + + // Successful parses + static struct { + const char* fname; + uint64_t number; + FileType type; + } cases[] = { + { "100.log", 100, kLogFile }, + { "0.log", 0, kLogFile }, + { "0.sst", 0, kTableFile }, + { "CURRENT", 0, kCurrentFile }, + { "LOCK", 0, kDBLockFile }, + { "MANIFEST-2", 2, kDescriptorFile }, + { "MANIFEST-7", 7, kDescriptorFile }, + { "LOG", 0, kInfoLogFile }, + { "LOG.old", 0, kInfoLogFile }, + { "18446744073709551615.log", 18446744073709551615ull, kLogFile }, + }; + for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + std::string f = cases[i].fname; + ASSERT_TRUE(ParseFileName(f, &number, &type)) << f; + ASSERT_EQ(cases[i].type, type) << f; + ASSERT_EQ(cases[i].number, number) << f; + } + + // Errors + static const char* errors[] = { + "", + "foo", + "foo-dx-100.log", + ".log", + "", + "manifest", + "CURREN", + "CURRENTX", + "MANIFES", + "MANIFEST", + "MANIFEST-", + "XMANIFEST-3", + "MANIFEST-3x", + "LOC", + "LOCKx", + "LO", + "LOGx", + "18446744073709551616.log", + "184467440737095516150.log", + "100", + "100.", + "100.lop" + }; + for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { + std::string f = errors[i]; + ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; + } +} + +TEST(FileNameTest, Construction) { + uint64_t number; + FileType type; + std::string fname; + + fname = CurrentFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0, number); + ASSERT_EQ(kCurrentFile, type); + + fname = LockFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0, number); + ASSERT_EQ(kDBLockFile, type); + + fname = LogFileName("foo", 192); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(192, number); + ASSERT_EQ(kLogFile, type); + + fname = TableFileName("bar", 200); + ASSERT_EQ("bar/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(200, number); + ASSERT_EQ(kTableFile, type); + + fname = DescriptorFileName("bar", 100); + ASSERT_EQ("bar/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(100, number); + ASSERT_EQ(kDescriptorFile, type); + + fname = TempFileName("tmp", 999); + ASSERT_EQ("tmp/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(999, number); + ASSERT_EQ(kTempFile, type); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/db/leveldb_main.cc b/Subtrees/hyperleveldb/db/leveldb_main.cc new file mode 100644 index 0000000000..cd6d729d47 --- /dev/null +++ b/Subtrees/hyperleveldb/db/leveldb_main.cc @@ -0,0 +1,238 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include "dbformat.h" +#include "filename.h" +#include "log_reader.h" +#include "version_edit.h" +#include "write_batch_internal.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/iterator.h" +#include "../hyperleveldb/options.h" +#include "../hyperleveldb/status.h" +#include "../hyperleveldb/table.h" +#include "../hyperleveldb/write_batch.h" +#include "../util/logging.h" + +namespace hyperleveldb { + +namespace { + +bool GuessType(const std::string& fname, FileType* type) { + size_t pos = fname.rfind('/'); + std::string basename; + if (pos == std::string::npos) { + basename = fname; + } else { + basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1); + } + uint64_t ignored; + return ParseFileName(basename, &ignored, type); +} + +// Notified when log reader encounters corruption. +class CorruptionReporter : public log::Reader::Reporter { + public: + virtual void Corruption(size_t bytes, const Status& status) { + printf("corruption: %d bytes; %s\n", + static_cast(bytes), + status.ToString().c_str()); + } +}; + +// Print contents of a log file. (*func)() is called on every record. +bool PrintLogContents(Env* env, const std::string& fname, + void (*func)(Slice)) { + SequentialFile* file; + Status s = env->NewSequentialFile(fname, &file); + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + return false; + } + CorruptionReporter reporter; + log::Reader reader(file, &reporter, true, 0); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch)) { + printf("--- offset %llu; ", + static_cast(reader.LastRecordOffset())); + (*func)(record); + } + delete file; + return true; +} + +// Called on every item found in a WriteBatch. +class WriteBatchItemPrinter : public WriteBatch::Handler { + public: + uint64_t offset_; + uint64_t sequence_; + + virtual void Put(const Slice& key, const Slice& value) { + printf(" put '%s' '%s'\n", + EscapeString(key).c_str(), + EscapeString(value).c_str()); + } + virtual void Delete(const Slice& key) { + printf(" del '%s'\n", + EscapeString(key).c_str()); + } +}; + + +// Called on every log record (each one of which is a WriteBatch) +// found in a kLogFile. +static void WriteBatchPrinter(Slice record) { + if (record.size() < 12) { + printf("log record length %d is too small\n", + static_cast(record.size())); + return; + } + WriteBatch batch; + WriteBatchInternal::SetContents(&batch, record); + printf("sequence %llu\n", + static_cast(WriteBatchInternal::Sequence(&batch))); + WriteBatchItemPrinter batch_item_printer; + Status s = batch.Iterate(&batch_item_printer); + if (!s.ok()) { + printf(" error: %s\n", s.ToString().c_str()); + } +} + +bool DumpLog(Env* env, const std::string& fname) { + return PrintLogContents(env, fname, WriteBatchPrinter); +} + +// Called on every log record (each one of which is a WriteBatch) +// found in a kDescriptorFile. +static void VersionEditPrinter(Slice record) { + VersionEdit edit; + Status s = edit.DecodeFrom(record); + if (!s.ok()) { + printf("%s\n", s.ToString().c_str()); + return; + } + printf("%s", edit.DebugString().c_str()); +} + +bool DumpDescriptor(Env* env, const std::string& fname) { + return PrintLogContents(env, fname, VersionEditPrinter); +} + +bool DumpTable(Env* env, const std::string& fname) { + uint64_t file_size; + RandomAccessFile* file = NULL; + Table* table = NULL; + Status s = env->GetFileSize(fname, &file_size); + if (s.ok()) { + s = env->NewRandomAccessFile(fname, &file); + } + if (s.ok()) { + // We use the default comparator, which may or may not match the + // comparator used in this database. However this should not cause + // problems since we only use Table operations that do not require + // any comparisons. In particular, we do not call Seek or Prev. + s = Table::Open(Options(), file, file_size, &table); + } + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + delete table; + delete file; + return false; + } + + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = table->NewIterator(ro); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey key; + if (!ParseInternalKey(iter->key(), &key)) { + printf("badkey '%s' => '%s'\n", + EscapeString(iter->key()).c_str(), + EscapeString(iter->value()).c_str()); + } else { + char kbuf[20]; + const char* type; + if (key.type == kTypeDeletion) { + type = "del"; + } else if (key.type == kTypeValue) { + type = "val"; + } else { + snprintf(kbuf, sizeof(kbuf), "%d", static_cast(key.type)); + type = kbuf; + } + printf("'%s' @ %8llu : %s => '%s'\n", + EscapeString(key.user_key).c_str(), + static_cast(key.sequence), + type, + EscapeString(iter->value()).c_str()); + } + } + s = iter->status(); + if (!s.ok()) { + printf("iterator error: %s\n", s.ToString().c_str()); + } + + delete iter; + delete table; + delete file; + return true; +} + +bool DumpFile(Env* env, const std::string& fname) { + FileType ftype; + if (!GuessType(fname, &ftype)) { + fprintf(stderr, "%s: unknown file type\n", fname.c_str()); + return false; + } + switch (ftype) { + case kLogFile: return DumpLog(env, fname); + case kDescriptorFile: return DumpDescriptor(env, fname); + case kTableFile: return DumpTable(env, fname); + + default: { + fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str()); + break; + } + } + return false; +} + +bool HandleDumpCommand(Env* env, char** files, int num) { + bool ok = true; + for (int i = 0; i < num; i++) { + ok &= DumpFile(env, files[i]); + } + return ok; +} + +} +} // namespace hyperleveldb + +static void Usage() { + fprintf( + stderr, + "Usage: leveldbutil command...\n" + " dump files... -- dump contents of specified files\n" + ); +} + +int main(int argc, char** argv) { + leveldb::Env* env = leveldb::Env::Default(); + bool ok = true; + if (argc < 2) { + Usage(); + ok = false; + } else { + std::string command = argv[1]; + if (command == "dump") { + ok = leveldb::HandleDumpCommand(env, argv+2, argc-2); + } else { + Usage(); + ok = false; + } + } + return (ok ? 0 : 1); +} diff --git a/Subtrees/hyperleveldb/db/log_format.h b/Subtrees/hyperleveldb/db/log_format.h new file mode 100644 index 0000000000..9637fc8e79 --- /dev/null +++ b/Subtrees/hyperleveldb/db/log_format.h @@ -0,0 +1,35 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Log format information shared by reader and writer. +// See ../doc/log_format.txt for more detail. + +#ifndef STORAGE_HYPERLEVELDB_DB_LOG_FORMAT_H_ +#define STORAGE_HYPERLEVELDB_DB_LOG_FORMAT_H_ + +namespace hyperleveldb { +namespace log { + +enum RecordType { + // Zero is reserved for preallocated files + kZeroType = 0, + + kFullType = 1, + + // For fragments + kFirstType = 2, + kMiddleType = 3, + kLastType = 4 +}; +static const int kMaxRecordType = kLastType; + +static const int kBlockSize = 32768; + +// Header is checksum (4 bytes), type (1 byte), length (2 bytes). +static const int kHeaderSize = 4 + 1 + 2; + +} // namespace log +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_LOG_FORMAT_H_ diff --git a/Subtrees/hyperleveldb/db/log_reader.cc b/Subtrees/hyperleveldb/db/log_reader.cc new file mode 100644 index 0000000000..47d4200dff --- /dev/null +++ b/Subtrees/hyperleveldb/db/log_reader.cc @@ -0,0 +1,259 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "log_reader.h" + +#include +#include "../hyperleveldb/env.h" +#include "../util/coding.h" +#include "../util/crc32c.h" + +namespace hyperleveldb { +namespace log { + +Reader::Reporter::~Reporter() { +} + +Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum, + uint64_t initial_offset) + : file_(file), + reporter_(reporter), + checksum_(checksum), + backing_store_(new char[kBlockSize]), + buffer_(), + eof_(false), + last_record_offset_(0), + end_of_buffer_offset_(0), + initial_offset_(initial_offset) { +} + +Reader::~Reader() { + delete[] backing_store_; +} + +bool Reader::SkipToInitialBlock() { + size_t offset_in_block = initial_offset_ % kBlockSize; + uint64_t block_start_location = initial_offset_ - offset_in_block; + + // Don't search a block if we'd be in the trailer + if (offset_in_block > kBlockSize - 6) { + offset_in_block = 0; + block_start_location += kBlockSize; + } + + end_of_buffer_offset_ = block_start_location; + + // Skip to start of first block that can contain the initial record + if (block_start_location > 0) { + Status skip_status = file_->Skip(block_start_location); + if (!skip_status.ok()) { + ReportDrop(block_start_location, skip_status); + return false; + } + } + + return true; +} + +bool Reader::ReadRecord(Slice* record, std::string* scratch) { + if (last_record_offset_ < initial_offset_) { + if (!SkipToInitialBlock()) { + return false; + } + } + + scratch->clear(); + record->clear(); + bool in_fragmented_record = false; + // Record offset of the logical record that we're reading + // 0 is a dummy value to make compilers happy + uint64_t prospective_record_offset = 0; + + Slice fragment; + while (true) { + uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size(); + const unsigned int record_type = ReadPhysicalRecord(&fragment); + switch (record_type) { + case kFullType: + if (in_fragmented_record) { + // Handle bug in earlier versions of log::Writer where + // it could emit an empty kFirstType record at the tail end + // of a block followed by a kFullType or kFirstType record + // at the beginning of the next block. + if (scratch->empty()) { + in_fragmented_record = false; + } else { + ReportCorruption(scratch->size(), "partial record without end(1)"); + } + } + prospective_record_offset = physical_record_offset; + scratch->clear(); + *record = fragment; + last_record_offset_ = prospective_record_offset; + return true; + + case kFirstType: + if (in_fragmented_record) { + // Handle bug in earlier versions of log::Writer where + // it could emit an empty kFirstType record at the tail end + // of a block followed by a kFullType or kFirstType record + // at the beginning of the next block. + if (scratch->empty()) { + in_fragmented_record = false; + } else { + ReportCorruption(scratch->size(), "partial record without end(2)"); + } + } + prospective_record_offset = physical_record_offset; + scratch->assign(fragment.data(), fragment.size()); + in_fragmented_record = true; + break; + + case kMiddleType: + if (!in_fragmented_record) { + ReportCorruption(fragment.size(), + "missing start of fragmented record(1)"); + } else { + scratch->append(fragment.data(), fragment.size()); + } + break; + + case kLastType: + if (!in_fragmented_record) { + ReportCorruption(fragment.size(), + "missing start of fragmented record(2)"); + } else { + scratch->append(fragment.data(), fragment.size()); + *record = Slice(*scratch); + last_record_offset_ = prospective_record_offset; + return true; + } + break; + + case kEof: + if (in_fragmented_record) { + ReportCorruption(scratch->size(), "partial record without end(3)"); + scratch->clear(); + } + return false; + + case kBadRecord: + if (in_fragmented_record) { + ReportCorruption(scratch->size(), "error in middle of record"); + in_fragmented_record = false; + scratch->clear(); + } + break; + + default: { + char buf[40]; + snprintf(buf, sizeof(buf), "unknown record type %u", record_type); + ReportCorruption( + (fragment.size() + (in_fragmented_record ? scratch->size() : 0)), + buf); + in_fragmented_record = false; + scratch->clear(); + break; + } + } + } + return false; +} + +uint64_t Reader::LastRecordOffset() { + return last_record_offset_; +} + +void Reader::ReportCorruption(size_t bytes, const char* reason) { + ReportDrop(bytes, Status::Corruption(reason)); +} + +void Reader::ReportDrop(size_t bytes, const Status& reason) { + if (reporter_ != NULL && + end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) { + reporter_->Corruption(bytes, reason); + } +} + +unsigned int Reader::ReadPhysicalRecord(Slice* result) { + while (true) { + if (buffer_.size() < kHeaderSize) { + if (!eof_) { + // Last read was a full read, so this is a trailer to skip + buffer_.clear(); + Status status = file_->Read(kBlockSize, &buffer_, backing_store_); + end_of_buffer_offset_ += buffer_.size(); + if (!status.ok()) { + buffer_.clear(); + ReportDrop(kBlockSize, status); + eof_ = true; + return kEof; + } else if (buffer_.size() < kBlockSize) { + eof_ = true; + } + continue; + } else if (buffer_.size() == 0) { + // End of file + return kEof; + } else { + size_t drop_size = buffer_.size(); + buffer_.clear(); + ReportCorruption(drop_size, "truncated record at end of file"); + return kEof; + } + } + + // Parse the header + const char* header = buffer_.data(); + const uint32_t a = static_cast(header[4]) & 0xff; + const uint32_t b = static_cast(header[5]) & 0xff; + const unsigned int type = header[6]; + const uint32_t length = a | (b << 8); + if (kHeaderSize + length > buffer_.size()) { + size_t drop_size = buffer_.size(); + buffer_.clear(); + ReportCorruption(drop_size, "bad record length"); + return kBadRecord; + } + + if (type == kZeroType && length == 0) { + // Skip zero length record without reporting any drops since + // such records are produced by the mmap based writing code in + // env_posix.cc that preallocates file regions. + buffer_.clear(); + return kBadRecord; + } + + // Check crc + if (checksum_) { + uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header)); + uint32_t actual_crc = crc32c::Value(header + 6, 1 + length); + if (actual_crc != expected_crc) { + // Drop the rest of the buffer since "length" itself may have + // been corrupted and if we trust it, we could find some + // fragment of a real log record that just happens to look + // like a valid log record. + size_t drop_size = buffer_.size(); + buffer_.clear(); + ReportCorruption(drop_size, "checksum mismatch"); + return kBadRecord; + } + } + + buffer_.remove_prefix(kHeaderSize + length); + + // Skip physical record that started before initial_offset_ + if (end_of_buffer_offset_ - buffer_.size() - kHeaderSize - length < + initial_offset_) { + result->clear(); + return kBadRecord; + } + + *result = Slice(header + kHeaderSize, length); + return type; + } +} + +} // namespace log +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/log_reader.h b/Subtrees/hyperleveldb/db/log_reader.h new file mode 100644 index 0000000000..44d4948019 --- /dev/null +++ b/Subtrees/hyperleveldb/db/log_reader.h @@ -0,0 +1,108 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_LOG_READER_H_ +#define STORAGE_HYPERLEVELDB_DB_LOG_READER_H_ + +#include + +#include "log_format.h" +#include "../hyperleveldb/slice.h" +#include "../hyperleveldb/status.h" + +namespace hyperleveldb { + +class SequentialFile; + +namespace log { + +class Reader { + public: + // Interface for reporting errors. + class Reporter { + public: + virtual ~Reporter(); + + // Some corruption was detected. "size" is the approximate number + // of bytes dropped due to the corruption. + virtual void Corruption(size_t bytes, const Status& status) = 0; + }; + + // Create a reader that will return log records from "*file". + // "*file" must remain live while this Reader is in use. + // + // If "reporter" is non-NULL, it is notified whenever some data is + // dropped due to a detected corruption. "*reporter" must remain + // live while this Reader is in use. + // + // If "checksum" is true, verify checksums if available. + // + // The Reader will start reading at the first record located at physical + // position >= initial_offset within the file. + Reader(SequentialFile* file, Reporter* reporter, bool checksum, + uint64_t initial_offset); + + ~Reader(); + + // Read the next record into *record. Returns true if read + // successfully, false if we hit end of the input. May use + // "*scratch" as temporary storage. The contents filled in *record + // will only be valid until the next mutating operation on this + // reader or the next mutation to *scratch. + bool ReadRecord(Slice* record, std::string* scratch); + + // Returns the physical offset of the last record returned by ReadRecord. + // + // Undefined before the first call to ReadRecord. + uint64_t LastRecordOffset(); + + private: + SequentialFile* const file_; + Reporter* const reporter_; + bool const checksum_; + char* const backing_store_; + Slice buffer_; + bool eof_; // Last Read() indicated EOF by returning < kBlockSize + + // Offset of the last record returned by ReadRecord. + uint64_t last_record_offset_; + // Offset of the first location past the end of buffer_. + uint64_t end_of_buffer_offset_; + + // Offset at which to start looking for the first record to return + uint64_t const initial_offset_; + + // Extend record types with the following special values + enum { + kEof = kMaxRecordType + 1, + // Returned whenever we find an invalid physical record. + // Currently there are three situations in which this happens: + // * The record has an invalid CRC (ReadPhysicalRecord reports a drop) + // * The record is a 0-length record (No drop is reported) + // * The record is below constructor's initial_offset (No drop is reported) + kBadRecord = kMaxRecordType + 2 + }; + + // Skips all blocks that are completely before "initial_offset_". + // + // Returns true on success. Handles reporting. + bool SkipToInitialBlock(); + + // Return type, or one of the preceding special values + unsigned int ReadPhysicalRecord(Slice* result); + + // Reports dropped bytes to the reporter. + // buffer_ must be updated to remove the dropped bytes prior to invocation. + void ReportCorruption(size_t bytes, const char* reason); + void ReportDrop(size_t bytes, const Status& reason); + + // No copying allowed + Reader(const Reader&); + void operator=(const Reader&); +}; + +} // namespace log +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_LOG_READER_H_ diff --git a/Subtrees/hyperleveldb/db/log_test.cc b/Subtrees/hyperleveldb/db/log_test.cc new file mode 100644 index 0000000000..8df1ac159f --- /dev/null +++ b/Subtrees/hyperleveldb/db/log_test.cc @@ -0,0 +1,509 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "log_reader.h" +#include "log_writer.h" +#include "../hyperleveldb/env.h" +#include "../util/coding.h" +#include "../util/crc32c.h" +#include "../util/random.h" +#include "../util/testharness.h" + +namespace hyperleveldb { +namespace log { + +// Construct a string of the specified length made out of the supplied +// partial string. +static std::string BigString(const std::string& partial_string, size_t n) { + std::string result; + while (result.size() < n) { + result.append(partial_string); + } + result.resize(n); + return result; +} + +// Construct a string from a number +static std::string NumberString(int n) { + char buf[50]; + snprintf(buf, sizeof(buf), "%d.", n); + return std::string(buf); +} + +// Return a skewed potentially long string +static std::string RandomSkewedString(int i, Random* rnd) { + return BigString(NumberString(i), rnd->Skewed(17)); +} + +class LogTest { + private: + class StringDest : public WritableFile { + public: + std::string contents_; + + virtual Status Close() { return Status::OK(); } + virtual Status Flush() { return Status::OK(); } + virtual Status Sync() { return Status::OK(); } + virtual Status WriteAt(uint64_t offset, const Slice& slice) { + std::string tmp = contents_.substr(0, offset); + tmp.append(slice.data(), slice.size()); + if (contents_.size() > offset + slice.size()) { + tmp += contents_.substr(offset + slice.size()); + } + contents_ = tmp; + return Status::OK(); + } + virtual Status Append(const Slice& slice) { + contents_.append(slice.data(), slice.size()); + return Status::OK(); + } + }; + + class StringSource : public SequentialFile { + public: + Slice contents_; + bool force_error_; + bool returned_partial_; + StringSource() : force_error_(false), returned_partial_(false) { } + + virtual Status Read(size_t n, Slice* result, char* scratch) { + ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error"; + + if (force_error_) { + force_error_ = false; + returned_partial_ = true; + return Status::Corruption("read error"); + } + + if (contents_.size() < n) { + n = contents_.size(); + returned_partial_ = true; + } + *result = Slice(contents_.data(), n); + contents_.remove_prefix(n); + return Status::OK(); + } + + virtual Status Skip(uint64_t n) { + if (n > contents_.size()) { + contents_.clear(); + return Status::NotFound("in-memory file skipepd past end"); + } + + contents_.remove_prefix(n); + + return Status::OK(); + } + }; + + class ReportCollector : public Reader::Reporter { + public: + size_t dropped_bytes_; + std::string message_; + + ReportCollector() : dropped_bytes_(0) { } + virtual void Corruption(size_t bytes, const Status& status) { + dropped_bytes_ += bytes; + message_.append(status.ToString()); + } + }; + + StringDest dest_; + StringSource source_; + ReportCollector report_; + bool reading_; + Writer writer_; + Reader reader_; + + // Record metadata for testing initial offset functionality + static size_t initial_offset_record_sizes_[]; + static uint64_t initial_offset_last_record_offsets_[]; + + public: + LogTest() : reading_(false), + writer_(&dest_), + reader_(&source_, &report_, true/*checksum*/, + 0/*initial_offset*/) { + } + + void Write(const std::string& msg) { + ASSERT_TRUE(!reading_) << "Write() after starting to read"; + writer_.AddRecord(Slice(msg)); + } + + size_t WrittenBytes() const { + return dest_.contents_.size(); + } + + std::string Read() { + if (!reading_) { + reading_ = true; + source_.contents_ = Slice(dest_.contents_); + } + std::string scratch; + Slice record; + if (reader_.ReadRecord(&record, &scratch)) { + return record.ToString(); + } else { + return "EOF"; + } + } + + void IncrementByte(int offset, int delta) { + dest_.contents_[offset] += delta; + } + + void SetByte(int offset, char new_byte) { + dest_.contents_[offset] = new_byte; + } + + void ShrinkSize(int bytes) { + dest_.contents_.resize(dest_.contents_.size() - bytes); + } + + void FixChecksum(int header_offset, int len) { + // Compute crc of type/len/data + uint32_t crc = crc32c::Value(&dest_.contents_[header_offset+6], 1 + len); + crc = crc32c::Mask(crc); + EncodeFixed32(&dest_.contents_[header_offset], crc); + } + + void ForceError() { + source_.force_error_ = true; + } + + size_t DroppedBytes() const { + return report_.dropped_bytes_; + } + + std::string ReportMessage() const { + return report_.message_; + } + + // Returns OK iff recorded error message contains "msg" + std::string MatchError(const std::string& msg) const { + if (report_.message_.find(msg) == std::string::npos) { + return report_.message_; + } else { + return "OK"; + } + } + + void WriteInitialOffsetLog() { + for (int i = 0; i < 4; i++) { + std::string record(initial_offset_record_sizes_[i], + static_cast('a' + i)); + Write(record); + } + } + + void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) { + WriteInitialOffsetLog(); + reading_ = true; + source_.contents_ = Slice(dest_.contents_); + Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/, + WrittenBytes() + offset_past_end); + Slice record; + std::string scratch; + ASSERT_TRUE(!offset_reader->ReadRecord(&record, &scratch)); + delete offset_reader; + } + + void CheckInitialOffsetRecord(uint64_t initial_offset, + int expected_record_offset) { + WriteInitialOffsetLog(); + reading_ = true; + source_.contents_ = Slice(dest_.contents_); + Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/, + initial_offset); + Slice record; + std::string scratch; + ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch)); + ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset], + record.size()); + ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset], + offset_reader->LastRecordOffset()); + ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]); + delete offset_reader; + } + +}; + +size_t LogTest::initial_offset_record_sizes_[] = + {10000, // Two sizable records in first block + 10000, + 2 * log::kBlockSize - 1000, // Span three blocks + 1}; + +uint64_t LogTest::initial_offset_last_record_offsets_[] = + {0, + kHeaderSize + 10000, + 2 * (kHeaderSize + 10000), + 2 * (kHeaderSize + 10000) + + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize}; + + +TEST(LogTest, Empty) { + ASSERT_EQ("EOF", Read()); +} + +TEST(LogTest, ReadWrite) { + Write("foo"); + Write("bar"); + Write(""); + Write("xxxx"); + ASSERT_EQ("foo", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("xxxx", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("EOF", Read()); // Make sure reads at eof work +} + +TEST(LogTest, ManyBlocks) { + for (int i = 0; i < 100000; i++) { + Write(NumberString(i)); + } + for (int i = 0; i < 100000; i++) { + ASSERT_EQ(NumberString(i), Read()); + } + ASSERT_EQ("EOF", Read()); +} + +TEST(LogTest, Fragmentation) { + Write("small"); + Write(BigString("medium", 50000)); + Write(BigString("large", 100000)); + ASSERT_EQ("small", Read()); + ASSERT_EQ(BigString("medium", 50000), Read()); + ASSERT_EQ(BigString("large", 100000), Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST(LogTest, MarginalTrailer) { + // Make a trailer that is exactly the same length as an empty record. + const int n = kBlockSize - 2*kHeaderSize; + Write(BigString("foo", n)); + ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes()); + Write(""); + Write("bar"); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST(LogTest, MarginalTrailer2) { + // Make a trailer that is exactly the same length as an empty record. + const int n = kBlockSize - 2*kHeaderSize; + Write(BigString("foo", n)); + ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes()); + Write("bar"); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(0, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); +} + +TEST(LogTest, ShortTrailer) { + const int n = kBlockSize - 2*kHeaderSize + 4; + Write(BigString("foo", n)); + ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); + Write(""); + Write("bar"); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST(LogTest, AlignedEof) { + const int n = kBlockSize - 2*kHeaderSize + 4; + Write(BigString("foo", n)); + ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST(LogTest, RandomRead) { + const int N = 500; + Random write_rnd(301); + for (int i = 0; i < N; i++) { + Write(RandomSkewedString(i, &write_rnd)); + } + Random read_rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_EQ(RandomSkewedString(i, &read_rnd), Read()); + } + ASSERT_EQ("EOF", Read()); +} + +// Tests of all the error paths in log_reader.cc follow: + +TEST(LogTest, ReadError) { + Write("foo"); + ForceError(); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(kBlockSize, DroppedBytes()); + ASSERT_EQ("OK", MatchError("read error")); +} + +TEST(LogTest, BadRecordType) { + Write("foo"); + // Type is stored in header[6] + IncrementByte(6, 100); + FixChecksum(0, 3); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("unknown record type")); +} + +TEST(LogTest, TruncatedTrailingRecord) { + Write("foo"); + ShrinkSize(4); // Drop all payload as well as a header byte + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(kHeaderSize - 1, DroppedBytes()); + ASSERT_EQ("OK", MatchError("truncated record at end of file")); +} + +TEST(LogTest, BadLength) { + Write("foo"); + ShrinkSize(1); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(kHeaderSize + 2, DroppedBytes()); + ASSERT_EQ("OK", MatchError("bad record length")); +} + +TEST(LogTest, ChecksumMismatch) { + Write("foo"); + IncrementByte(0, 10); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(10, DroppedBytes()); + ASSERT_EQ("OK", MatchError("checksum mismatch")); +} + +TEST(LogTest, UnexpectedMiddleType) { + Write("foo"); + SetByte(6, kMiddleType); + FixChecksum(0, 3); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("missing start")); +} + +TEST(LogTest, UnexpectedLastType) { + Write("foo"); + SetByte(6, kLastType); + FixChecksum(0, 3); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("missing start")); +} + +TEST(LogTest, UnexpectedFullType) { + Write("foo"); + Write("bar"); + SetByte(6, kFirstType); + FixChecksum(0, 3); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("partial record without end")); +} + +TEST(LogTest, UnexpectedFirstType) { + Write("foo"); + Write(BigString("bar", 100000)); + SetByte(6, kFirstType); + FixChecksum(0, 3); + ASSERT_EQ(BigString("bar", 100000), Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("partial record without end")); +} + +TEST(LogTest, ErrorJoinsRecords) { + // Consider two fragmented records: + // first(R1) last(R1) first(R2) last(R2) + // where the middle two fragments disappear. We do not want + // first(R1),last(R2) to get joined and returned as a valid record. + + // Write records that span two blocks + Write(BigString("foo", kBlockSize)); + Write(BigString("bar", kBlockSize)); + Write("correct"); + + // Wipe the middle block + for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) { + SetByte(offset, 'x'); + } + + ASSERT_EQ("correct", Read()); + ASSERT_EQ("EOF", Read()); + const int dropped = DroppedBytes(); + ASSERT_LE(dropped, 2*kBlockSize + 100); + ASSERT_GE(dropped, 2*kBlockSize); +} + +TEST(LogTest, ReadStart) { + CheckInitialOffsetRecord(0, 0); +} + +TEST(LogTest, ReadSecondOneOff) { + CheckInitialOffsetRecord(1, 1); +} + +TEST(LogTest, ReadSecondTenThousand) { + CheckInitialOffsetRecord(10000, 1); +} + +TEST(LogTest, ReadSecondStart) { + CheckInitialOffsetRecord(10007, 1); +} + +TEST(LogTest, ReadThirdOneOff) { + CheckInitialOffsetRecord(10008, 2); +} + +TEST(LogTest, ReadThirdStart) { + CheckInitialOffsetRecord(20014, 2); +} + +TEST(LogTest, ReadFourthOneOff) { + CheckInitialOffsetRecord(20015, 3); +} + +TEST(LogTest, ReadFourthFirstBlockTrailer) { + CheckInitialOffsetRecord(log::kBlockSize - 4, 3); +} + +TEST(LogTest, ReadFourthMiddleBlock) { + CheckInitialOffsetRecord(log::kBlockSize + 1, 3); +} + +TEST(LogTest, ReadFourthLastBlock) { + CheckInitialOffsetRecord(2 * log::kBlockSize + 1, 3); +} + +TEST(LogTest, ReadFourthStart) { + CheckInitialOffsetRecord( + 2 * (kHeaderSize + 1000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, + 3); +} + +TEST(LogTest, ReadEnd) { + CheckOffsetPastEndReturnsNoRecords(0); +} + +TEST(LogTest, ReadPastEnd) { + CheckOffsetPastEndReturnsNoRecords(5); +} + +} // namespace log +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/db/log_writer.cc b/Subtrees/hyperleveldb/db/log_writer.cc new file mode 100644 index 0000000000..574888c1b1 --- /dev/null +++ b/Subtrees/hyperleveldb/db/log_writer.cc @@ -0,0 +1,132 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "log_writer.h" + +#include +#include "../hyperleveldb/env.h" +#include "../util/coding.h" +#include "../util/crc32c.h" +#include "../util/mutexlock.h" + +namespace hyperleveldb { +namespace log { + +Writer::Writer(WritableFile* dest) + : dest_(dest), + offset_mtx_(), + offset_(0) { + for (int i = 0; i <= kMaxRecordType; i++) { + char t = static_cast(i); + type_crc_[i] = crc32c::Value(&t, 1); + } +} + +Writer::~Writer() { +} + +Status Writer::AddRecord(const Slice& slice) { + // computation of block_offset requires a pow2 + assert(kBlockSize == 32768); + uint64_t start_offset; + uint64_t end_offset; + + { + MutexLock l(&offset_mtx_); + start_offset = offset_; + end_offset = offset_; + // compute the new offset_ + uint64_t left = slice.size(); + do { + uint64_t block_offset = end_offset & (kBlockSize - 1); + const uint64_t leftover = kBlockSize - block_offset; + assert(leftover > 0); + if (leftover < kHeaderSize) { + end_offset += leftover; + block_offset = 0; + } + // Invariant: we never leave < kHeaderSize bytes in a block. + assert(kBlockSize - block_offset - kHeaderSize >= 0); + + const uint64_t avail = kBlockSize - block_offset - kHeaderSize; + const uint64_t fragment_length = (left < avail) ? left : avail; + + end_offset += kHeaderSize + fragment_length; + left -= fragment_length; + } while (left > 0); + offset_ = end_offset; + } + + const char* ptr = slice.data(); + size_t left = slice.size(); + uint64_t offset = start_offset; + + // Fragment the record if necessary and emit it. Note that if slice + // is empty, we still want to iterate once to emit a single + // zero-length record + Status s; + bool begin = true; + do { + uint64_t block_offset = offset & (kBlockSize - 1); + const uint64_t leftover = kBlockSize - block_offset; + assert(leftover > 0); + if (leftover < kHeaderSize) { + // Switch to a new block + // Fill the trailer (literal below relies on kHeaderSize being 7) + assert(kHeaderSize == 7); + dest_->WriteAt(offset, Slice("\x00\x00\x00\x00\x00\x00", leftover)); + block_offset = 0; + offset += leftover; + } + // Invariant: we never leave < kHeaderSize bytes in a block. + assert(kBlockSize - block_offset - kHeaderSize >= 0); + + const size_t avail = kBlockSize - block_offset - kHeaderSize; + const size_t fragment_length = (left < avail) ? left : avail; + + RecordType type; + const bool end = (left == fragment_length); + if (begin && end) { + type = kFullType; + } else if (begin) { + type = kFirstType; + } else if (end) { + type = kLastType; + } else { + type = kMiddleType; + } + + s = EmitPhysicalRecordAt(type, ptr, offset, fragment_length); + offset += kHeaderSize + fragment_length; + ptr += fragment_length; + left -= fragment_length; + begin = false; + } while (s.ok() && left > 0); + return s; +} + +Status Writer::EmitPhysicalRecordAt(RecordType t, const char* ptr, uint64_t offset, size_t n) { + assert(n <= 0xffff); // Must fit in two bytes + + // Format the header + char buf[kHeaderSize]; + buf[4] = static_cast(n & 0xff); + buf[5] = static_cast(n >> 8); + buf[6] = static_cast(t); + + // Compute the crc of the record type and the payload. + uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n); + crc = crc32c::Mask(crc); // Adjust for storage + EncodeFixed32(buf, crc); + + // Write the header and the payload + Status s = dest_->WriteAt(offset, Slice(buf, kHeaderSize)); + if (s.ok()) { + s = dest_->WriteAt(offset + kHeaderSize, Slice(ptr, n)); + } + return s; +} + +} // namespace log +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/log_writer.h b/Subtrees/hyperleveldb/db/log_writer.h new file mode 100644 index 0000000000..586e7708fe --- /dev/null +++ b/Subtrees/hyperleveldb/db/log_writer.h @@ -0,0 +1,50 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_LOG_WRITER_H_ +#define STORAGE_HYPERLEVELDB_DB_LOG_WRITER_H_ + +#include +#include "log_format.h" +#include "../hyperleveldb/slice.h" +#include "../hyperleveldb/status.h" +#include "../port/port.h" + +namespace hyperleveldb { + +class WritableFile; + +namespace log { + +class Writer { + public: + // Create a writer that will append data to "*dest". + // "*dest" must be initially empty. + // "*dest" must remain live while this Writer is in use. + explicit Writer(WritableFile* dest); + ~Writer(); + + Status AddRecord(const Slice& slice); + + private: + WritableFile* dest_; + port::Mutex offset_mtx_; + uint64_t offset_; // Current offset in file + + // crc32c values for all supported record types. These are + // pre-computed to reduce the overhead of computing the crc of the + // record type stored in the header. + uint32_t type_crc_[kMaxRecordType + 1]; + + Status EmitPhysicalRecordAt(RecordType type, const char* ptr, uint64_t offset, size_t length); + + // No copying allowed + Writer(const Writer&); + void operator=(const Writer&); +}; + +} // namespace log +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_LOG_WRITER_H_ diff --git a/Subtrees/hyperleveldb/db/memtable.cc b/Subtrees/hyperleveldb/db/memtable.cc new file mode 100644 index 0000000000..a20a8e2eaa --- /dev/null +++ b/Subtrees/hyperleveldb/db/memtable.cc @@ -0,0 +1,160 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "memtable.h" +#include "dbformat.h" +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/iterator.h" +#include "../util/coding.h" +#include "../util/mutexlock.h" + +namespace hyperleveldb { + +static Slice GetLengthPrefixedSlice(const char* data) { + uint32_t len; + const char* p = data; + p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted + return Slice(p, len); +} + +MemTable::MemTable(const InternalKeyComparator& cmp) + : comparator_(cmp), + refs_(0), + table_(comparator_, &arena_) { +} + +MemTable::~MemTable() { + assert(refs_ == 0); +} + +size_t MemTable::ApproximateMemoryUsage() { + MutexLock l(&mtx_); + return arena_.MemoryUsage(); +} + +int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr) + const { + // Internal keys are encoded as length-prefixed strings. + Slice a = GetLengthPrefixedSlice(aptr); + Slice b = GetLengthPrefixedSlice(bptr); + return comparator.Compare(a, b); +} + +// Encode a suitable internal key target for "target" and return it. +// Uses *scratch as scratch space, and the returned pointer will point +// into this scratch space. +static const char* EncodeKey(std::string* scratch, const Slice& target) { + scratch->clear(); + PutVarint32(scratch, target.size()); + scratch->append(target.data(), target.size()); + return scratch->data(); +} + +class MemTableIterator: public Iterator { + public: + explicit MemTableIterator(MemTable::Table* table) : iter_(table) { } + + virtual bool Valid() const { return iter_.Valid(); } + virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); } + virtual void SeekToFirst() { iter_.SeekToFirst(); } + virtual void SeekToLast() { iter_.SeekToLast(); } + virtual void Next() { iter_.Next(); } + virtual void Prev() { iter_.Prev(); } + virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); } + virtual Slice value() const { + Slice key_slice = GetLengthPrefixedSlice(iter_.key()); + return GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); + } + + virtual Status status() const { return Status::OK(); } + + private: + MemTable::Table::Iterator iter_; + std::string tmp_; // For passing to EncodeKey + + // No copying allowed + MemTableIterator(const MemTableIterator&); + void operator=(const MemTableIterator&); +}; + +Iterator* MemTable::NewIterator() { + return new MemTableIterator(&table_); +} + +void MemTable::Add(SequenceNumber s, ValueType type, + const Slice& key, + const Slice& value) { + // Format of an entry is concatenation of: + // key_size : varint32 of internal_key.size() + // key bytes : char[internal_key.size()] + // value_size : varint32 of value.size() + // value bytes : char[value.size()] + size_t key_size = key.size(); + size_t val_size = value.size(); + size_t internal_key_size = key_size + 8; + const size_t encoded_len = + VarintLength(internal_key_size) + internal_key_size + + VarintLength(val_size) + val_size; + char* buf = NULL; + + { + MutexLock l(&mtx_); + buf = arena_.Allocate(encoded_len); + } + + char* p = EncodeVarint32(buf, internal_key_size); + memcpy(p, key.data(), key_size); + p += key_size; + EncodeFixed64(p, (s << 8) | type); + p += 8; + p = EncodeVarint32(p, val_size); + memcpy(p, value.data(), val_size); + assert((p + val_size) - buf == encoded_len); + Table::InsertHint ih(&table_, buf); + + { + MutexLock l(&mtx_); + table_.InsertWithHint(&ih, buf); + } +} + +bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) { + Slice memkey = key.memtable_key(); + Table::Iterator iter(&table_); + iter.Seek(memkey.data()); + if (iter.Valid()) { + // entry format is: + // klength varint32 + // userkey char[klength] + // tag uint64 + // vlength varint32 + // value char[vlength] + // Check that it belongs to same user key. We do not check the + // sequence number since the Seek() call above should have skipped + // all entries with overly large sequence numbers. + const char* entry = iter.key(); + uint32_t key_length; + const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length); + if (comparator_.comparator.user_comparator()->Compare( + Slice(key_ptr, key_length - 8), + key.user_key()) == 0) { + // Correct user key + const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); + switch (static_cast(tag & 0xff)) { + case kTypeValue: { + Slice v = GetLengthPrefixedSlice(key_ptr + key_length); + value->assign(v.data(), v.size()); + return true; + } + case kTypeDeletion: + *s = Status::NotFound(Slice()); + return true; + } + } + } + return false; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/memtable.h b/Subtrees/hyperleveldb/db/memtable.h new file mode 100644 index 0000000000..834b43b6db --- /dev/null +++ b/Subtrees/hyperleveldb/db/memtable.h @@ -0,0 +1,94 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_MEMTABLE_H_ +#define STORAGE_HYPERLEVELDB_DB_MEMTABLE_H_ + +#include +#include "../hyperleveldb/db.h" +#include "dbformat.h" +#include "skiplist.h" +#include "../util/arena.h" + +namespace hyperleveldb { + +class InternalKeyComparator; +class Mutex; +class MemTableIterator; + +class MemTable { + public: + // MemTables are reference counted. The initial reference count + // is zero and the caller must call Ref() at least once. + explicit MemTable(const InternalKeyComparator& comparator); + + // Increase reference count. + // XXX use a release increment if not using GCC intrinsics + void Ref() { __sync_add_and_fetch(&refs_, 1); } + + // Drop reference count. Delete if no more references exist. + // XXX use an acquire decrement if not using GCC intrinsics + void Unref() { + int refs = __sync_sub_and_fetch(&refs_, 1); + assert(refs >= 0); + if (refs <= 0) { + delete this; + } + } + + // Returns an estimate of the number of bytes of data in use by this + // data structure. + // + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable. + size_t ApproximateMemoryUsage(); + + // Return an iterator that yields the contents of the memtable. + // + // The caller must ensure that the underlying MemTable remains live + // while the returned iterator is live. The keys returned by this + // iterator are internal keys encoded by AppendInternalKey in the + // db/format.{h,cc} module. + Iterator* NewIterator(); + + // Add an entry into memtable that maps key to value at the + // specified sequence number and with the specified type. + // Typically value will be empty if type==kTypeDeletion. + void Add(SequenceNumber seq, ValueType type, + const Slice& key, + const Slice& value); + + // If memtable contains a value for key, store it in *value and return true. + // If memtable contains a deletion for key, store a NotFound() error + // in *status and return true. + // Else, return false. + bool Get(const LookupKey& key, std::string* value, Status* s); + + private: + ~MemTable(); // Private since only Unref() should be used to delete it + + struct KeyComparator { + const InternalKeyComparator comparator; + explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { } + int operator()(const char* a, const char* b) const; + }; + friend class MemTableIterator; + friend class MemTableBackwardIterator; + + typedef SkipList Table; + + KeyComparator comparator_; + int refs_; + port::Mutex mtx_; + Arena arena_; + Table table_; + + // No copying allowed + MemTable(const MemTable&); + void operator=(const MemTable&); +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_MEMTABLE_H_ diff --git a/Subtrees/hyperleveldb/db/repair.cc b/Subtrees/hyperleveldb/db/repair.cc new file mode 100644 index 0000000000..e366fcbc0b --- /dev/null +++ b/Subtrees/hyperleveldb/db/repair.cc @@ -0,0 +1,389 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// We recover the contents of the descriptor from the other files we find. +// (1) Any log files are first converted to tables +// (2) We scan every table to compute +// (a) smallest/largest for the table +// (b) largest sequence number in the table +// (3) We generate descriptor contents: +// - log number is set to zero +// - next-file-number is set to 1 + largest file number we found +// - last-sequence-number is set to largest sequence# found across +// all tables (see 2c) +// - compaction pointers are cleared +// - every table file is added at level 0 +// +// Possible optimization 1: +// (a) Compute total size and use to pick appropriate max-level M +// (b) Sort tables by largest sequence# in the table +// (c) For each table: if it overlaps earlier table, place in level-0, +// else place in level-M. +// Possible optimization 2: +// Store per-table metadata (smallest, largest, largest-seq#, ...) +// in the table's meta section to speed up ScanTable. + +#include "builder.h" +#include "db_impl.h" +#include "dbformat.h" +#include "filename.h" +#include "log_reader.h" +#include "log_writer.h" +#include "memtable.h" +#include "table_cache.h" +#include "version_edit.h" +#include "write_batch_internal.h" +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/db.h" +#include "../hyperleveldb/env.h" + +namespace hyperleveldb { + +namespace { + +class Repairer { + public: + Repairer(const std::string& dbname, const Options& options) + : dbname_(dbname), + env_(options.env), + icmp_(options.comparator), + ipolicy_(options.filter_policy), + options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)), + owns_info_log_(options_.info_log != options.info_log), + owns_cache_(options_.block_cache != options.block_cache), + next_file_number_(1) { + // TableCache can be small since we expect each table to be opened once. + table_cache_ = new TableCache(dbname_, &options_, 10); + } + + ~Repairer() { + delete table_cache_; + if (owns_info_log_) { + delete options_.info_log; + } + if (owns_cache_) { + delete options_.block_cache; + } + } + + Status Run() { + Status status = FindFiles(); + if (status.ok()) { + ConvertLogFilesToTables(); + ExtractMetaData(); + status = WriteDescriptor(); + } + if (status.ok()) { + unsigned long long bytes = 0; + for (size_t i = 0; i < tables_.size(); i++) { + bytes += tables_[i].meta.file_size; + } + Log(options_.info_log, + "**** Repaired leveldb %s; " + "recovered %d files; %llu bytes. " + "Some data may have been lost. " + "****", + dbname_.c_str(), + static_cast(tables_.size()), + bytes); + } + return status; + } + + private: + struct TableInfo { + FileMetaData meta; + SequenceNumber max_sequence; + }; + + std::string const dbname_; + Env* const env_; + InternalKeyComparator const icmp_; + InternalFilterPolicy const ipolicy_; + Options const options_; + bool owns_info_log_; + bool owns_cache_; + TableCache* table_cache_; + VersionEdit edit_; + + std::vector manifests_; + std::vector table_numbers_; + std::vector logs_; + std::vector tables_; + uint64_t next_file_number_; + + Status FindFiles() { + std::vector filenames; + Status status = env_->GetChildren(dbname_, &filenames); + if (!status.ok()) { + return status; + } + if (filenames.empty()) { + return Status::IOError(dbname_, "repair found no files"); + } + + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type)) { + if (type == kDescriptorFile) { + manifests_.push_back(filenames[i]); + } else { + if (number + 1 > next_file_number_) { + next_file_number_ = number + 1; + } + if (type == kLogFile) { + logs_.push_back(number); + } else if (type == kTableFile) { + table_numbers_.push_back(number); + } else { + // Ignore other files + } + } + } + } + return status; + } + + void ConvertLogFilesToTables() { + for (size_t i = 0; i < logs_.size(); i++) { + std::string logname = LogFileName(dbname_, logs_[i]); + Status status = ConvertLogToTable(logs_[i]); + if (!status.ok()) { + Log(options_.info_log, "Log #%llu: ignoring conversion error: %s", + (unsigned long long) logs_[i], + status.ToString().c_str()); + } + ArchiveFile(logname); + } + } + + Status ConvertLogToTable(uint64_t log) { + struct LogReporter : public log::Reader::Reporter { + Env* env; + Logger* info_log; + uint64_t lognum; + virtual void Corruption(size_t bytes, const Status& s) { + // We print error messages for corruption, but continue repairing. + Log(info_log, "Log #%llu: dropping %d bytes; %s", + (unsigned long long) lognum, + static_cast(bytes), + s.ToString().c_str()); + } + }; + + // Open the log file + std::string logname = LogFileName(dbname_, log); + SequentialFile* lfile; + Status status = env_->NewSequentialFile(logname, &lfile); + if (!status.ok()) { + return status; + } + + // Create the log reader. + LogReporter reporter; + reporter.env = env_; + reporter.info_log = options_.info_log; + reporter.lognum = log; + // We intentially make log::Reader do checksumming so that + // corruptions cause entire commits to be skipped instead of + // propagating bad information (like overly large sequence + // numbers). + log::Reader reader(lfile, &reporter, false/*do not checksum*/, + 0/*initial_offset*/); + + // Read all the records and add to a memtable + std::string scratch; + Slice record; + WriteBatch batch; + MemTable* mem = new MemTable(icmp_); + mem->Ref(); + int counter = 0; + while (reader.ReadRecord(&record, &scratch)) { + if (record.size() < 12) { + reporter.Corruption( + record.size(), Status::Corruption("log record too small")); + continue; + } + WriteBatchInternal::SetContents(&batch, record); + status = WriteBatchInternal::InsertInto(&batch, mem); + if (status.ok()) { + counter += WriteBatchInternal::Count(&batch); + } else { + Log(options_.info_log, "Log #%llu: ignoring %s", + (unsigned long long) log, + status.ToString().c_str()); + status = Status::OK(); // Keep going with rest of file + } + } + delete lfile; + + // Do not record a version edit for this conversion to a Table + // since ExtractMetaData() will also generate edits. + FileMetaData meta; + meta.number = next_file_number_++; + Iterator* iter = mem->NewIterator(); + status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta); + delete iter; + mem->Unref(); + mem = NULL; + if (status.ok()) { + if (meta.file_size > 0) { + table_numbers_.push_back(meta.number); + } + } + Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s", + (unsigned long long) log, + counter, + (unsigned long long) meta.number, + status.ToString().c_str()); + return status; + } + + void ExtractMetaData() { + std::vector kept; + for (size_t i = 0; i < table_numbers_.size(); i++) { + TableInfo t; + t.meta.number = table_numbers_[i]; + Status status = ScanTable(&t); + if (!status.ok()) { + std::string fname = TableFileName(dbname_, table_numbers_[i]); + Log(options_.info_log, "Table #%llu: ignoring %s", + (unsigned long long) table_numbers_[i], + status.ToString().c_str()); + ArchiveFile(fname); + } else { + tables_.push_back(t); + } + } + } + + Status ScanTable(TableInfo* t) { + std::string fname = TableFileName(dbname_, t->meta.number); + int counter = 0; + Status status = env_->GetFileSize(fname, &t->meta.file_size); + if (status.ok()) { + Iterator* iter = table_cache_->NewIterator( + ReadOptions(), t->meta.number, t->meta.file_size); + bool empty = true; + ParsedInternalKey parsed; + t->max_sequence = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + Slice key = iter->key(); + if (!ParseInternalKey(key, &parsed)) { + Log(options_.info_log, "Table #%llu: unparsable key %s", + (unsigned long long) t->meta.number, + EscapeString(key).c_str()); + continue; + } + + counter++; + if (empty) { + empty = false; + t->meta.smallest.DecodeFrom(key); + } + t->meta.largest.DecodeFrom(key); + if (parsed.sequence > t->max_sequence) { + t->max_sequence = parsed.sequence; + } + } + if (!iter->status().ok()) { + status = iter->status(); + } + delete iter; + } + Log(options_.info_log, "Table #%llu: %d entries %s", + (unsigned long long) t->meta.number, + counter, + status.ToString().c_str()); + return status; + } + + Status WriteDescriptor() { + std::string tmp = TempFileName(dbname_, 1); + WritableFile* file; + Status status = env_->NewWritableFile(tmp, &file); + if (!status.ok()) { + return status; + } + + SequenceNumber max_sequence = 0; + for (size_t i = 0; i < tables_.size(); i++) { + if (max_sequence < tables_[i].max_sequence) { + max_sequence = tables_[i].max_sequence; + } + } + + edit_.SetComparatorName(icmp_.user_comparator()->Name()); + edit_.SetLogNumber(0); + edit_.SetNextFile(next_file_number_); + edit_.SetLastSequence(max_sequence); + + for (size_t i = 0; i < tables_.size(); i++) { + // TODO(opt): separate out into multiple levels + const TableInfo& t = tables_[i]; + edit_.AddFile(0, t.meta.number, t.meta.file_size, + t.meta.smallest, t.meta.largest); + } + + //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str()); + { + log::Writer log(file); + std::string record; + edit_.EncodeTo(&record); + status = log.AddRecord(record); + } + if (status.ok()) { + status = file->Close(); + } + delete file; + file = NULL; + + if (!status.ok()) { + env_->DeleteFile(tmp); + } else { + // Discard older manifests + for (size_t i = 0; i < manifests_.size(); i++) { + ArchiveFile(dbname_ + "/" + manifests_[i]); + } + + // Install new manifest + status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1)); + if (status.ok()) { + status = SetCurrentFile(env_, dbname_, 1); + } else { + env_->DeleteFile(tmp); + } + } + return status; + } + + void ArchiveFile(const std::string& fname) { + // Move into another directory. E.g., for + // dir/foo + // rename to + // dir/lost/foo + const char* slash = strrchr(fname.c_str(), '/'); + std::string new_dir; + if (slash != NULL) { + new_dir.assign(fname.data(), slash - fname.data()); + } + new_dir.append("/lost"); + env_->CreateDir(new_dir); // Ignore error + std::string new_file = new_dir; + new_file.append("/"); + new_file.append((slash == NULL) ? fname.c_str() : slash + 1); + Status s = env_->RenameFile(fname, new_file); + Log(options_.info_log, "Archiving %s: %s\n", + fname.c_str(), s.ToString().c_str()); + } +}; +} // namespace + +Status RepairDB(const std::string& dbname, const Options& options) { + Repairer repairer(dbname, options); + return repairer.Run(); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/skiplist.h b/Subtrees/hyperleveldb/db/skiplist.h new file mode 100644 index 0000000000..12604ae337 --- /dev/null +++ b/Subtrees/hyperleveldb/db/skiplist.h @@ -0,0 +1,460 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Thread safety +// ------------- +// +// Writes require external synchronization, most likely a mutex. +// Reads require a guarantee that the SkipList will not be destroyed +// while the read is in progress. Apart from that, reads progress +// without any internal locking or synchronization. +// +// Invariants: +// +// (1) Allocated nodes are never deleted until the SkipList is +// destroyed. This is trivially guaranteed by the code since we +// never delete any skip list nodes. +// +// (2) The contents of a Node except for the next/prev pointers are +// immutable after the Node has been linked into the SkipList. +// Only Insert() modifies the list, and it is careful to initialize +// a node and use release-stores to publish the nodes in one or +// more lists. +// +// ... prev vs. next pointer ordering ... + +#include +#include +#include "../port/port.h" +#include "../util/arena.h" +#include "../util/random.h" + +namespace hyperleveldb { + +class Arena; + +template +class SkipList { + private: + struct Node; + enum { kMaxHeight = 12 }; + + public: + // Create a new SkipList object that will use "cmp" for comparing keys, + // and will allocate memory using "*arena". Objects allocated in the arena + // must remain allocated for the lifetime of the skiplist object. + explicit SkipList(Comparator cmp, Arena* arena); + + // Insert key into the list. + // REQUIRES: nothing that compares equal to key is currently in the list. + // REQUIRES: external synchronization. + void Insert(const Key& key); + + // Insert key into the list using the iterator as a hint. + // REQUIRES: nothing that compares equal to key is currently in the list. + // REQUIRES: external synchronization. + class InsertHint; + void InsertWithHint(InsertHint* ih, const Key& key); + + // Returns true iff an entry that compares equal to key is in the list. + bool Contains(const Key& key) const; + + // Perform expensive iteration over the skip list prior to insert so that the + // cost of a synchronized insert is reduced when the structure is full. + // REQUIRES: same synchronization as is necessary for a read. + class InsertHint { + public: + InsertHint(const SkipList* list, const Key& key); + + private: + const SkipList* list_; + Node* x_; + Node* prev_[kMaxHeight]; + Node* obs_[kMaxHeight]; + + // No copying allowed + InsertHint(const InsertHint&); + void operator=(const InsertHint&); + friend class SkipList; + }; + + // Iteration over the contents of a skip list + class Iterator { + public: + // Initialize an iterator over the specified list. + // The returned iterator is not valid. + explicit Iterator(const SkipList* list); + + // Returns true iff the iterator is positioned at a valid node. + bool Valid() const; + + // Returns the key at the current position. + // REQUIRES: Valid() + const Key& key() const; + + // Advances to the next position. + // REQUIRES: Valid() + void Next(); + + // Advances to the previous position. + // REQUIRES: Valid() + void Prev(); + + // Advance to the first entry with a key >= target + void Seek(const Key& target); + + // Position at the first entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToFirst(); + + // Position at the last entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToLast(); + + private: + const SkipList* list_; + Node* node_; + // Intentionally copyable + }; + + private: + // Immutable after construction + Comparator const compare_; + Arena* const arena_; // Arena used for allocations of nodes + + Node* const head_; + + // Modified only by Insert(). Read racily by readers, but stale + // values are ok. + port::AtomicPointer max_height_; // Height of the entire list + + inline int GetMaxHeight() const { + return static_cast( + reinterpret_cast(max_height_.NoBarrier_Load())); + } + + // Read/written only by Insert(). + Random rnd_; + + Node* NewNode(const Key& key, int height); + int RandomHeight(); + bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } + + // Return true if key is greater than the data stored in "n" + bool KeyIsAfterNode(const Key& key, Node* n) const; + + // Return the earliest node that comes at or after key. + // Return NULL if there is no such node. + // + // If prev is non-NULL, fills prev[level] with pointer to previous + // node at "level" for every level in [0..max_height_-1]. + Node* FindGreaterOrEqual(const Key& key, Node** prev, Node** obs) const; + + // Return the latest node with a key < key. + // Return head_ if there is no such node. + Node* FindLessThan(const Key& key) const; + + // Return the last node in the list. + // Return head_ if list is empty. + Node* FindLast() const; + + // Update the state of the InsertHint to reflect the latest values + void UpdateHint(InsertHint* ih, const Key& k); + + // No copying allowed + SkipList(const SkipList&); + void operator=(const SkipList&); +}; + +// Implementation details follow +template +struct SkipList::Node { + explicit Node(const Key& k) : key(k) { } + + Key const key; + + // Accessors/mutators for links. Wrapped in methods so we can + // add the appropriate barriers as necessary. + Node* Next(int n) { + assert(n >= 0); + // Use an 'acquire load' so that we observe a fully initialized + // version of the returned Node. + return reinterpret_cast(next_[n].Acquire_Load()); + } + void SetNext(int n, Node* x) { + assert(n >= 0); + // Use a 'release store' so that anybody who reads through this + // pointer observes a fully initialized version of the inserted node. + next_[n].Release_Store(x); + } + + // No-barrier variants that can be safely used in a few locations. + Node* NoBarrier_Next(int n) { + assert(n >= 0); + return reinterpret_cast(next_[n].NoBarrier_Load()); + } + void NoBarrier_SetNext(int n, Node* x) { + assert(n >= 0); + next_[n].NoBarrier_Store(x); + } + + private: + // Array of length equal to the node height. next_[0] is lowest level link. + port::AtomicPointer next_[1]; +}; + +template +typename SkipList::Node* +SkipList::NewNode(const Key& key, int height) { + char* mem = arena_->AllocateAligned( + sizeof(Node) + sizeof(port::AtomicPointer) * (height - 1)); + return new (mem) Node(key); +} + +template +inline SkipList::Iterator::Iterator(const SkipList* list) { + list_ = list; + node_ = NULL; +} + +template +inline bool SkipList::Iterator::Valid() const { + return node_ != NULL; +} + +template +inline const Key& SkipList::Iterator::key() const { + assert(Valid()); + return node_->key; +} + +template +inline void SkipList::Iterator::Next() { + assert(Valid()); + node_ = node_->Next(0); +} + +template +inline void SkipList::Iterator::Prev() { + // Instead of using explicit "prev" links, we just search for the + // last node that falls before key. + assert(Valid()); + node_ = list_->FindLessThan(node_->key); + if (node_ == list_->head_) { + node_ = NULL; + } +} + +template +inline void SkipList::Iterator::Seek(const Key& target) { + node_ = list_->FindGreaterOrEqual(target, NULL, NULL); +} + +template +inline void SkipList::Iterator::SeekToFirst() { + node_ = list_->head_->Next(0); +} + +template +inline void SkipList::Iterator::SeekToLast() { + node_ = list_->FindLast(); + if (node_ == list_->head_) { + node_ = NULL; + } +} + +template +int SkipList::RandomHeight() { + // Increase height with probability 1 in kBranching + static const unsigned int kBranching = 4; + int height = 1; + while (height < kMaxHeight && ((rnd_.Next() % kBranching) == 0)) { + height++; + } + assert(height > 0); + assert(height <= kMaxHeight); + return height; +} + +template +bool SkipList::KeyIsAfterNode(const Key& key, Node* n) const { + // NULL n is considered infinite + return (n != NULL) && (compare_(n->key, key) < 0); +} + +template +typename SkipList::Node* SkipList::FindGreaterOrEqual(const Key& key, Node** prev, Node** obs) + const { + Node* x = head_; + int level = GetMaxHeight() - 1; + while (true) { + Node* next = x->Next(level); + if (KeyIsAfterNode(key, next)) { + // Keep searching in this list + x = next; + } else { + if (prev != NULL) prev[level] = x; + if (obs != NULL) obs[level] = next; + if (level == 0) { + return next; + } else { + // Switch to next list + level--; + } + } + } +} + +template +typename SkipList::Node* +SkipList::FindLessThan(const Key& key) const { + Node* x = head_; + int level = GetMaxHeight() - 1; + while (true) { + assert(x == head_ || compare_(x->key, key) < 0); + Node* next = x->Next(level); + if (next == NULL || compare_(next->key, key) >= 0) { + if (level == 0) { + return x; + } else { + // Switch to next list + level--; + } + } else { + x = next; + } + } +} + +template +typename SkipList::Node* SkipList::FindLast() + const { + Node* x = head_; + int level = GetMaxHeight() - 1; + while (true) { + Node* next = x->Next(level); + if (next == NULL) { + if (level == 0) { + return x; + } else { + // Switch to next list + level--; + } + } else { + x = next; + } + } +} + +template +SkipList::SkipList(Comparator cmp, Arena* arena) + : compare_(cmp), + arena_(arena), + head_(NewNode(0 /* any key will do */, kMaxHeight)), + max_height_(reinterpret_cast(1)), + rnd_(0xdeadbeef) { + for (int i = 0; i < kMaxHeight; i++) { + head_->SetNext(i, NULL); + } +} + +template +void SkipList::Insert(const Key& key) { + InsertHint ih(this, key); + return InsertWithHint(&ih, key); +} + +template +SkipList::InsertHint::InsertHint(const SkipList* list, const Key& key) + : list_(list), + x_(NULL) { + for (int i = 0; i < kMaxHeight; ++i) + { + prev_[i] = list_->head_; + obs_[i] = NULL; + } + x_ = list_->FindGreaterOrEqual(key, prev_, obs_); +} + +template +void SkipList::UpdateHint(InsertHint* ih, const Key& key) { + // TODO(opt): We can be smarter here by using the skip list structure to + // advance. It's assumed that a small number of insertions to the SkipList + // happen between the time ih was created and now. + for (int level = 0; level < kMaxHeight; ++level) { + Node* x = ih->prev_[level]; + while (true) { + Node* next = x->Next(level); + if (next == ih->obs_[level] || !KeyIsAfterNode(key, next)) { + ih->prev_[level] = x; + ih->obs_[level] = next; + break; + } + x = next; + } + } + ih->x_ = ih->obs_[0]; +} + +template +void SkipList::InsertWithHint(InsertHint* ih, const Key& key) { + // Advance pointers to account for any data written between the creation of + // the InsertHint and this call. + UpdateHint(ih, key); + Node* prev[kMaxHeight]; + Node* x = ih->x_; + for (int i = 0; i < kMaxHeight; ++i) { + prev[i] = ih->prev_[i]; + } + +#if 0 + Node* check_prev[kMaxHeight]; + Node* check_x = FindGreaterOrEqual(key, check_prev, NULL); + + for (int i = 0; i < GetMaxHeight(); ++i) { + assert(check_prev[i] == prev[i]); + assert(check_x == x); + } +#endif + + // Our data structure does not allow duplicate insertion + assert(x == NULL || !Equal(key, x->key)); + + int height = RandomHeight(); + if (height > GetMaxHeight()) { + for (int i = GetMaxHeight(); i < height; i++) { + prev[i] = head_; + } + //fprintf(stderr, "Change height from %d to %d\n", max_height_, height); + + // It is ok to mutate max_height_ without any synchronization + // with concurrent readers. A concurrent reader that observes + // the new value of max_height_ will see either the old value of + // new level pointers from head_ (NULL), or a new value set in + // the loop below. In the former case the reader will + // immediately drop to the next level since NULL sorts after all + // keys. In the latter case the reader will use the new node. + max_height_.NoBarrier_Store(reinterpret_cast(height)); + } + + x = NewNode(key, height); + for (int i = 0; i < height; i++) { + // NoBarrier_SetNext() suffices since we will add a barrier when + // we publish a pointer to "x" in prev[i]. + x->NoBarrier_SetNext(i, prev[i]->NoBarrier_Next(i)); + prev[i]->SetNext(i, x); + } +} + +template +bool SkipList::Contains(const Key& key) const { + Node* x = FindGreaterOrEqual(key, NULL, NULL); + if (x != NULL && Equal(key, x->key)) { + return true; + } else { + return false; + } +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/skiplist_test.cc b/Subtrees/hyperleveldb/db/skiplist_test.cc new file mode 100644 index 0000000000..50e0da74dd --- /dev/null +++ b/Subtrees/hyperleveldb/db/skiplist_test.cc @@ -0,0 +1,378 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "skiplist.h" +#include +#include "../hyperleveldb/env.h" +#include "../util/arena.h" +#include "../util/hash.h" +#include "../util/random.h" +#include "../util/testharness.h" + +namespace hyperleveldb { + +typedef uint64_t Key; + +struct Comparator { + int operator()(const Key& a, const Key& b) const { + if (a < b) { + return -1; + } else if (a > b) { + return +1; + } else { + return 0; + } + } +}; + +class SkipTest { }; + +TEST(SkipTest, Empty) { + Arena arena; + Comparator cmp; + SkipList list(cmp, &arena); + ASSERT_TRUE(!list.Contains(10)); + + SkipList::Iterator iter(&list); + ASSERT_TRUE(!iter.Valid()); + iter.SeekToFirst(); + ASSERT_TRUE(!iter.Valid()); + iter.Seek(100); + ASSERT_TRUE(!iter.Valid()); + iter.SeekToLast(); + ASSERT_TRUE(!iter.Valid()); +} + +TEST(SkipTest, InsertAndLookup) { + const int N = 2000; + const int R = 5000; + Random rnd(1000); + std::set keys; + Arena arena; + Comparator cmp; + SkipList list(cmp, &arena); + for (int i = 0; i < N; i++) { + Key key = rnd.Next() % R; + if (keys.insert(key).second) { + list.Insert(key); + } + } + + for (int i = 0; i < R; i++) { + if (list.Contains(i)) { + ASSERT_EQ(keys.count(i), 1); + } else { + ASSERT_EQ(keys.count(i), 0); + } + } + + // Simple iterator tests + { + SkipList::Iterator iter(&list); + ASSERT_TRUE(!iter.Valid()); + + iter.Seek(0); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.begin()), iter.key()); + + iter.SeekToFirst(); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.begin()), iter.key()); + + iter.SeekToLast(); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.rbegin()), iter.key()); + } + + // Forward iteration test + for (int i = 0; i < R; i++) { + SkipList::Iterator iter(&list); + iter.Seek(i); + + // Compare against model iterator + std::set::iterator model_iter = keys.lower_bound(i); + for (int j = 0; j < 3; j++) { + if (model_iter == keys.end()) { + ASSERT_TRUE(!iter.Valid()); + break; + } else { + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*model_iter, iter.key()); + ++model_iter; + iter.Next(); + } + } + } + + // Backward iteration test + { + SkipList::Iterator iter(&list); + iter.SeekToLast(); + + // Compare against model iterator + for (std::set::reverse_iterator model_iter = keys.rbegin(); + model_iter != keys.rend(); + ++model_iter) { + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*model_iter, iter.key()); + iter.Prev(); + } + ASSERT_TRUE(!iter.Valid()); + } +} + +// We want to make sure that with a single writer and multiple +// concurrent readers (with no synchronization other than when a +// reader's iterator is created), the reader always observes all the +// data that was present in the skip list when the iterator was +// constructor. Because insertions are happening concurrently, we may +// also observe new values that were inserted since the iterator was +// constructed, but we should never miss any values that were present +// at iterator construction time. +// +// We generate multi-part keys: +// +// where: +// key is in range [0..K-1] +// gen is a generation number for key +// hash is hash(key,gen) +// +// The insertion code picks a random key, sets gen to be 1 + the last +// generation number inserted for that key, and sets hash to Hash(key,gen). +// +// At the beginning of a read, we snapshot the last inserted +// generation number for each key. We then iterate, including random +// calls to Next() and Seek(). For every key we encounter, we +// check that it is either expected given the initial snapshot or has +// been concurrently added since the iterator started. +class ConcurrentTest { + private: + static const uint32_t K = 4; + + static uint64_t key(Key key) { return (key >> 40); } + static uint64_t gen(Key key) { return (key >> 8) & 0xffffffffu; } + static uint64_t hash(Key key) { return key & 0xff; } + + static uint64_t HashNumbers(uint64_t k, uint64_t g) { + uint64_t data[2] = { k, g }; + return Hash(reinterpret_cast(data), sizeof(data), 0); + } + + static Key MakeKey(uint64_t k, uint64_t g) { + assert(sizeof(Key) == sizeof(uint64_t)); + assert(k <= K); // We sometimes pass K to seek to the end of the skiplist + assert(g <= 0xffffffffu); + return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff)); + } + + static bool IsValidKey(Key k) { + return hash(k) == (HashNumbers(key(k), gen(k)) & 0xff); + } + + static Key RandomTarget(Random* rnd) { + switch (rnd->Next() % 10) { + case 0: + // Seek to beginning + return MakeKey(0, 0); + case 1: + // Seek to end + return MakeKey(K, 0); + default: + // Seek to middle + return MakeKey(rnd->Next() % K, 0); + } + } + + // Per-key generation + struct State { + port::AtomicPointer generation[K]; + void Set(int k, intptr_t v) { + generation[k].Release_Store(reinterpret_cast(v)); + } + intptr_t Get(int k) { + return reinterpret_cast(generation[k].Acquire_Load()); + } + + State() { + for (int k = 0; k < K; k++) { + Set(k, 0); + } + } + }; + + // Current state of the test + State current_; + + Arena arena_; + + // SkipList is not protected by mu_. We just use a single writer + // thread to modify it. + SkipList list_; + + public: + ConcurrentTest() : list_(Comparator(), &arena_) { } + + // REQUIRES: External synchronization + void WriteStep(Random* rnd) { + const uint32_t k = rnd->Next() % K; + const intptr_t g = current_.Get(k) + 1; + const Key key = MakeKey(k, g); + list_.Insert(key); + current_.Set(k, g); + } + + void ReadStep(Random* rnd) { + // Remember the initial committed state of the skiplist. + State initial_state; + for (int k = 0; k < K; k++) { + initial_state.Set(k, current_.Get(k)); + } + + Key pos = RandomTarget(rnd); + SkipList::Iterator iter(&list_); + iter.Seek(pos); + while (true) { + Key current; + if (!iter.Valid()) { + current = MakeKey(K, 0); + } else { + current = iter.key(); + ASSERT_TRUE(IsValidKey(current)) << current; + } + ASSERT_LE(pos, current) << "should not go backwards"; + + // Verify that everything in [pos,current) was not present in + // initial_state. + while (pos < current) { + ASSERT_LT(key(pos), K) << pos; + + // Note that generation 0 is never inserted, so it is ok if + // <*,0,*> is missing. + ASSERT_TRUE((gen(pos) == 0) || + (gen(pos) > initial_state.Get(key(pos))) + ) << "key: " << key(pos) + << "; gen: " << gen(pos) + << "; initgen: " + << initial_state.Get(key(pos)); + + // Advance to next key in the valid key space + if (key(pos) < key(current)) { + pos = MakeKey(key(pos) + 1, 0); + } else { + pos = MakeKey(key(pos), gen(pos) + 1); + } + } + + if (!iter.Valid()) { + break; + } + + if (rnd->Next() % 2) { + iter.Next(); + pos = MakeKey(key(pos), gen(pos) + 1); + } else { + Key new_target = RandomTarget(rnd); + if (new_target > pos) { + pos = new_target; + iter.Seek(new_target); + } + } + } + } +}; +const uint32_t ConcurrentTest::K; + +// Simple test that does single-threaded testing of the ConcurrentTest +// scaffolding. +TEST(SkipTest, ConcurrentWithoutThreads) { + ConcurrentTest test; + Random rnd(test::RandomSeed()); + for (int i = 0; i < 10000; i++) { + test.ReadStep(&rnd); + test.WriteStep(&rnd); + } +} + +class TestState { + public: + ConcurrentTest t_; + int seed_; + port::AtomicPointer quit_flag_; + + enum ReaderState { + STARTING, + RUNNING, + DONE + }; + + explicit TestState(int s) + : seed_(s), + quit_flag_(NULL), + state_(STARTING), + state_cv_(&mu_) {} + + void Wait(ReaderState s) { + mu_.Lock(); + while (state_ != s) { + state_cv_.Wait(); + } + mu_.Unlock(); + } + + void Change(ReaderState s) { + mu_.Lock(); + state_ = s; + state_cv_.Signal(); + mu_.Unlock(); + } + + private: + port::Mutex mu_; + ReaderState state_; + port::CondVar state_cv_; +}; + +static void ConcurrentReader(void* arg) { + TestState* state = reinterpret_cast(arg); + Random rnd(state->seed_); + int64_t reads = 0; + state->Change(TestState::RUNNING); + while (!state->quit_flag_.Acquire_Load()) { + state->t_.ReadStep(&rnd); + ++reads; + } + state->Change(TestState::DONE); +} + +static void RunConcurrent(int run) { + const int seed = test::RandomSeed() + (run * 100); + Random rnd(seed); + const int N = 1000; + const int kSize = 1000; + for (int i = 0; i < N; i++) { + if ((i % 100) == 0) { + fprintf(stderr, "Run %d of %d\n", i, N); + } + TestState state(seed + 1); + Env::Default()->Schedule(ConcurrentReader, &state); + state.Wait(TestState::RUNNING); + for (int i = 0; i < kSize; i++) { + state.t_.WriteStep(&rnd); + } + state.quit_flag_.Release_Store(&state); // Any non-NULL arg will do + state.Wait(TestState::DONE); + } +} + +TEST(SkipTest, Concurrent1) { RunConcurrent(1); } +TEST(SkipTest, Concurrent2) { RunConcurrent(2); } +TEST(SkipTest, Concurrent3) { RunConcurrent(3); } +TEST(SkipTest, Concurrent4) { RunConcurrent(4); } +TEST(SkipTest, Concurrent5) { RunConcurrent(5); } + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/db/snapshot.h b/Subtrees/hyperleveldb/db/snapshot.h new file mode 100644 index 0000000000..0dff9b3746 --- /dev/null +++ b/Subtrees/hyperleveldb/db/snapshot.h @@ -0,0 +1,66 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_SNAPSHOT_H_ +#define STORAGE_HYPERLEVELDB_DB_SNAPSHOT_H_ + +#include "../hyperleveldb/db.h" + +namespace hyperleveldb { + +class SnapshotList; + +// Snapshots are kept in a doubly-linked list in the DB. +// Each SnapshotImpl corresponds to a particular sequence number. +class SnapshotImpl : public Snapshot { + public: + SequenceNumber number_; // const after creation + + private: + friend class SnapshotList; + + // SnapshotImpl is kept in a doubly-linked circular list + SnapshotImpl* prev_; + SnapshotImpl* next_; + + SnapshotList* list_; // just for sanity checks +}; + +class SnapshotList { + public: + SnapshotList() { + list_.prev_ = &list_; + list_.next_ = &list_; + } + + bool empty() const { return list_.next_ == &list_; } + SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; } + SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; } + + const SnapshotImpl* New(SequenceNumber seq) { + SnapshotImpl* s = new SnapshotImpl; + s->number_ = seq; + s->list_ = this; + s->next_ = &list_; + s->prev_ = list_.prev_; + s->prev_->next_ = s; + s->next_->prev_ = s; + return s; + } + + void Delete(const SnapshotImpl* s) { + assert(s->list_ == this); + s->prev_->next_ = s->next_; + s->next_->prev_ = s->prev_; + delete s; + } + + private: + // Dummy head of doubly-linked list of snapshots + SnapshotImpl list_; +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_SNAPSHOT_H_ diff --git a/Subtrees/hyperleveldb/db/table_cache.cc b/Subtrees/hyperleveldb/db/table_cache.cc new file mode 100644 index 0000000000..64d107893c --- /dev/null +++ b/Subtrees/hyperleveldb/db/table_cache.cc @@ -0,0 +1,121 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table_cache.h" + +#include "filename.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/table.h" +#include "../util/coding.h" + +namespace hyperleveldb { + +struct TableAndFile { + RandomAccessFile* file; + Table* table; +}; + +static void DeleteEntry(const Slice& key, void* value) { + TableAndFile* tf = reinterpret_cast(value); + delete tf->table; + delete tf->file; + delete tf; +} + +static void UnrefEntry(void* arg1, void* arg2) { + Cache* cache = reinterpret_cast(arg1); + Cache::Handle* h = reinterpret_cast(arg2); + cache->Release(h); +} + +TableCache::TableCache(const std::string& dbname, + const Options* options, + int entries) + : env_(options->env), + dbname_(dbname), + options_(options), + cache_(NewLRUCache(entries)) { +} + +TableCache::~TableCache() { + delete cache_; +} + +Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, + Cache::Handle** handle) { + Status s; + char buf[sizeof(file_number)]; + EncodeFixed64(buf, file_number); + Slice key(buf, sizeof(buf)); + *handle = cache_->Lookup(key); + if (*handle == NULL) { + std::string fname = TableFileName(dbname_, file_number); + RandomAccessFile* file = NULL; + Table* table = NULL; + s = env_->NewRandomAccessFile(fname, &file); + if (s.ok()) { + s = Table::Open(*options_, file, file_size, &table); + } + + if (!s.ok()) { + assert(table == NULL); + delete file; + // We do not cache error results so that if the error is transient, + // or somebody repairs the file, we recover automatically. + } else { + TableAndFile* tf = new TableAndFile; + tf->file = file; + tf->table = table; + *handle = cache_->Insert(key, tf, 1, &DeleteEntry); + } + } + return s; +} + +Iterator* TableCache::NewIterator(const ReadOptions& options, + uint64_t file_number, + uint64_t file_size, + Table** tableptr) { + if (tableptr != NULL) { + *tableptr = NULL; + } + + Cache::Handle* handle = NULL; + Status s = FindTable(file_number, file_size, &handle); + if (!s.ok()) { + return NewErrorIterator(s); + } + + Table* table = reinterpret_cast(cache_->Value(handle))->table; + Iterator* result = table->NewIterator(options); + result->RegisterCleanup(&UnrefEntry, cache_, handle); + if (tableptr != NULL) { + *tableptr = table; + } + return result; +} + +Status TableCache::Get(const ReadOptions& options, + uint64_t file_number, + uint64_t file_size, + const Slice& k, + void* arg, + void (*saver)(void*, const Slice&, const Slice&)) { + Cache::Handle* handle = NULL; + Status s = FindTable(file_number, file_size, &handle); + if (s.ok()) { + Table* t = reinterpret_cast(cache_->Value(handle))->table; + s = t->InternalGet(options, k, arg, saver); + cache_->Release(handle); + } + return s; +} + +void TableCache::Evict(uint64_t file_number) { + char buf[sizeof(file_number)]; + EncodeFixed64(buf, file_number); + cache_->Erase(Slice(buf, sizeof(buf))); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/table_cache.h b/Subtrees/hyperleveldb/db/table_cache.h new file mode 100644 index 0000000000..1afeecf56f --- /dev/null +++ b/Subtrees/hyperleveldb/db/table_cache.h @@ -0,0 +1,61 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Thread-safe (provides internal synchronization) + +#ifndef STORAGE_HYPERLEVELDB_DB_TABLE_CACHE_H_ +#define STORAGE_HYPERLEVELDB_DB_TABLE_CACHE_H_ + +#include +#include +#include "dbformat.h" +#include "../hyperleveldb/cache.h" +#include "../hyperleveldb/table.h" +#include "../port/port.h" + +namespace hyperleveldb { + +class Env; + +class TableCache { + public: + TableCache(const std::string& dbname, const Options* options, int entries); + ~TableCache(); + + // Return an iterator for the specified file number (the corresponding + // file length must be exactly "file_size" bytes). If "tableptr" is + // non-NULL, also sets "*tableptr" to point to the Table object + // underlying the returned iterator, or NULL if no Table object underlies + // the returned iterator. The returned "*tableptr" object is owned by + // the cache and should not be deleted, and is valid for as long as the + // returned iterator is live. + Iterator* NewIterator(const ReadOptions& options, + uint64_t file_number, + uint64_t file_size, + Table** tableptr = NULL); + + // If a seek to internal key "k" in specified file finds an entry, + // call (*handle_result)(arg, found_key, found_value). + Status Get(const ReadOptions& options, + uint64_t file_number, + uint64_t file_size, + const Slice& k, + void* arg, + void (*handle_result)(void*, const Slice&, const Slice&)); + + // Evict any entry for the specified file number + void Evict(uint64_t file_number); + + private: + Env* const env_; + const std::string dbname_; + const Options* options_; + Cache* cache_; + + Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**); +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_TABLE_CACHE_H_ diff --git a/Subtrees/hyperleveldb/db/version_edit.cc b/Subtrees/hyperleveldb/db/version_edit.cc new file mode 100644 index 0000000000..f76b47ddf1 --- /dev/null +++ b/Subtrees/hyperleveldb/db/version_edit.cc @@ -0,0 +1,266 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "version_edit.h" + +#include "version_set.h" +#include "../util/coding.h" + +namespace hyperleveldb { + +// Tag numbers for serialized VersionEdit. These numbers are written to +// disk and should not be changed. +enum Tag { + kComparator = 1, + kLogNumber = 2, + kNextFileNumber = 3, + kLastSequence = 4, + kCompactPointer = 5, + kDeletedFile = 6, + kNewFile = 7, + // 8 was used for large value refs + kPrevLogNumber = 9 +}; + +void VersionEdit::Clear() { + comparator_.clear(); + log_number_ = 0; + prev_log_number_ = 0; + last_sequence_ = 0; + next_file_number_ = 0; + has_comparator_ = false; + has_log_number_ = false; + has_prev_log_number_ = false; + has_next_file_number_ = false; + has_last_sequence_ = false; + deleted_files_.clear(); + new_files_.clear(); +} + +void VersionEdit::EncodeTo(std::string* dst) const { + if (has_comparator_) { + PutVarint32(dst, kComparator); + PutLengthPrefixedSlice(dst, comparator_); + } + if (has_log_number_) { + PutVarint32(dst, kLogNumber); + PutVarint64(dst, log_number_); + } + if (has_prev_log_number_) { + PutVarint32(dst, kPrevLogNumber); + PutVarint64(dst, prev_log_number_); + } + if (has_next_file_number_) { + PutVarint32(dst, kNextFileNumber); + PutVarint64(dst, next_file_number_); + } + if (has_last_sequence_) { + PutVarint32(dst, kLastSequence); + PutVarint64(dst, last_sequence_); + } + + for (size_t i = 0; i < compact_pointers_.size(); i++) { + PutVarint32(dst, kCompactPointer); + PutVarint32(dst, compact_pointers_[i].first); // level + PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode()); + } + + for (DeletedFileSet::const_iterator iter = deleted_files_.begin(); + iter != deleted_files_.end(); + ++iter) { + PutVarint32(dst, kDeletedFile); + PutVarint32(dst, iter->first); // level + PutVarint64(dst, iter->second); // file number + } + + for (size_t i = 0; i < new_files_.size(); i++) { + const FileMetaData& f = new_files_[i].second; + PutVarint32(dst, kNewFile); + PutVarint32(dst, new_files_[i].first); // level + PutVarint64(dst, f.number); + PutVarint64(dst, f.file_size); + PutLengthPrefixedSlice(dst, f.smallest.Encode()); + PutLengthPrefixedSlice(dst, f.largest.Encode()); + } +} + +static bool GetInternalKey(Slice* input, InternalKey* dst) { + Slice str; + if (GetLengthPrefixedSlice(input, &str)) { + dst->DecodeFrom(str); + return true; + } else { + return false; + } +} + +static bool GetLevel(Slice* input, int* level) { + uint32_t v; + if (GetVarint32(input, &v) && + v < config::kNumLevels) { + *level = v; + return true; + } else { + return false; + } +} + +Status VersionEdit::DecodeFrom(const Slice& src) { + Clear(); + Slice input = src; + const char* msg = NULL; + uint32_t tag; + + // Temporary storage for parsing + int level; + uint64_t number; + FileMetaData f; + Slice str; + InternalKey key; + + while (msg == NULL && GetVarint32(&input, &tag)) { + switch (tag) { + case kComparator: + if (GetLengthPrefixedSlice(&input, &str)) { + comparator_ = str.ToString(); + has_comparator_ = true; + } else { + msg = "comparator name"; + } + break; + + case kLogNumber: + if (GetVarint64(&input, &log_number_)) { + has_log_number_ = true; + } else { + msg = "log number"; + } + break; + + case kPrevLogNumber: + if (GetVarint64(&input, &prev_log_number_)) { + has_prev_log_number_ = true; + } else { + msg = "previous log number"; + } + break; + + case kNextFileNumber: + if (GetVarint64(&input, &next_file_number_)) { + has_next_file_number_ = true; + } else { + msg = "next file number"; + } + break; + + case kLastSequence: + if (GetVarint64(&input, &last_sequence_)) { + has_last_sequence_ = true; + } else { + msg = "last sequence number"; + } + break; + + case kCompactPointer: + if (GetLevel(&input, &level) && + GetInternalKey(&input, &key)) { + compact_pointers_.push_back(std::make_pair(level, key)); + } else { + msg = "compaction pointer"; + } + break; + + case kDeletedFile: + if (GetLevel(&input, &level) && + GetVarint64(&input, &number)) { + deleted_files_.insert(std::make_pair(level, number)); + } else { + msg = "deleted file"; + } + break; + + case kNewFile: + if (GetLevel(&input, &level) && + GetVarint64(&input, &f.number) && + GetVarint64(&input, &f.file_size) && + GetInternalKey(&input, &f.smallest) && + GetInternalKey(&input, &f.largest)) { + new_files_.push_back(std::make_pair(level, f)); + } else { + msg = "new-file entry"; + } + break; + + default: + msg = "unknown tag"; + break; + } + } + + if (msg == NULL && !input.empty()) { + msg = "invalid tag"; + } + + Status result; + if (msg != NULL) { + result = Status::Corruption("VersionEdit", msg); + } + return result; +} + +std::string VersionEdit::DebugString() const { + std::string r; + r.append("VersionEdit {"); + if (has_comparator_) { + r.append("\n Comparator: "); + r.append(comparator_); + } + if (has_log_number_) { + r.append("\n LogNumber: "); + AppendNumberTo(&r, log_number_); + } + if (has_prev_log_number_) { + r.append("\n PrevLogNumber: "); + AppendNumberTo(&r, prev_log_number_); + } + if (has_next_file_number_) { + r.append("\n NextFile: "); + AppendNumberTo(&r, next_file_number_); + } + if (has_last_sequence_) { + r.append("\n LastSeq: "); + AppendNumberTo(&r, last_sequence_); + } + for (size_t i = 0; i < compact_pointers_.size(); i++) { + r.append("\n CompactPointer: "); + AppendNumberTo(&r, compact_pointers_[i].first); + r.append(" "); + r.append(compact_pointers_[i].second.DebugString()); + } + for (DeletedFileSet::const_iterator iter = deleted_files_.begin(); + iter != deleted_files_.end(); + ++iter) { + r.append("\n DeleteFile: "); + AppendNumberTo(&r, iter->first); + r.append(" "); + AppendNumberTo(&r, iter->second); + } + for (size_t i = 0; i < new_files_.size(); i++) { + const FileMetaData& f = new_files_[i].second; + r.append("\n AddFile: "); + AppendNumberTo(&r, new_files_[i].first); + r.append(" "); + AppendNumberTo(&r, f.number); + r.append(" "); + AppendNumberTo(&r, f.file_size); + r.append(" "); + r.append(f.smallest.DebugString()); + r.append(" .. "); + r.append(f.largest.DebugString()); + } + r.append("\n}\n"); + return r; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/version_edit.h b/Subtrees/hyperleveldb/db/version_edit.h new file mode 100644 index 0000000000..e5d65aa028 --- /dev/null +++ b/Subtrees/hyperleveldb/db/version_edit.h @@ -0,0 +1,107 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_VERSION_EDIT_H_ +#define STORAGE_HYPERLEVELDB_DB_VERSION_EDIT_H_ + +#include +#include +#include +#include "dbformat.h" + +namespace hyperleveldb { + +class VersionSet; + +struct FileMetaData { + int refs; + int allowed_seeks; // Seeks allowed until compaction + uint64_t number; + uint64_t file_size; // File size in bytes + InternalKey smallest; // Smallest internal key served by table + InternalKey largest; // Largest internal key served by table + + FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { } +}; + +class VersionEdit { + public: + VersionEdit() { Clear(); } + ~VersionEdit() { } + + void Clear(); + + void SetComparatorName(const Slice& name) { + has_comparator_ = true; + comparator_ = name.ToString(); + } + void SetLogNumber(uint64_t num) { + has_log_number_ = true; + log_number_ = num; + } + void SetPrevLogNumber(uint64_t num) { + has_prev_log_number_ = true; + prev_log_number_ = num; + } + void SetNextFile(uint64_t num) { + has_next_file_number_ = true; + next_file_number_ = num; + } + void SetLastSequence(SequenceNumber seq) { + has_last_sequence_ = true; + last_sequence_ = seq; + } + void SetCompactPointer(int level, const InternalKey& key) { + compact_pointers_.push_back(std::make_pair(level, key)); + } + + // Add the specified file at the specified number. + // REQUIRES: This version has not been saved (see VersionSet::SaveTo) + // REQUIRES: "smallest" and "largest" are smallest and largest keys in file + void AddFile(int level, uint64_t file, + uint64_t file_size, + const InternalKey& smallest, + const InternalKey& largest) { + FileMetaData f; + f.number = file; + f.file_size = file_size; + f.smallest = smallest; + f.largest = largest; + new_files_.push_back(std::make_pair(level, f)); + } + + // Delete the specified "file" from the specified "level". + void DeleteFile(int level, uint64_t file) { + deleted_files_.insert(std::make_pair(level, file)); + } + + void EncodeTo(std::string* dst) const; + Status DecodeFrom(const Slice& src); + + std::string DebugString() const; + + private: + friend class VersionSet; + + typedef std::set< std::pair > DeletedFileSet; + + std::string comparator_; + uint64_t log_number_; + uint64_t prev_log_number_; + uint64_t next_file_number_; + SequenceNumber last_sequence_; + bool has_comparator_; + bool has_log_number_; + bool has_prev_log_number_; + bool has_next_file_number_; + bool has_last_sequence_; + + std::vector< std::pair > compact_pointers_; + DeletedFileSet deleted_files_; + std::vector< std::pair > new_files_; +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_VERSION_EDIT_H_ diff --git a/Subtrees/hyperleveldb/db/version_edit_test.cc b/Subtrees/hyperleveldb/db/version_edit_test.cc new file mode 100644 index 0000000000..0688369781 --- /dev/null +++ b/Subtrees/hyperleveldb/db/version_edit_test.cc @@ -0,0 +1,46 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "version_edit.h" +#include "../util/testharness.h" + +namespace hyperleveldb { + +static void TestEncodeDecode(const VersionEdit& edit) { + std::string encoded, encoded2; + edit.EncodeTo(&encoded); + VersionEdit parsed; + Status s = parsed.DecodeFrom(encoded); + ASSERT_TRUE(s.ok()) << s.ToString(); + parsed.EncodeTo(&encoded2); + ASSERT_EQ(encoded, encoded2); +} + +class VersionEditTest { }; + +TEST(VersionEditTest, EncodeDecode) { + static const uint64_t kBig = 1ull << 50; + + VersionEdit edit; + for (int i = 0; i < 4; i++) { + TestEncodeDecode(edit); + edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, + InternalKey("foo", kBig + 500 + i, kTypeValue), + InternalKey("zoo", kBig + 600 + i, kTypeDeletion)); + edit.DeleteFile(4, kBig + 700 + i); + edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue)); + } + + edit.SetComparatorName("foo"); + edit.SetLogNumber(kBig + 100); + edit.SetNextFile(kBig + 200); + edit.SetLastSequence(kBig + 1000); + TestEncodeDecode(edit); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/db/version_set.cc b/Subtrees/hyperleveldb/db/version_set.cc new file mode 100644 index 0000000000..77665e9f12 --- /dev/null +++ b/Subtrees/hyperleveldb/db/version_set.cc @@ -0,0 +1,1502 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_set.h" + +#include +#include +#include "dbformat.h" +#include "filename.h" +#include "log_reader.h" +#include "log_writer.h" +#include "memtable.h" +#include "table_cache.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/table_builder.h" +#include "../table/merger.h" +#include "../table/two_level_iterator.h" +#include "../util/coding.h" +#include "../util/logging.h" + +namespace hyperleveldb { + +static double MaxBytesForLevel(int level) { + assert(level < leveldb::config::kNumLevels); + static const double bytes[] = {10 * 1048576.0, + 100 * 1048576.0, + 100 * 1048576.0, + 1000 * 1048576.0, + 10000 * 1048576.0, + 100000 * 1048576.0, + 1000000 * 1048576.0}; + return bytes[level]; +} + +static uint64_t MaxFileSizeForLevel(int level) { + assert(level < leveldb::config::kNumLevels); + static const uint64_t bytes[] = {8 * 1048576, + 8 * 1048576, + 8 * 1048576, + 8 * 1048576, + 8 * 1048576, + 8 * 1048576, + 8 * 1048576}; + return bytes[level]; +} + +static uint64_t MaxCompactionBytesForLevel(int level) { + assert(level < leveldb::config::kNumLevels); + static const uint64_t bytes[] = {128 * 1048576, + 128 * 1048576, + 128 * 1048576, + 256 * 1048576, + 256 * 1048576, + 256 * 1048576, + 256 * 1048576}; + return bytes[level]; +} + +static int64_t TotalFileSize(const std::vector& files) { + int64_t sum = 0; + for (size_t i = 0; i < files.size(); i++) { + sum += files[i]->file_size; + } + return sum; +} + +namespace { +std::string IntSetToString(const std::set& s) { + std::string result = "{"; + for (std::set::const_iterator it = s.begin(); + it != s.end(); + ++it) { + result += (result.size() > 1) ? "," : ""; + result += NumberToString(*it); + } + result += "}"; + return result; +} +} // namespace + +Version::~Version() { + assert(refs_ == 0); + + // Remove from linked list + prev_->next_ = next_; + next_->prev_ = prev_; + + // Drop references to files + for (int level = 0; level < config::kNumLevels; level++) { + for (size_t i = 0; i < files_[level].size(); i++) { + FileMetaData* f = files_[level][i]; + assert(f->refs > 0); + f->refs--; + if (f->refs <= 0) { + delete f; + } + } + } +} + +int FindFile(const InternalKeyComparator& icmp, + const std::vector& files, + const Slice& key) { + uint32_t left = 0; + uint32_t right = files.size(); + while (left < right) { + uint32_t mid = (left + right) / 2; + const FileMetaData* f = files[mid]; + if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) { + // Key at "mid.largest" is < "target". Therefore all + // files at or before "mid" are uninteresting. + left = mid + 1; + } else { + // Key at "mid.largest" is >= "target". Therefore all files + // after "mid" are uninteresting. + right = mid; + } + } + return right; +} + +static bool AfterFile(const Comparator* ucmp, + const Slice* user_key, const FileMetaData* f) { + // NULL user_key occurs before all keys and is therefore never after *f + return (user_key != NULL && + ucmp->Compare(*user_key, f->largest.user_key()) > 0); +} + +static bool BeforeFile(const Comparator* ucmp, + const Slice* user_key, const FileMetaData* f) { + // NULL user_key occurs after all keys and is therefore never before *f + return (user_key != NULL && + ucmp->Compare(*user_key, f->smallest.user_key()) < 0); +} + +bool SomeFileOverlapsRange( + const InternalKeyComparator& icmp, + bool disjoint_sorted_files, + const std::vector& files, + const Slice* smallest_user_key, + const Slice* largest_user_key) { + const Comparator* ucmp = icmp.user_comparator(); + if (!disjoint_sorted_files) { + // Need to check against all files + for (size_t i = 0; i < files.size(); i++) { + const FileMetaData* f = files[i]; + if (AfterFile(ucmp, smallest_user_key, f) || + BeforeFile(ucmp, largest_user_key, f)) { + // No overlap + } else { + return true; // Overlap + } + } + return false; + } + + // Binary search over file list + uint32_t index = 0; + if (smallest_user_key != NULL) { + // Find the earliest possible internal key for smallest_user_key + InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek); + index = FindFile(icmp, files, small.Encode()); + } + + if (index >= files.size()) { + // beginning of range is after all files, so no overlap. + return false; + } + + return !BeforeFile(ucmp, largest_user_key, files[index]); +} + +// An internal iterator. For a given version/level pair, yields +// information about the files in the level. For a given entry, key() +// is the largest key that occurs in the file, and value() is an +// 16-byte value containing the file number and file size, both +// encoded using EncodeFixed64. +class Version::LevelFileNumIterator : public Iterator { + public: + LevelFileNumIterator(const InternalKeyComparator& icmp, + const std::vector* flist) + : icmp_(icmp), + flist_(flist), + index_(flist->size()) { // Marks as invalid + } + virtual bool Valid() const { + return index_ < flist_->size(); + } + virtual void Seek(const Slice& target) { + index_ = FindFile(icmp_, *flist_, target); + } + virtual void SeekToFirst() { index_ = 0; } + virtual void SeekToLast() { + index_ = flist_->empty() ? 0 : flist_->size() - 1; + } + virtual void Next() { + assert(Valid()); + index_++; + } + virtual void Prev() { + assert(Valid()); + if (index_ == 0) { + index_ = flist_->size(); // Marks as invalid + } else { + index_--; + } + } + Slice key() const { + assert(Valid()); + return (*flist_)[index_]->largest.Encode(); + } + Slice value() const { + assert(Valid()); + EncodeFixed64(value_buf_, (*flist_)[index_]->number); + EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size); + return Slice(value_buf_, sizeof(value_buf_)); + } + virtual Status status() const { return Status::OK(); } + private: + const InternalKeyComparator icmp_; + const std::vector* const flist_; + uint32_t index_; + + // Backing store for value(). Holds the file number and size. + mutable char value_buf_[16]; +}; + +static Iterator* GetFileIterator(void* arg, + const ReadOptions& options, + const Slice& file_value) { + TableCache* cache = reinterpret_cast(arg); + if (file_value.size() != 16) { + return NewErrorIterator( + Status::Corruption("FileReader invoked with unexpected value")); + } else { + return cache->NewIterator(options, + DecodeFixed64(file_value.data()), + DecodeFixed64(file_value.data() + 8)); + } +} + +Iterator* Version::NewConcatenatingIterator(const ReadOptions& options, + int level) const { + return NewTwoLevelIterator( + new LevelFileNumIterator(vset_->icmp_, &files_[level]), + &GetFileIterator, vset_->table_cache_, options); +} + +void Version::AddIterators(const ReadOptions& options, + std::vector* iters) { + // Merge all level zero files together since they may overlap + for (size_t i = 0; i < files_[0].size(); i++) { + iters->push_back( + vset_->table_cache_->NewIterator( + options, files_[0][i]->number, files_[0][i]->file_size)); + } + + // For levels > 0, we can use a concatenating iterator that sequentially + // walks through the non-overlapping files in the level, opening them + // lazily. + for (int level = 1; level < config::kNumLevels; level++) { + if (!files_[level].empty()) { + iters->push_back(NewConcatenatingIterator(options, level)); + } + } +} + +// Callback from TableCache::Get() +namespace { +enum SaverState { + kNotFound, + kFound, + kDeleted, + kCorrupt, +}; +struct Saver { + SaverState state; + const Comparator* ucmp; + Slice user_key; + std::string* value; +}; +} +static void SaveValue(void* arg, const Slice& ikey, const Slice& v) { + Saver* s = reinterpret_cast(arg); + ParsedInternalKey parsed_key; + if (!ParseInternalKey(ikey, &parsed_key)) { + s->state = kCorrupt; + } else { + if (s->ucmp->Compare(parsed_key.user_key, s->user_key) == 0) { + s->state = (parsed_key.type == kTypeValue) ? kFound : kDeleted; + if (s->state == kFound) { + s->value->assign(v.data(), v.size()); + } + } + } +} + +static bool NewestFirst(FileMetaData* a, FileMetaData* b) { + return a->number > b->number; +} + +Status Version::Get(const ReadOptions& options, + const LookupKey& k, + std::string* value, + GetStats* stats) { + Slice ikey = k.internal_key(); + Slice user_key = k.user_key(); + const Comparator* ucmp = vset_->icmp_.user_comparator(); + Status s; + + stats->seek_file = NULL; + stats->seek_file_level = -1; + FileMetaData* last_file_read = NULL; + int last_file_read_level = -1; + + // We can search level-by-level since entries never hop across + // levels. Therefore we are guaranteed that if we find data + // in an smaller level, later levels are irrelevant. + std::vector tmp; + FileMetaData* tmp2; + for (int level = 0; level < config::kNumLevels; level++) { + size_t num_files = files_[level].size(); + if (num_files == 0) continue; + + // Get the list of files to search in this level + FileMetaData* const* files = &files_[level][0]; + if (level == 0) { + // Level-0 files may overlap each other. Find all files that + // overlap user_key and process them in order from newest to oldest. + tmp.reserve(num_files); + for (uint32_t i = 0; i < num_files; i++) { + FileMetaData* f = files[i]; + if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 && + ucmp->Compare(user_key, f->largest.user_key()) <= 0) { + tmp.push_back(f); + } + } + if (tmp.empty()) continue; + + std::sort(tmp.begin(), tmp.end(), NewestFirst); + files = &tmp[0]; + num_files = tmp.size(); + } else { + // Binary search to find earliest index whose largest key >= ikey. + uint32_t index = FindFile(vset_->icmp_, files_[level], ikey); + if (index >= num_files) { + files = NULL; + num_files = 0; + } else { + tmp2 = files[index]; + if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) { + // All of "tmp2" is past any data for user_key + files = NULL; + num_files = 0; + } else { + files = &tmp2; + num_files = 1; + } + } + } + + for (uint32_t i = 0; i < num_files; ++i) { + if (last_file_read != NULL && stats->seek_file == NULL) { + // We have had more than one seek for this read. Charge the 1st file. + stats->seek_file = last_file_read; + stats->seek_file_level = last_file_read_level; + } + + FileMetaData* f = files[i]; + last_file_read = f; + last_file_read_level = level; + + Saver saver; + saver.state = kNotFound; + saver.ucmp = ucmp; + saver.user_key = user_key; + saver.value = value; + s = vset_->table_cache_->Get(options, f->number, f->file_size, + ikey, &saver, SaveValue); + if (!s.ok()) { + return s; + } + switch (saver.state) { + case kNotFound: + break; // Keep searching in other files + case kFound: + return s; + case kDeleted: + s = Status::NotFound(Slice()); // Use empty error message for speed + return s; + case kCorrupt: + s = Status::Corruption("corrupted key for ", user_key); + return s; + } + } + } + + return Status::NotFound(Slice()); // Use an empty error message for speed +} + +void Version::Ref() { + ++refs_; +} + +void Version::Unref() { + assert(this != &vset_->dummy_versions_); + assert(refs_ >= 1); + --refs_; + if (refs_ == 0) { + delete this; + } +} + +bool Version::OverlapInLevel(int level, + const Slice* smallest_user_key, + const Slice* largest_user_key) { + return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level], + smallest_user_key, largest_user_key); +} + +int Version::PickLevelForMemTableOutput( + const Slice& smallest_user_key, + const Slice& largest_user_key) { + int level = 0; + if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) { + // Push to next level if there is no overlap in next level, + // and the #bytes overlapping in the level after that are limited. + InternalKey start(smallest_user_key, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey limit(largest_user_key, 0, static_cast(0)); + std::vector overlaps; + while (level < config::kMaxMemCompactLevel) { + if (OverlapInLevel(level + 1, &smallest_user_key, &largest_user_key)) { + break; + } + GetOverlappingInputs(level + 2, &start, &limit, &overlaps); + const int64_t sum = TotalFileSize(overlaps); + level++; + } + } + return level; +} + +// Store in "*inputs" all files in "level" that overlap [begin,end] +void Version::GetOverlappingInputs( + int level, + const InternalKey* begin, + const InternalKey* end, + std::vector* inputs) { + inputs->clear(); + Slice user_begin, user_end; + if (begin != NULL) { + user_begin = begin->user_key(); + } + if (end != NULL) { + user_end = end->user_key(); + } + const Comparator* user_cmp = vset_->icmp_.user_comparator(); + for (size_t i = 0; i < files_[level].size(); ) { + FileMetaData* f = files_[level][i++]; + const Slice file_start = f->smallest.user_key(); + const Slice file_limit = f->largest.user_key(); + if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) { + // "f" is completely before specified range; skip it + } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) { + // "f" is completely after specified range; skip it + } else { + inputs->push_back(f); + if (level == 0) { + // Level-0 files may overlap each other. So check if the newly + // added file has expanded the range. If so, restart search. + if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) { + user_begin = file_start; + inputs->clear(); + i = 0; + } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) { + user_end = file_limit; + inputs->clear(); + i = 0; + } + } + } + } +} + +std::string Version::DebugString() const { + std::string r; + for (int level = 0; level < config::kNumLevels; level++) { + // E.g., + // --- level 1 --- + // 17:123['a' .. 'd'] + // 20:43['e' .. 'g'] + r.append("--- level "); + AppendNumberTo(&r, level); + r.append(" ---\n"); + const std::vector& files = files_[level]; + for (size_t i = 0; i < files.size(); i++) { + r.push_back(' '); + AppendNumberTo(&r, files[i]->number); + r.push_back(':'); + AppendNumberTo(&r, files[i]->file_size); + r.append("["); + r.append(files[i]->smallest.DebugString()); + r.append(" .. "); + r.append(files[i]->largest.DebugString()); + r.append("]\n"); + } + } + return r; +} + +// A helper class so we can efficiently apply a whole sequence +// of edits to a particular state without creating intermediate +// Versions that contain full copies of the intermediate state. +class VersionSet::Builder { + private: + // Helper to sort by v->files_[file_number].smallest + struct BySmallestKey { + const InternalKeyComparator* internal_comparator; + + bool operator()(FileMetaData* f1, FileMetaData* f2) const { + int r = internal_comparator->Compare(f1->smallest, f2->smallest); + if (r != 0) { + return (r < 0); + } else { + // Break ties by file number + return (f1->number < f2->number); + } + } + }; + + typedef std::set FileSet; + struct LevelState { + std::set deleted_files; + FileSet* added_files; + }; + + VersionSet* vset_; + Version* base_; + LevelState levels_[config::kNumLevels]; + + public: + // Initialize a builder with the files from *base and other info from *vset + Builder(VersionSet* vset, Version* base) + : vset_(vset), + base_(base) { + base_->Ref(); + BySmallestKey cmp; + cmp.internal_comparator = &vset_->icmp_; + for (int level = 0; level < config::kNumLevels; level++) { + levels_[level].added_files = new FileSet(cmp); + } + } + + ~Builder() { + for (int level = 0; level < config::kNumLevels; level++) { + const FileSet* added = levels_[level].added_files; + std::vector to_unref; + to_unref.reserve(added->size()); + for (FileSet::const_iterator it = added->begin(); + it != added->end(); ++it) { + to_unref.push_back(*it); + } + delete added; + for (uint32_t i = 0; i < to_unref.size(); i++) { + FileMetaData* f = to_unref[i]; + f->refs--; + if (f->refs <= 0) { + delete f; + } + } + } + base_->Unref(); + } + + // Apply all of the edits in *edit to the current state. + void Apply(VersionEdit* edit) { + // Update compaction pointers + for (size_t i = 0; i < edit->compact_pointers_.size(); i++) { + const int level = edit->compact_pointers_[i].first; + vset_->compact_pointer_[level] = + edit->compact_pointers_[i].second.Encode().ToString(); + } + + // Delete files + const VersionEdit::DeletedFileSet& del = edit->deleted_files_; + for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin(); + iter != del.end(); + ++iter) { + const int level = iter->first; + const uint64_t number = iter->second; + levels_[level].deleted_files.insert(number); + } + + // Add new files + for (size_t i = 0; i < edit->new_files_.size(); i++) { + const int level = edit->new_files_[i].first; + FileMetaData* f = new FileMetaData(edit->new_files_[i].second); + f->refs = 1; + + // We arrange to automatically compact this file after + // a certain number of seeks. Let's assume: + // (1) One seek costs 10ms + // (2) Writing or reading 1MB costs 10ms (100MB/s) + // (3) A compaction of 1MB does 25MB of IO: + // 1MB read from this level + // 10-12MB read from next level (boundaries may be misaligned) + // 10-12MB written to next level + // This implies that 25 seeks cost the same as the compaction + // of 1MB of data. I.e., one seek costs approximately the + // same as the compaction of 40KB of data. We are a little + // conservative and allow approximately one seek for every 16KB + // of data before triggering a compaction. + f->allowed_seeks = (f->file_size / 16384); + if (f->allowed_seeks < 100) f->allowed_seeks = 100; + + levels_[level].deleted_files.erase(f->number); + levels_[level].added_files->insert(f); + } + } + + // Save the current state in *v. + void SaveTo(Version* v) { + BySmallestKey cmp; + cmp.internal_comparator = &vset_->icmp_; + for (int level = 0; level < config::kNumLevels; level++) { + // Merge the set of added files with the set of pre-existing files. + // Drop any deleted files. Store the result in *v. + const std::vector& base_files = base_->files_[level]; + std::vector::const_iterator base_iter = base_files.begin(); + std::vector::const_iterator base_end = base_files.end(); + const FileSet* added = levels_[level].added_files; + v->files_[level].reserve(base_files.size() + added->size()); + for (FileSet::const_iterator added_iter = added->begin(); + added_iter != added->end(); + ++added_iter) { + // Add all smaller files listed in base_ + for (std::vector::const_iterator bpos + = std::upper_bound(base_iter, base_end, *added_iter, cmp); + base_iter != bpos; + ++base_iter) { + MaybeAddFile(v, level, *base_iter); + } + + MaybeAddFile(v, level, *added_iter); + } + + // Add remaining base files + for (; base_iter != base_end; ++base_iter) { + MaybeAddFile(v, level, *base_iter); + } + +#ifndef NDEBUG + // Make sure there is no overlap in levels > 0 + if (level > 0) { + for (uint32_t i = 1; i < v->files_[level].size(); i++) { + const InternalKey& prev_end = v->files_[level][i-1]->largest; + const InternalKey& this_begin = v->files_[level][i]->smallest; + if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) { + fprintf(stderr, "overlapping ranges in same level %s vs. %s\n", + prev_end.DebugString().c_str(), + this_begin.DebugString().c_str()); + abort(); + } + } + } +#endif + } + } + + void MaybeAddFile(Version* v, int level, FileMetaData* f) { + if (levels_[level].deleted_files.count(f->number) > 0) { + // File is deleted: do nothing + } else { + std::vector* files = &v->files_[level]; + if (level > 0 && !files->empty()) { + // Must not overlap + assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest, + f->smallest) < 0); + } + f->refs++; + files->push_back(f); + } + } +}; + +VersionSet::VersionSet(const std::string& dbname, + const Options* options, + TableCache* table_cache, + const InternalKeyComparator* cmp) + : env_(options->env), + dbname_(dbname), + options_(options), + table_cache_(table_cache), + icmp_(*cmp), + next_file_number_(2), + manifest_file_number_(0), // Filled by Recover() + last_sequence_(0), + log_number_(0), + prev_log_number_(0), + descriptor_file_(NULL), + descriptor_log_(NULL), + dummy_versions_(this), + current_(NULL) { + AppendVersion(new Version(this)); +} + +VersionSet::~VersionSet() { + current_->Unref(); + assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty + delete descriptor_log_; + delete descriptor_file_; +} + +void VersionSet::AppendVersion(Version* v) { + // Make "v" current + assert(v->refs_ == 0); + assert(v != current_); + if (current_ != NULL) { + current_->Unref(); + } + current_ = v; + v->Ref(); + + // Append to linked list + v->prev_ = dummy_versions_.prev_; + v->next_ = &dummy_versions_; + v->prev_->next_ = v; + v->next_->prev_ = v; +} + +Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu, port::CondVar* cv, bool* wt) { + while (*wt) { + cv->Wait(); + } + *wt = true; + if (edit->has_log_number_) { + assert(edit->log_number_ >= log_number_); + assert(edit->log_number_ < next_file_number_); + } else { + edit->SetLogNumber(log_number_); + } + + if (!edit->has_prev_log_number_) { + edit->SetPrevLogNumber(prev_log_number_); + } + + edit->SetNextFile(next_file_number_); + edit->SetLastSequence(last_sequence_); + + Version* v = new Version(this); + { + Builder builder(this, current_); + builder.Apply(edit); + builder.SaveTo(v); + } + Finalize(v); + + // Initialize new descriptor log file if necessary by creating + // a temporary file that contains a snapshot of the current version. + std::string new_manifest_file; + Status s; + if (descriptor_log_ == NULL) { + // No reason to unlock *mu here since we only hit this path in the + // first call to LogAndApply (when opening the database). + assert(descriptor_file_ == NULL); + new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_); + edit->SetNextFile(next_file_number_); + s = env_->NewWritableFile(new_manifest_file, &descriptor_file_); + if (s.ok()) { + descriptor_log_ = new log::Writer(descriptor_file_); + s = WriteSnapshot(descriptor_log_); + } + } + + // Unlock during expensive MANIFEST log write + { + mu->Unlock(); + + // Write new record to MANIFEST log + if (s.ok()) { + std::string record; + edit->EncodeTo(&record); + s = descriptor_log_->AddRecord(record); + if (s.ok()) { + // XXX Unlock during expensive MANIFEST log write + s = descriptor_file_->Sync(); + } + if (!s.ok()) { + Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str()); + if (ManifestContains(record)) { + Log(options_->info_log, + "MANIFEST contains log record despite error; advancing to new " + "version to prevent mismatch between in-memory and logged state"); + s = Status::OK(); + } + } + } + + // If we just created a new descriptor file, install it by writing a + // new CURRENT file that points to it. + if (s.ok() && !new_manifest_file.empty()) { + s = SetCurrentFile(env_, dbname_, manifest_file_number_); + // No need to double-check MANIFEST in case of error since it + // will be discarded below. + } + + mu->Lock(); + } + + // Install the new version + if (s.ok()) { + AppendVersion(v); + log_number_ = edit->log_number_; + prev_log_number_ = edit->prev_log_number_; + } else { + delete v; + if (!new_manifest_file.empty()) { + delete descriptor_log_; + delete descriptor_file_; + descriptor_log_ = NULL; + descriptor_file_ = NULL; + env_->DeleteFile(new_manifest_file); + } + } + + *wt = false; + cv->Signal(); + return s; +} + +Status VersionSet::Recover() { + struct LogReporter : public log::Reader::Reporter { + Status* status; + virtual void Corruption(size_t bytes, const Status& s) { + if (this->status->ok()) *this->status = s; + } + }; + + // Read "CURRENT" file, which contains a pointer to the current manifest file + std::string current; + Status s = ReadFileToString(env_, CurrentFileName(dbname_), ¤t); + if (!s.ok()) { + return s; + } + if (current.empty() || current[current.size()-1] != '\n') { + return Status::Corruption("CURRENT file does not end with newline"); + } + current.resize(current.size() - 1); + + std::string dscname = dbname_ + "/" + current; + SequentialFile* file; + s = env_->NewSequentialFile(dscname, &file); + if (!s.ok()) { + return s; + } + + bool have_log_number = false; + bool have_prev_log_number = false; + bool have_next_file = false; + bool have_last_sequence = false; + uint64_t next_file = 0; + uint64_t last_sequence = 0; + uint64_t log_number = 0; + uint64_t prev_log_number = 0; + Builder builder(this, current_); + + { + LogReporter reporter; + reporter.status = &s; + log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch) && s.ok()) { + VersionEdit edit; + s = edit.DecodeFrom(record); + if (s.ok()) { + if (edit.has_comparator_ && + edit.comparator_ != icmp_.user_comparator()->Name()) { + s = Status::InvalidArgument( + edit.comparator_ + " does not match existing comparator ", + icmp_.user_comparator()->Name()); + } + } + + if (s.ok()) { + builder.Apply(&edit); + } + + if (edit.has_log_number_) { + log_number = edit.log_number_; + have_log_number = true; + } + + if (edit.has_prev_log_number_) { + prev_log_number = edit.prev_log_number_; + have_prev_log_number = true; + } + + if (edit.has_next_file_number_) { + next_file = edit.next_file_number_; + have_next_file = true; + } + + if (edit.has_last_sequence_) { + last_sequence = edit.last_sequence_; + have_last_sequence = true; + } + } + } + delete file; + file = NULL; + + if (s.ok()) { + if (!have_next_file) { + s = Status::Corruption("no meta-nextfile entry in descriptor"); + } else if (!have_log_number) { + s = Status::Corruption("no meta-lognumber entry in descriptor"); + } else if (!have_last_sequence) { + s = Status::Corruption("no last-sequence-number entry in descriptor"); + } + + if (!have_prev_log_number) { + prev_log_number = 0; + } + + MarkFileNumberUsed(prev_log_number); + MarkFileNumberUsed(log_number); + } + + if (s.ok()) { + Version* v = new Version(this); + builder.SaveTo(v); + // Install recovered version + Finalize(v); + AppendVersion(v); + manifest_file_number_ = next_file; + next_file_number_ = next_file + 1; + last_sequence_ = last_sequence; + log_number_ = log_number; + prev_log_number_ = prev_log_number; + } + + return s; +} + +void VersionSet::MarkFileNumberUsed(uint64_t number) { + if (next_file_number_ <= number) { + next_file_number_ = number + 1; + } +} + +void VersionSet::Finalize(Version* v) { + // Compute the ratio of disk usage to its limit + for (int level = 0; level + 1 < config::kNumLevels; ++level) { + double score; + if (level == 0) { + // We treat level-0 specially by bounding the number of files + // instead of number of bytes for two reasons: + // + // (1) With larger write-buffer sizes, it is nice not to do too + // many level-0 compactions. + // + // (2) The files in level-0 are merged on every read and + // therefore we wish to avoid too many files when the individual + // file size is small (perhaps because of a small write-buffer + // setting, or very high compression ratios, or lots of + // overwrites/deletions). + score = v->files_[level].size() / + static_cast(config::kL0_CompactionTrigger); + } else { + // Compute the ratio of current size to size limit. + const uint64_t level_bytes = TotalFileSize(v->files_[level]); + score = static_cast(level_bytes) / MaxBytesForLevel(level); + } + v->compaction_scores_[level] = score; + } +} + +Status VersionSet::WriteSnapshot(log::Writer* log) { + // TODO: Break up into multiple records to reduce memory usage on recovery? + + // Save metadata + VersionEdit edit; + edit.SetComparatorName(icmp_.user_comparator()->Name()); + + // Save compaction pointers + for (int level = 0; level < config::kNumLevels; level++) { + if (!compact_pointer_[level].empty()) { + InternalKey key; + key.DecodeFrom(compact_pointer_[level]); + edit.SetCompactPointer(level, key); + } + } + + // Save files + for (int level = 0; level < config::kNumLevels; level++) { + const std::vector& files = current_->files_[level]; + for (size_t i = 0; i < files.size(); i++) { + const FileMetaData* f = files[i]; + edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest); + } + } + + std::string record; + edit.EncodeTo(&record); + return log->AddRecord(record); +} + +int VersionSet::NumLevelFiles(int level) const { + assert(level >= 0); + assert(level < config::kNumLevels); + return current_->files_[level].size(); +} + +const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const { + // Update code if kNumLevels changes + assert(config::kNumLevels == 7); + snprintf(scratch->buffer, sizeof(scratch->buffer), + "files[ %d %d %d %d %d %d %d ]", + int(current_->files_[0].size()), + int(current_->files_[1].size()), + int(current_->files_[2].size()), + int(current_->files_[3].size()), + int(current_->files_[4].size()), + int(current_->files_[5].size()), + int(current_->files_[6].size())); + return scratch->buffer; +} + +// Return true iff the manifest contains the specified record. +bool VersionSet::ManifestContains(const std::string& record) const { + std::string fname = DescriptorFileName(dbname_, manifest_file_number_); + Log(options_->info_log, "ManifestContains: checking %s\n", fname.c_str()); + SequentialFile* file = NULL; + Status s = env_->NewSequentialFile(fname, &file); + if (!s.ok()) { + Log(options_->info_log, "ManifestContains: %s\n", s.ToString().c_str()); + return false; + } + log::Reader reader(file, NULL, true/*checksum*/, 0); + Slice r; + std::string scratch; + bool result = false; + while (reader.ReadRecord(&r, &scratch)) { + if (r == Slice(record)) { + result = true; + break; + } + } + delete file; + Log(options_->info_log, "ManifestContains: result = %d\n", result ? 1 : 0); + return result; +} + +uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) { + uint64_t result = 0; + for (int level = 0; level < config::kNumLevels; level++) { + const std::vector& files = v->files_[level]; + for (size_t i = 0; i < files.size(); i++) { + if (icmp_.Compare(files[i]->largest, ikey) <= 0) { + // Entire file is before "ikey", so just add the file size + result += files[i]->file_size; + } else if (icmp_.Compare(files[i]->smallest, ikey) > 0) { + // Entire file is after "ikey", so ignore + if (level > 0) { + // Files other than level 0 are sorted by meta->smallest, so + // no further files in this level will contain data for + // "ikey". + break; + } + } else { + // "ikey" falls in the range for this table. Add the + // approximate offset of "ikey" within the table. + Table* tableptr; + Iterator* iter = table_cache_->NewIterator( + ReadOptions(), files[i]->number, files[i]->file_size, &tableptr); + if (tableptr != NULL) { + result += tableptr->ApproximateOffsetOf(ikey.Encode()); + } + delete iter; + } + } + } + return result; +} + +void VersionSet::AddLiveFiles(std::set* live) { + for (Version* v = dummy_versions_.next_; + v != &dummy_versions_; + v = v->next_) { + for (int level = 0; level < config::kNumLevels; level++) { + const std::vector& files = v->files_[level]; + for (size_t i = 0; i < files.size(); i++) { + live->insert(files[i]->number); + } + } + } +} + +int64_t VersionSet::NumLevelBytes(int level) const { + assert(level >= 0); + assert(level < config::kNumLevels); + return TotalFileSize(current_->files_[level]); +} + +int64_t VersionSet::MaxNextLevelOverlappingBytes() { + int64_t result = 0; + std::vector overlaps; + for (int level = 1; level < config::kNumLevels - 1; level++) { + for (size_t i = 0; i < current_->files_[level].size(); i++) { + const FileMetaData* f = current_->files_[level][i]; + current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest, + &overlaps); + const int64_t sum = TotalFileSize(overlaps); + if (sum > result) { + result = sum; + } + } + } + return result; +} + +// Stores the minimal range that covers all entries in inputs in +// *smallest, *largest. +// REQUIRES: inputs is not empty +void VersionSet::GetRange(const std::vector& inputs, + InternalKey* smallest, + InternalKey* largest) { + assert(!inputs.empty()); + smallest->Clear(); + largest->Clear(); + for (size_t i = 0; i < inputs.size(); i++) { + FileMetaData* f = inputs[i]; + if (i == 0) { + *smallest = f->smallest; + *largest = f->largest; + } else { + if (icmp_.Compare(f->smallest, *smallest) < 0) { + *smallest = f->smallest; + } + if (icmp_.Compare(f->largest, *largest) > 0) { + *largest = f->largest; + } + } + } +} + +// Stores the minimal range that covers all entries in inputs1 and inputs2 +// in *smallest, *largest. +// REQUIRES: inputs is not empty +void VersionSet::GetRange2(const std::vector& inputs1, + const std::vector& inputs2, + InternalKey* smallest, + InternalKey* largest) { + std::vector all = inputs1; + all.insert(all.end(), inputs2.begin(), inputs2.end()); + GetRange(all, smallest, largest); +} + +Iterator* VersionSet::MakeInputIterator(Compaction* c) { + ReadOptions options; + options.verify_checksums = options_->paranoid_checks; + options.fill_cache = false; + + // Level-0 files have to be merged together. For other levels, + // we will make a concatenating iterator per level. + // TODO(opt): use concatenating iterator for level-0 if there is no overlap + const int space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2); + Iterator** list = new Iterator*[space]; + int num = 0; + for (int which = 0; which < 2; which++) { + if (!c->inputs_[which].empty()) { + if (c->level() + which == 0) { + const std::vector& files = c->inputs_[which]; + for (size_t i = 0; i < files.size(); i++) { + list[num++] = table_cache_->NewIterator( + options, files[i]->number, files[i]->file_size); + } + } else { + // Create concatenating iterator for the files from this level + list[num++] = NewTwoLevelIterator( + new Version::LevelFileNumIterator(icmp_, &c->inputs_[which]), + &GetFileIterator, table_cache_, options); + } + } + } + assert(num <= space); + Iterator* result = NewMergingIterator(&icmp_, list, num); + delete[] list; + return result; +} + +struct CompactionBoundary { + size_t start; + size_t limit; + CompactionBoundary() : start(0), limit(0) {} + CompactionBoundary(size_t s, size_t l) : start(s), limit(l) {} +}; + +struct CmpByRange { + CmpByRange(const Comparator* cmp) : cmp_(cmp) {} + bool operator () (const FileMetaData* lhs, const FileMetaData* rhs) { + int smallest = cmp_->Compare(lhs->smallest.user_key(), rhs->smallest.user_key()); + if (smallest == 0) { + return cmp_->Compare(lhs->largest.user_key(), rhs->largest.user_key()) < 0; + } + return smallest < 0; + } + private: + const Comparator* cmp_; +}; + +// Stores the compaction boundaries between level and level + 1 +void VersionSet::GetCompactionBoundaries(int level, + std::vector* LA, + std::vector* LB, + std::vector* LA_sizes, + std::vector* LB_sizes, + std::vector* boundaries) +{ + const Comparator* user_cmp = icmp_.user_comparator(); + *LA = current_->files_[level + 0]; + *LB = current_->files_[level + 1]; + *LA_sizes = std::vector(LA->size() + 1, 0); + *LB_sizes = std::vector(LB->size() + 1, 0); + std::sort(LA->begin(), LA->end(), CmpByRange(user_cmp)); + std::sort(LB->begin(), LB->end(), CmpByRange(user_cmp)); + boundaries->resize(LA->size()); + + // compute sizes + for (size_t i = 0; i < LA->size(); ++i) { + (*LA_sizes)[i + 1] = (*LA_sizes)[i] + (*LA)[i]->file_size; + } + for (size_t i = 0; i < LB->size(); ++i) { + (*LB_sizes)[i + 1] = (*LB_sizes)[i] + (*LB)[i]->file_size; + } + + // compute boundaries + size_t start = 0; + size_t limit = 0; + // figure out which range of LB each LA covers + for (size_t i = 0; i < LA->size(); ++i) { + // find smallest start s.t. LB[start] overlaps LA[i] + while (start < LB->size() && + user_cmp->Compare((*LB)[start]->largest.user_key(), + (*LA)[i]->smallest.user_key()) < 0) { + ++start; + } + limit = std::max(start, limit); + // find smallest limit >= start s.t. LB[limit] does not overlap LA[i] + while (limit < LB->size() && + user_cmp->Compare((*LB)[limit]->smallest.user_key(), + (*LA)[i]->largest.user_key()) <= 0) { + ++limit; + } + (*boundaries)[i].start = start; + (*boundaries)[i].limit = limit; + } +} + +int VersionSet::PickCompactionLevel(bool* locked) { + // Find an unlocked level has score >= 1 where level + 1 has score < 1. + int level = config::kNumLevels; + for (int i = 0; i + 1 < config::kNumLevels; ++i) { + if (locked[i] || locked[i + 1]) { + continue; + } + if (current_->compaction_scores_[i + 0] >= 1.0 && + current_->compaction_scores_[i + 1] < 1.0) { + level = i; + break; + } + } + return level; +} + +static bool OldestFirst(FileMetaData* a, FileMetaData* b) { + return a->number < b->number; +} + +Compaction* VersionSet::PickCompaction(int level) { + assert(0 <= level && level < config::kNumLevels); + bool trivial = false; + + if (current_->files_[level].empty()) { + return NULL; + } + + Compaction* c = new Compaction(level); + c->input_version_ = current_; + c->input_version_->Ref(); + + if (level > 0) { + std::vector LA; + std::vector LB; + std::vector LA_sizes; + std::vector LB_sizes; + std::vector boundaries; + GetCompactionBoundaries(level, &LA, &LB, &LA_sizes, &LB_sizes, &boundaries); + + // find the best set of files: maximize the ratio of sizeof(LA)/sizeof(LB) + // while keeping sizeof(LA)+sizeof(LB) < some threshold. If there's a tie + // for ratio, minimize size. + size_t best_idx_start = 0; + size_t best_idx_limit = 0; + uint64_t best_size = 0; + double best_ratio = -1; + for (size_t i = 0; i < boundaries.size(); ++i) { + for (size_t j = i; j < boundaries.size(); ++j) { + uint64_t sz_a = LA_sizes[j + 1] - LA_sizes[i]; + uint64_t sz_b = LB_sizes[boundaries[j].limit] - LB_sizes[boundaries[i].start]; + if (boundaries[j].start == boundaries[j].limit) { + trivial = true; + break; + } + if (sz_a + sz_b >= MaxCompactionBytesForLevel(level)) { + break; + } + assert(sz_b > 0); // true because we exclude trivial moves + double ratio = double(sz_a) / double(sz_b); + if (ratio > best_ratio || + (ratio == best_ratio && sz_a + sz_b < best_size)) { + best_ratio = ratio; + best_size = sz_a + sz_b; + best_idx_start = i; + best_idx_limit = j + 1; + } + } + } + + // Trivial moves have a near-0 cost, so do them first. + if (trivial) { + for (size_t i = 0; i < LA.size(); ++i) { + if (boundaries[i].start == boundaries[i].limit) { + c->inputs_[0].push_back(LA[i]); + } + } + trivial = level != 0; + c->SetRatio(1.0); + // If the best we could do would be wasteful and the best level has more + // data in it than the next level would have, move it all + } else if (level < 4 && best_ratio >= 0.0 && + LA_sizes.back() * best_ratio >= LB_sizes.back()) { + for (size_t i = 0 ; i < LA.size(); ++i) { + c->inputs_[0].push_back(LA[i]); + } + c->SetRatio(double(LA_sizes.back()) / double(LB_sizes.back())); + // otherwise go with the best ratio + } else if (best_ratio >= 0.0) { + for (size_t i = best_idx_start; i < best_idx_limit; ++i) { + assert(i >= 0 && i < LA.size()); + c->inputs_[0].push_back(LA[i]); + } + for (size_t i = boundaries[best_idx_start].start; + i < boundaries[best_idx_limit - 1].limit; ++i) { + assert(i >= 0 && i < LB.size()); + c->inputs_[1].push_back(LB[i]); + } + c->SetRatio(best_ratio); + // otherwise just pick the file with least overlap + } else { + assert(level >= 0); + assert(level+1 < config::kNumLevels); + // Pick the file that overlaps with the fewest files in the next level + size_t largest = boundaries.size(); + size_t smallest = boundaries.size(); + for (size_t i = 0; i < boundaries.size(); ++i) { + if (smallest == boundaries.size() || + boundaries[smallest].limit - boundaries[smallest].start > + boundaries[i].limit - boundaries[i].start) { + smallest = i; + } + } + assert(smallest < boundaries.size()); + c->inputs_[0].push_back(LA[smallest]); + for (size_t i = boundaries[smallest].start; i < boundaries[smallest].limit; ++i) { + c->inputs_[1].push_back(LB[i]); + } + } + } else { + std::vector tmp(current_->files_[0]); + std::sort(tmp.begin(), tmp.end(), OldestFirst); + for (size_t i = 0; i < tmp.size() && c->inputs_[0].size() < 32; ++i) { + c->inputs_[0].push_back(tmp[i]); + } + } + + assert(!c->inputs_[0].empty()); + + if (!trivial) { + SetupOtherInputs(c); + } + return c; +} + +void VersionSet::SetupOtherInputs(Compaction* c) { + const int level = c->level(); + InternalKey smallest, largest; + GetRange(c->inputs_[0], &smallest, &largest); + current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]); + + // Update the place where we will do the next compaction for this level. + // We update this immediately instead of waiting for the VersionEdit + // to be applied so that if the compaction fails, we will try a different + // key range next time. + compact_pointer_[level] = largest.Encode().ToString(); + c->edit_.SetCompactPointer(level, largest); +} + +Compaction* VersionSet::CompactRange( + int level, + const InternalKey* begin, + const InternalKey* end) { + std::vector inputs; + current_->GetOverlappingInputs(level, begin, end, &inputs); + if (inputs.empty()) { + return NULL; + } + + // Avoid compacting too much in one shot in case the range is large. + // But we cannot do this for level-0 since level-0 files can overlap + // and we must not pick one file and drop another older file if the + // two files overlap. + if (level > 0) { + const uint64_t limit = MaxFileSizeForLevel(level); + uint64_t total = 0; + for (size_t i = 0; i < inputs.size(); i++) { + uint64_t s = inputs[i]->file_size; + total += s; + if (total >= limit) { + inputs.resize(i + 1); + break; + } + } + } + + Compaction* c = new Compaction(level); + c->input_version_ = current_; + c->input_version_->Ref(); + c->inputs_[0] = inputs; + SetupOtherInputs(c); + return c; +} + +Compaction::Compaction(int level) + : level_(level), + max_output_file_size_(MaxFileSizeForLevel(level)), + input_version_(NULL), + ratio_(0) { + for (int i = 0; i < config::kNumLevels; i++) { + level_ptrs_[i] = 0; + } +} + +Compaction::~Compaction() { + if (input_version_ != NULL) { + input_version_->Unref(); + } +} + +bool Compaction::IsTrivialMove() const { + return num_input_files(1) == 0; +} + +void Compaction::AddInputDeletions(VersionEdit* edit) { + for (int which = 0; which < 2; which++) { + for (size_t i = 0; i < inputs_[which].size(); i++) { + edit->DeleteFile(level_ + which, inputs_[which][i]->number); + } + } +} + +bool Compaction::IsBaseLevelForKey(const Slice& user_key) { + // Maybe use binary search to find right entry instead of linear search? + const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator(); + for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) { + const std::vector& files = input_version_->files_[lvl]; + for (; level_ptrs_[lvl] < files.size(); ) { + FileMetaData* f = files[level_ptrs_[lvl]]; + if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) { + // We've advanced far enough + if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) { + // Key falls in this file's range, so definitely not base level + return false; + } + break; + } + level_ptrs_[lvl]++; + } + } + return true; +} + +void Compaction::ReleaseInputs() { + if (input_version_ != NULL) { + input_version_->Unref(); + input_version_ = NULL; + } +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/version_set.h b/Subtrees/hyperleveldb/db/version_set.h new file mode 100644 index 0000000000..8c0a0655b8 --- /dev/null +++ b/Subtrees/hyperleveldb/db/version_set.h @@ -0,0 +1,390 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// The representation of a DBImpl consists of a set of Versions. The +// newest version is called "current". Older versions may be kept +// around to provide a consistent view to live iterators. +// +// Each Version keeps track of a set of Table files per level. The +// entire set of versions is maintained in a VersionSet. +// +// Version,VersionSet are thread-compatible, but require external +// synchronization on all accesses. + +#ifndef STORAGE_HYPERLEVELDB_DB_VERSION_SET_H_ +#define STORAGE_HYPERLEVELDB_DB_VERSION_SET_H_ + +#include +#include +#include +#include "dbformat.h" +#include "version_edit.h" +#include "../port/port.h" +#include "../port/thread_annotations.h" + +namespace hyperleveldb { + +namespace log { class Writer; } + +class Compaction; +class CompactionBoundary; +class Iterator; +class MemTable; +class TableBuilder; +class TableCache; +class Version; +class VersionSet; +class WritableFile; + +// Return the smallest index i such that files[i]->largest >= key. +// Return files.size() if there is no such file. +// REQUIRES: "files" contains a sorted list of non-overlapping files. +extern int FindFile(const InternalKeyComparator& icmp, + const std::vector& files, + const Slice& key); + +// Returns true iff some file in "files" overlaps the user key range +// [*smallest,*largest]. +// smallest==NULL represents a key smaller than all keys in the DB. +// largest==NULL represents a key largest than all keys in the DB. +// REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges +// in sorted order. +extern bool SomeFileOverlapsRange( + const InternalKeyComparator& icmp, + bool disjoint_sorted_files, + const std::vector& files, + const Slice* smallest_user_key, + const Slice* largest_user_key); + +class Version { + public: + // Append to *iters a sequence of iterators that will + // yield the contents of this Version when merged together. + // REQUIRES: This version has been saved (see VersionSet::SaveTo) + void AddIterators(const ReadOptions&, std::vector* iters); + + // Lookup the value for key. If found, store it in *val and + // return OK. Else return a non-OK status. Fills *stats. + // REQUIRES: lock is not held + struct GetStats { + FileMetaData* seek_file; + int seek_file_level; + }; + Status Get(const ReadOptions&, const LookupKey& key, std::string* val, + GetStats* stats); + + // Reference count management (so Versions do not disappear out from + // under live iterators) + void Ref(); + void Unref(); + + void GetOverlappingInputs( + int level, + const InternalKey* begin, // NULL means before all keys + const InternalKey* end, // NULL means after all keys + std::vector* inputs); + + // Returns true iff some file in the specified level overlaps + // some part of [*smallest_user_key,*largest_user_key]. + // smallest_user_key==NULL represents a key smaller than all keys in the DB. + // largest_user_key==NULL represents a key largest than all keys in the DB. + bool OverlapInLevel(int level, + const Slice* smallest_user_key, + const Slice* largest_user_key); + + // Return the level at which we should place a new memtable compaction + // result that covers the range [smallest_user_key,largest_user_key]. + int PickLevelForMemTableOutput(const Slice& smallest_user_key, + const Slice& largest_user_key); + + int NumFiles(int level) const { return files_[level].size(); } + + // Return a human readable string that describes this version's contents. + std::string DebugString() const; + + private: + friend class Compaction; + friend class VersionSet; + + class LevelFileNumIterator; + Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const; + + VersionSet* vset_; // VersionSet to which this Version belongs + Version* next_; // Next version in linked list + Version* prev_; // Previous version in linked list + int refs_; // Number of live refs to this version + + // List of files per level + std::vector files_[config::kNumLevels]; + + // Level that should be compacted next and its compaction score. + // Score < 1 means compaction is not strictly needed. These fields + // are initialized by Finalize(). + double compaction_scores_[config::kNumLevels]; + + explicit Version(VersionSet* vset) + : vset_(vset), next_(this), prev_(this), refs_(0) { + for (int i = 0; i < config::kNumLevels; ++i) { + compaction_scores_[i] = -1; + } + } + + ~Version(); + + // No copying allowed + Version(const Version&); + void operator=(const Version&); +}; + +class VersionSet { + public: + VersionSet(const std::string& dbname, + const Options* options, + TableCache* table_cache, + const InternalKeyComparator*); + ~VersionSet(); + + // Apply *edit to the current version to form a new descriptor that + // is both saved to persistent state and installed as the new + // current version. Will release *mu while actually writing to the file. + // REQUIRES: *mu is held on entry. + // REQUIRES: no other thread concurrently calls LogAndApply() + Status LogAndApply(VersionEdit* edit, port::Mutex* mu, port::CondVar* cv, bool* wt) + EXCLUSIVE_LOCKS_REQUIRED(mu); + + // Recover the last saved descriptor from persistent storage. + Status Recover(); + + // Return the current version. + Version* current() const { return current_; } + + // Return the current manifest file number + uint64_t ManifestFileNumber() const { return manifest_file_number_; } + + // Allocate and return a new file number + uint64_t NewFileNumber() { return next_file_number_++; } + + // Arrange to reuse "file_number" unless a newer file number has + // already been allocated. + // REQUIRES: "file_number" was returned by a call to NewFileNumber(). + void ReuseFileNumber(uint64_t file_number) { + if (next_file_number_ == file_number + 1) { + next_file_number_ = file_number; + } + } + + // Return the number of Table files at the specified level. + int NumLevelFiles(int level) const; + + // Return the combined file size of all files at the specified level. + int64_t NumLevelBytes(int level) const; + + // Return the last sequence number. + uint64_t LastSequence() const { return last_sequence_; } + + // Set the last sequence number to s. + void SetLastSequence(uint64_t s) { + assert(s >= last_sequence_); + last_sequence_ = s; + } + + // Mark the specified file number as used. + void MarkFileNumberUsed(uint64_t number); + + // Return the current log file number. + uint64_t LogNumber() const { return log_number_; } + + // Return the log file number for the log file that is currently + // being compacted, or zero if there is no such log file. + uint64_t PrevLogNumber() const { return prev_log_number_; } + + // Pick level for a new compaction. + // Returns kNumLevels if there is no compaction to be done. + // Otherwise returns the lowest unlocked level that may compact upwards. + int PickCompactionLevel(bool* locked); + + // Pick inputs for a new compaction at the specified level. + // Returns NULL if there is no compaction to be done. + // Otherwise returns a pointer to a heap-allocated object that + // describes the compaction. Caller should delete the result. + Compaction* PickCompaction(int level); + + // Return a compaction object for compacting the range [begin,end] in + // the specified level. Returns NULL if there is nothing in that + // level that overlaps the specified range. Caller should delete + // the result. + Compaction* CompactRange( + int level, + const InternalKey* begin, + const InternalKey* end); + + // Return the maximum overlapping data (in bytes) at next level for any + // file at a level >= 1. + int64_t MaxNextLevelOverlappingBytes(); + + // Create an iterator that reads over the compaction inputs for "*c". + // The caller should delete the iterator when no longer needed. + Iterator* MakeInputIterator(Compaction* c); + + // Returns true iff some level needs a compaction. + bool NeedsCompaction(bool* levels) const { + Version* v = current_; + for (int i = 0; i + 1 < config::kNumLevels; ++i) { + if (!levels[i] && !levels[i + 1] && + v->compaction_scores_[i] >= 1.0 && + (i + 2 == config::kNumLevels || + v->compaction_scores_[i + 1] < 1.0)) { + return true; + } + } + return false; + } + + // Add all files listed in any live version to *live. + // May also mutate some internal state. + void AddLiveFiles(std::set* live); + + // Return the approximate offset in the database of the data for + // "key" as of version "v". + uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key); + + // Return a human-readable short (single-line) summary of the number + // of files per level. Uses *scratch as backing store. + struct LevelSummaryStorage { + char buffer[100]; + }; + const char* LevelSummary(LevelSummaryStorage* scratch) const; + + private: + class Builder; + + friend class Compaction; + friend class Version; + + void Finalize(Version* v); + + void GetRange(const std::vector& inputs, + InternalKey* smallest, + InternalKey* largest); + + void GetRange2(const std::vector& inputs1, + const std::vector& inputs2, + InternalKey* smallest, + InternalKey* largest); + + void GetCompactionBoundaries(int level, + std::vector* LA, + std::vector* LB, + std::vector* LA_sizes, + std::vector* LB_sizes, + std::vector* boundaries); + + void SetupOtherInputs(Compaction* c); + + // Save current contents to *log + Status WriteSnapshot(log::Writer* log); + + void AppendVersion(Version* v); + + bool ManifestContains(const std::string& record) const; + + Env* const env_; + const std::string dbname_; + const Options* const options_; + TableCache* const table_cache_; + const InternalKeyComparator icmp_; + uint64_t next_file_number_; + uint64_t manifest_file_number_; + uint64_t last_sequence_; + uint64_t log_number_; + uint64_t prev_log_number_; // 0 or backing store for memtable being compacted + + // Opened lazily + WritableFile* descriptor_file_; + log::Writer* descriptor_log_; + Version dummy_versions_; // Head of circular doubly-linked list of versions. + Version* current_; // == dummy_versions_.prev_ + + // Per-level key at which the next compaction at that level should start. + // Either an empty string, or a valid InternalKey. + std::string compact_pointer_[config::kNumLevels]; + + // No copying allowed + VersionSet(const VersionSet&); + void operator=(const VersionSet&); +}; + +// A Compaction encapsulates information about a compaction. +class Compaction { + public: + ~Compaction(); + + // Return the level that is being compacted. Inputs from "level" + // and "level+1" will be merged to produce a set of "level+1" files. + int level() const { return level_; } + + // Return the object that holds the edits to the descriptor done + // by this compaction. + VersionEdit* edit() { return &edit_; } + + // "which" must be either 0 or 1 + int num_input_files(int which) const { return inputs_[which].size(); } + + // Return the ith input file at "level()+which" ("which" must be 0 or 1). + FileMetaData* input(int which, int i) const { return inputs_[which][i]; } + + // Maximum size of files to build during this compaction. + uint64_t MaxOutputFileSize() const { return max_output_file_size_; } + + // Is this a trivial compaction that can be implemented by just + // moving a single input file to the next level (no merging or splitting) + bool IsTrivialMove() const; + + // Add all inputs to this compaction as delete operations to *edit. + void AddInputDeletions(VersionEdit* edit); + + // Returns true if the information we have available guarantees that + // the compaction is producing data in "level+1" for which no data exists + // in levels greater than "level+1". + bool IsBaseLevelForKey(const Slice& user_key); + + // Release the input version for the compaction, once the compaction + // is successful. + void ReleaseInputs(); + + // Set and get the ratio of inputs to outputs. + // If nonzero, this is the ratio of inputs to outputs. If zero, it indicates + // that the compaction was chosen without concern for the ratio of inputs to + // outputs. + void SetRatio(double ratio) { ratio_ = ratio; } + double ratio() { return ratio_; } + + private: + friend class Version; + friend class VersionSet; + + explicit Compaction(int level); + + int level_; + uint64_t max_output_file_size_; + Version* input_version_; + VersionEdit edit_; + + double ratio_; + + // Each compaction reads inputs from "level_" and "level_+1" + std::vector inputs_[2]; // The two sets of inputs + + // State for implementing IsBaseLevelForKey + + // level_ptrs_ holds indices into input_version_->levels_: our state + // is that we are positioned at one of the file ranges for each + // higher level than the ones involved in this compaction (i.e. for + // all L >= level_ + 2). + size_t level_ptrs_[config::kNumLevels]; +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_DB_VERSION_SET_H_ diff --git a/Subtrees/hyperleveldb/db/version_set_test.cc b/Subtrees/hyperleveldb/db/version_set_test.cc new file mode 100644 index 0000000000..c101609184 --- /dev/null +++ b/Subtrees/hyperleveldb/db/version_set_test.cc @@ -0,0 +1,179 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "version_set.h" +#include "../util/logging.h" +#include "../util/testharness.h" +#include "../util/testutil.h" + +namespace hyperleveldb { + +class FindFileTest { + public: + std::vector files_; + bool disjoint_sorted_files_; + + FindFileTest() : disjoint_sorted_files_(true) { } + + ~FindFileTest() { + for (int i = 0; i < files_.size(); i++) { + delete files_[i]; + } + } + + void Add(const char* smallest, const char* largest, + SequenceNumber smallest_seq = 100, + SequenceNumber largest_seq = 100) { + FileMetaData* f = new FileMetaData; + f->number = files_.size() + 1; + f->smallest = InternalKey(smallest, smallest_seq, kTypeValue); + f->largest = InternalKey(largest, largest_seq, kTypeValue); + files_.push_back(f); + } + + int Find(const char* key) { + InternalKey target(key, 100, kTypeValue); + InternalKeyComparator cmp(BytewiseComparator()); + return FindFile(cmp, files_, target.Encode()); + } + + bool Overlaps(const char* smallest, const char* largest) { + InternalKeyComparator cmp(BytewiseComparator()); + Slice s(smallest != NULL ? smallest : ""); + Slice l(largest != NULL ? largest : ""); + return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, files_, + (smallest != NULL ? &s : NULL), + (largest != NULL ? &l : NULL)); + } +}; + +TEST(FindFileTest, Empty) { + ASSERT_EQ(0, Find("foo")); + ASSERT_TRUE(! Overlaps("a", "z")); + ASSERT_TRUE(! Overlaps(NULL, "z")); + ASSERT_TRUE(! Overlaps("a", NULL)); + ASSERT_TRUE(! Overlaps(NULL, NULL)); +} + +TEST(FindFileTest, Single) { + Add("p", "q"); + ASSERT_EQ(0, Find("a")); + ASSERT_EQ(0, Find("p")); + ASSERT_EQ(0, Find("p1")); + ASSERT_EQ(0, Find("q")); + ASSERT_EQ(1, Find("q1")); + ASSERT_EQ(1, Find("z")); + + ASSERT_TRUE(! Overlaps("a", "b")); + ASSERT_TRUE(! Overlaps("z1", "z2")); + ASSERT_TRUE(Overlaps("a", "p")); + ASSERT_TRUE(Overlaps("a", "q")); + ASSERT_TRUE(Overlaps("a", "z")); + ASSERT_TRUE(Overlaps("p", "p1")); + ASSERT_TRUE(Overlaps("p", "q")); + ASSERT_TRUE(Overlaps("p", "z")); + ASSERT_TRUE(Overlaps("p1", "p2")); + ASSERT_TRUE(Overlaps("p1", "z")); + ASSERT_TRUE(Overlaps("q", "q")); + ASSERT_TRUE(Overlaps("q", "q1")); + + ASSERT_TRUE(! Overlaps(NULL, "j")); + ASSERT_TRUE(! Overlaps("r", NULL)); + ASSERT_TRUE(Overlaps(NULL, "p")); + ASSERT_TRUE(Overlaps(NULL, "p1")); + ASSERT_TRUE(Overlaps("q", NULL)); + ASSERT_TRUE(Overlaps(NULL, NULL)); +} + + +TEST(FindFileTest, Multiple) { + Add("150", "200"); + Add("200", "250"); + Add("300", "350"); + Add("400", "450"); + ASSERT_EQ(0, Find("100")); + ASSERT_EQ(0, Find("150")); + ASSERT_EQ(0, Find("151")); + ASSERT_EQ(0, Find("199")); + ASSERT_EQ(0, Find("200")); + ASSERT_EQ(1, Find("201")); + ASSERT_EQ(1, Find("249")); + ASSERT_EQ(1, Find("250")); + ASSERT_EQ(2, Find("251")); + ASSERT_EQ(2, Find("299")); + ASSERT_EQ(2, Find("300")); + ASSERT_EQ(2, Find("349")); + ASSERT_EQ(2, Find("350")); + ASSERT_EQ(3, Find("351")); + ASSERT_EQ(3, Find("400")); + ASSERT_EQ(3, Find("450")); + ASSERT_EQ(4, Find("451")); + + ASSERT_TRUE(! Overlaps("100", "149")); + ASSERT_TRUE(! Overlaps("251", "299")); + ASSERT_TRUE(! Overlaps("451", "500")); + ASSERT_TRUE(! Overlaps("351", "399")); + + ASSERT_TRUE(Overlaps("100", "150")); + ASSERT_TRUE(Overlaps("100", "200")); + ASSERT_TRUE(Overlaps("100", "300")); + ASSERT_TRUE(Overlaps("100", "400")); + ASSERT_TRUE(Overlaps("100", "500")); + ASSERT_TRUE(Overlaps("375", "400")); + ASSERT_TRUE(Overlaps("450", "450")); + ASSERT_TRUE(Overlaps("450", "500")); +} + +TEST(FindFileTest, MultipleNullBoundaries) { + Add("150", "200"); + Add("200", "250"); + Add("300", "350"); + Add("400", "450"); + ASSERT_TRUE(! Overlaps(NULL, "149")); + ASSERT_TRUE(! Overlaps("451", NULL)); + ASSERT_TRUE(Overlaps(NULL, NULL)); + ASSERT_TRUE(Overlaps(NULL, "150")); + ASSERT_TRUE(Overlaps(NULL, "199")); + ASSERT_TRUE(Overlaps(NULL, "200")); + ASSERT_TRUE(Overlaps(NULL, "201")); + ASSERT_TRUE(Overlaps(NULL, "400")); + ASSERT_TRUE(Overlaps(NULL, "800")); + ASSERT_TRUE(Overlaps("100", NULL)); + ASSERT_TRUE(Overlaps("200", NULL)); + ASSERT_TRUE(Overlaps("449", NULL)); + ASSERT_TRUE(Overlaps("450", NULL)); +} + +TEST(FindFileTest, OverlapSequenceChecks) { + Add("200", "200", 5000, 3000); + ASSERT_TRUE(! Overlaps("199", "199")); + ASSERT_TRUE(! Overlaps("201", "300")); + ASSERT_TRUE(Overlaps("200", "200")); + ASSERT_TRUE(Overlaps("190", "200")); + ASSERT_TRUE(Overlaps("200", "210")); +} + +TEST(FindFileTest, OverlappingFiles) { + Add("150", "600"); + Add("400", "500"); + disjoint_sorted_files_ = false; + ASSERT_TRUE(! Overlaps("100", "149")); + ASSERT_TRUE(! Overlaps("601", "700")); + ASSERT_TRUE(Overlaps("100", "150")); + ASSERT_TRUE(Overlaps("100", "200")); + ASSERT_TRUE(Overlaps("100", "300")); + ASSERT_TRUE(Overlaps("100", "400")); + ASSERT_TRUE(Overlaps("100", "500")); + ASSERT_TRUE(Overlaps("375", "400")); + ASSERT_TRUE(Overlaps("450", "450")); + ASSERT_TRUE(Overlaps("450", "500")); + ASSERT_TRUE(Overlaps("450", "700")); + ASSERT_TRUE(Overlaps("600", "700")); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/db/write_batch.cc b/Subtrees/hyperleveldb/db/write_batch.cc new file mode 100644 index 0000000000..c7e9e58fcb --- /dev/null +++ b/Subtrees/hyperleveldb/db/write_batch.cc @@ -0,0 +1,147 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// WriteBatch::rep_ := +// sequence: fixed64 +// count: fixed32 +// data: record[count] +// record := +// kTypeValue varstring varstring | +// kTypeDeletion varstring +// varstring := +// len: varint32 +// data: uint8[len] + +#include "../hyperleveldb/write_batch.h" + +#include "../hyperleveldb/db.h" +#include "dbformat.h" +#include "memtable.h" +#include "write_batch_internal.h" +#include "../util/coding.h" + +namespace hyperleveldb { + +// WriteBatch header has an 8-byte sequence number followed by a 4-byte count. +static const size_t kHeader = 12; + +WriteBatch::WriteBatch() { + Clear(); +} + +WriteBatch::~WriteBatch() { } + +WriteBatch::Handler::~Handler() { } + +void WriteBatch::Clear() { + rep_.clear(); + rep_.resize(kHeader); +} + +Status WriteBatch::Iterate(Handler* handler) const { + Slice input(rep_); + if (input.size() < kHeader) { + return Status::Corruption("malformed WriteBatch (too small)"); + } + + input.remove_prefix(kHeader); + Slice key, value; + int found = 0; + while (!input.empty()) { + found++; + char tag = input[0]; + input.remove_prefix(1); + switch (tag) { + case kTypeValue: + if (GetLengthPrefixedSlice(&input, &key) && + GetLengthPrefixedSlice(&input, &value)) { + handler->Put(key, value); + } else { + return Status::Corruption("bad WriteBatch Put"); + } + break; + case kTypeDeletion: + if (GetLengthPrefixedSlice(&input, &key)) { + handler->Delete(key); + } else { + return Status::Corruption("bad WriteBatch Delete"); + } + break; + default: + return Status::Corruption("unknown WriteBatch tag"); + } + } + if (found != WriteBatchInternal::Count(this)) { + return Status::Corruption("WriteBatch has wrong count"); + } else { + return Status::OK(); + } +} + +int WriteBatchInternal::Count(const WriteBatch* b) { + return DecodeFixed32(b->rep_.data() + 8); +} + +void WriteBatchInternal::SetCount(WriteBatch* b, int n) { + EncodeFixed32(&b->rep_[8], n); +} + +SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) { + return SequenceNumber(DecodeFixed64(b->rep_.data())); +} + +void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) { + EncodeFixed64(&b->rep_[0], seq); +} + +void WriteBatch::Put(const Slice& key, const Slice& value) { + WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); + rep_.push_back(static_cast(kTypeValue)); + PutLengthPrefixedSlice(&rep_, key); + PutLengthPrefixedSlice(&rep_, value); +} + +void WriteBatch::Delete(const Slice& key) { + WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); + rep_.push_back(static_cast(kTypeDeletion)); + PutLengthPrefixedSlice(&rep_, key); +} + +namespace { +class MemTableInserter : public WriteBatch::Handler { + public: + SequenceNumber sequence_; + MemTable* mem_; + + virtual void Put(const Slice& key, const Slice& value) { + mem_->Add(sequence_, kTypeValue, key, value); + sequence_++; + } + virtual void Delete(const Slice& key) { + mem_->Add(sequence_, kTypeDeletion, key, Slice()); + sequence_++; + } +}; +} // namespace + +Status WriteBatchInternal::InsertInto(const WriteBatch* b, + MemTable* memtable) { + MemTableInserter inserter; + inserter.sequence_ = WriteBatchInternal::Sequence(b); + inserter.mem_ = memtable; + return b->Iterate(&inserter); +} + +void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) { + assert(contents.size() >= kHeader); + b->rep_.assign(contents.data(), contents.size()); +} + +void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) { + SetCount(dst, Count(dst) + Count(src)); + assert(src->rep_.size() >= kHeader); + dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/db/write_batch_internal.h b/Subtrees/hyperleveldb/db/write_batch_internal.h new file mode 100644 index 0000000000..fdc39b1197 --- /dev/null +++ b/Subtrees/hyperleveldb/db/write_batch_internal.h @@ -0,0 +1,49 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_DB_WRITE_BATCH_INTERNAL_H_ +#define STORAGE_HYPERLEVELDB_DB_WRITE_BATCH_INTERNAL_H_ + +#include "../hyperleveldb/write_batch.h" + +namespace hyperleveldb { + +class MemTable; + +// WriteBatchInternal provides static methods for manipulating a +// WriteBatch that we don't want in the public WriteBatch interface. +class WriteBatchInternal { + public: + // Return the number of entries in the batch. + static int Count(const WriteBatch* batch); + + // Set the count for the number of entries in the batch. + static void SetCount(WriteBatch* batch, int n); + + // Return the seqeunce number for the start of this batch. + static SequenceNumber Sequence(const WriteBatch* batch); + + // Store the specified number as the seqeunce number for the start of + // this batch. + static void SetSequence(WriteBatch* batch, SequenceNumber seq); + + static Slice Contents(const WriteBatch* batch) { + return Slice(batch->rep_); + } + + static size_t ByteSize(const WriteBatch* batch) { + return batch->rep_.size(); + } + + static void SetContents(WriteBatch* batch, const Slice& contents); + + static Status InsertInto(const WriteBatch* batch, MemTable* memtable); + + static void Append(WriteBatch* dst, const WriteBatch* src); +}; + +} // namespace hyperleveldb + + +#endif // STORAGE_HYPERLEVELDB_DB_WRITE_BATCH_INTERNAL_H_ diff --git a/Subtrees/hyperleveldb/db/write_batch_test.cc b/Subtrees/hyperleveldb/db/write_batch_test.cc new file mode 100644 index 0000000000..16e7c2b7d3 --- /dev/null +++ b/Subtrees/hyperleveldb/db/write_batch_test.cc @@ -0,0 +1,120 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "hyperleveldb/db.h" + +#include "memtable.h" +#include "write_batch_internal.h" +#include "../hyperleveldb/env.h" +#include "../util/logging.h" +#include "../util/testharness.h" + +namespace hyperleveldb { + +static std::string PrintContents(WriteBatch* b) { + InternalKeyComparator cmp(BytewiseComparator()); + MemTable* mem = new MemTable(cmp); + mem->Ref(); + std::string state; + Status s = WriteBatchInternal::InsertInto(b, mem); + int count = 0; + Iterator* iter = mem->NewIterator(); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey ikey; + ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey)); + switch (ikey.type) { + case kTypeValue: + state.append("Put("); + state.append(ikey.user_key.ToString()); + state.append(", "); + state.append(iter->value().ToString()); + state.append(")"); + count++; + break; + case kTypeDeletion: + state.append("Delete("); + state.append(ikey.user_key.ToString()); + state.append(")"); + count++; + break; + } + state.append("@"); + state.append(NumberToString(ikey.sequence)); + } + delete iter; + if (!s.ok()) { + state.append("ParseError()"); + } else if (count != WriteBatchInternal::Count(b)) { + state.append("CountMismatch()"); + } + mem->Unref(); + return state; +} + +class WriteBatchTest { }; + +TEST(WriteBatchTest, Empty) { + WriteBatch batch; + ASSERT_EQ("", PrintContents(&batch)); + ASSERT_EQ(0, WriteBatchInternal::Count(&batch)); +} + +TEST(WriteBatchTest, Multiple) { + WriteBatch batch; + batch.Put(Slice("foo"), Slice("bar")); + batch.Delete(Slice("box")); + batch.Put(Slice("baz"), Slice("boo")); + WriteBatchInternal::SetSequence(&batch, 100); + ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch)); + ASSERT_EQ(3, WriteBatchInternal::Count(&batch)); + ASSERT_EQ("Put(baz, boo)@102" + "Delete(box)@101" + "Put(foo, bar)@100", + PrintContents(&batch)); +} + +TEST(WriteBatchTest, Corruption) { + WriteBatch batch; + batch.Put(Slice("foo"), Slice("bar")); + batch.Delete(Slice("box")); + WriteBatchInternal::SetSequence(&batch, 200); + Slice contents = WriteBatchInternal::Contents(&batch); + WriteBatchInternal::SetContents(&batch, + Slice(contents.data(),contents.size()-1)); + ASSERT_EQ("Put(foo, bar)@200" + "ParseError()", + PrintContents(&batch)); +} + +TEST(WriteBatchTest, Append) { + WriteBatch b1, b2; + WriteBatchInternal::SetSequence(&b1, 200); + WriteBatchInternal::SetSequence(&b2, 300); + WriteBatchInternal::Append(&b1, &b2); + ASSERT_EQ("", + PrintContents(&b1)); + b2.Put("a", "va"); + WriteBatchInternal::Append(&b1, &b2); + ASSERT_EQ("Put(a, va)@200", + PrintContents(&b1)); + b2.Clear(); + b2.Put("b", "vb"); + WriteBatchInternal::Append(&b1, &b2); + ASSERT_EQ("Put(a, va)@200" + "Put(b, vb)@201", + PrintContents(&b1)); + b2.Delete("foo"); + WriteBatchInternal::Append(&b1, &b2); + ASSERT_EQ("Put(a, va)@200" + "Put(b, vb)@202" + "Put(b, vb)@201" + "Delete(foo)@203", + PrintContents(&b1)); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/doc/bench/db_bench_sqlite3.cc b/Subtrees/hyperleveldb/doc/bench/db_bench_sqlite3.cc new file mode 100644 index 0000000000..4c622d5707 --- /dev/null +++ b/Subtrees/hyperleveldb/doc/bench/db_bench_sqlite3.cc @@ -0,0 +1,718 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include "util/histogram.h" +#include "util/random.h" +#include "util/testutil.h" + +// Comma-separated list of operations to run in the specified order +// Actual benchmarks: +// +// fillseq -- write N values in sequential key order in async mode +// fillseqsync -- write N/100 values in sequential key order in sync mode +// fillseqbatch -- batch write N values in sequential key order in async mode +// fillrandom -- write N values in random key order in async mode +// fillrandsync -- write N/100 values in random key order in sync mode +// fillrandbatch -- batch write N values in sequential key order in async mode +// overwrite -- overwrite N values in random key order in async mode +// fillrand100K -- write N/1000 100K values in random order in async mode +// fillseq100K -- write N/1000 100K values in sequential order in async mode +// readseq -- read N times sequentially +// readrandom -- read N times in random order +// readrand100K -- read N/1000 100K values in sequential order in async mode +static const char* FLAGS_benchmarks = + "fillseq," + "fillseqsync," + "fillseqbatch," + "fillrandom," + "fillrandsync," + "fillrandbatch," + "overwrite," + "overwritebatch," + "readrandom," + "readseq," + "fillrand100K," + "fillseq100K," + "readseq," + "readrand100K," + ; + +// Number of key/values to place in database +static int FLAGS_num = 1000000; + +// Number of read operations to do. If negative, do FLAGS_num reads. +static int FLAGS_reads = -1; + +// Size of each value +static int FLAGS_value_size = 100; + +// Print histogram of operation timings +static bool FLAGS_histogram = false; + +// Arrange to generate values that shrink to this fraction of +// their original size after compression +static double FLAGS_compression_ratio = 0.5; + +// Page size. Default 1 KB. +static int FLAGS_page_size = 1024; + +// Number of pages. +// Default cache size = FLAGS_page_size * FLAGS_num_pages = 4 MB. +static int FLAGS_num_pages = 4096; + +// If true, do not destroy the existing database. If you set this +// flag and also specify a benchmark that wants a fresh database, that +// benchmark will fail. +static bool FLAGS_use_existing_db = false; + +// If true, we allow batch writes to occur +static bool FLAGS_transaction = true; + +// If true, we enable Write-Ahead Logging +static bool FLAGS_WAL_enabled = true; + +// Use the db with the following name. +static const char* FLAGS_db = NULL; + +inline +static void ExecErrorCheck(int status, char *err_msg) { + if (status != SQLITE_OK) { + fprintf(stderr, "SQL error: %s\n", err_msg); + sqlite3_free(err_msg); + exit(1); + } +} + +inline +static void StepErrorCheck(int status) { + if (status != SQLITE_DONE) { + fprintf(stderr, "SQL step error: status = %d\n", status); + exit(1); + } +} + +inline +static void ErrorCheck(int status) { + if (status != SQLITE_OK) { + fprintf(stderr, "sqlite3 error: status = %d\n", status); + exit(1); + } +} + +inline +static void WalCheckpoint(sqlite3* db_) { + // Flush all writes to disk + if (FLAGS_WAL_enabled) { + sqlite3_wal_checkpoint_v2(db_, NULL, SQLITE_CHECKPOINT_FULL, NULL, NULL); + } +} + +namespace hyperleveldb { + +// Helper for quickly generating random data. +namespace { +class RandomGenerator { + private: + std::string data_; + int pos_; + + public: + RandomGenerator() { + // We use a limited amount of data over and over again and ensure + // that it is larger than the compression window (32KB), and also + // large enough to serve all typical value sizes we want to write. + Random rnd(301); + std::string piece; + while (data_.size() < 1048576) { + // Add a short fragment that is as compressible as specified + // by FLAGS_compression_ratio. + test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece); + data_.append(piece); + } + pos_ = 0; + } + + Slice Generate(int len) { + if (pos_ + len > data_.size()) { + pos_ = 0; + assert(len < data_.size()); + } + pos_ += len; + return Slice(data_.data() + pos_ - len, len); + } +}; + +static Slice TrimSpace(Slice s) { + int start = 0; + while (start < s.size() && isspace(s[start])) { + start++; + } + int limit = s.size(); + while (limit > start && isspace(s[limit-1])) { + limit--; + } + return Slice(s.data() + start, limit - start); +} + +} // namespace + +class Benchmark { + private: + sqlite3* db_; + int db_num_; + int num_; + int reads_; + double start_; + double last_op_finish_; + int64_t bytes_; + std::string message_; + Histogram hist_; + RandomGenerator gen_; + Random rand_; + + // State kept for progress messages + int done_; + int next_report_; // When to report next + + void PrintHeader() { + const int kKeySize = 16; + PrintEnvironment(); + fprintf(stdout, "Keys: %d bytes each\n", kKeySize); + fprintf(stdout, "Values: %d bytes each\n", FLAGS_value_size); + fprintf(stdout, "Entries: %d\n", num_); + fprintf(stdout, "RawSize: %.1f MB (estimated)\n", + ((static_cast(kKeySize + FLAGS_value_size) * num_) + / 1048576.0)); + PrintWarnings(); + fprintf(stdout, "------------------------------------------------\n"); + } + + void PrintWarnings() { +#if defined(__GNUC__) && !defined(__OPTIMIZE__) + fprintf(stdout, + "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n" + ); +#endif +#ifndef NDEBUG + fprintf(stdout, + "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); +#endif + } + + void PrintEnvironment() { + fprintf(stderr, "SQLite: version %s\n", SQLITE_VERSION); + +#if defined(__linux) + time_t now = time(NULL); + fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline + + FILE* cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo != NULL) { + char line[1000]; + int num_cpus = 0; + std::string cpu_type; + std::string cache_size; + while (fgets(line, sizeof(line), cpuinfo) != NULL) { + const char* sep = strchr(line, ':'); + if (sep == NULL) { + continue; + } + Slice key = TrimSpace(Slice(line, sep - 1 - line)); + Slice val = TrimSpace(Slice(sep + 1)); + if (key == "model name") { + ++num_cpus; + cpu_type = val.ToString(); + } else if (key == "cache size") { + cache_size = val.ToString(); + } + } + fclose(cpuinfo); + fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str()); + fprintf(stderr, "CPUCache: %s\n", cache_size.c_str()); + } +#endif + } + + void Start() { + start_ = Env::Default()->NowMicros() * 1e-6; + bytes_ = 0; + message_.clear(); + last_op_finish_ = start_; + hist_.Clear(); + done_ = 0; + next_report_ = 100; + } + + void FinishedSingleOp() { + if (FLAGS_histogram) { + double now = Env::Default()->NowMicros() * 1e-6; + double micros = (now - last_op_finish_) * 1e6; + hist_.Add(micros); + if (micros > 20000) { + fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); + fflush(stderr); + } + last_op_finish_ = now; + } + + done_++; + if (done_ >= next_report_) { + if (next_report_ < 1000) next_report_ += 100; + else if (next_report_ < 5000) next_report_ += 500; + else if (next_report_ < 10000) next_report_ += 1000; + else if (next_report_ < 50000) next_report_ += 5000; + else if (next_report_ < 100000) next_report_ += 10000; + else if (next_report_ < 500000) next_report_ += 50000; + else next_report_ += 100000; + fprintf(stderr, "... finished %d ops%30s\r", done_, ""); + fflush(stderr); + } + } + + void Stop(const Slice& name) { + double finish = Env::Default()->NowMicros() * 1e-6; + + // Pretend at least one op was done in case we are running a benchmark + // that does not call FinishedSingleOp(). + if (done_ < 1) done_ = 1; + + if (bytes_ > 0) { + char rate[100]; + snprintf(rate, sizeof(rate), "%6.1f MB/s", + (bytes_ / 1048576.0) / (finish - start_)); + if (!message_.empty()) { + message_ = std::string(rate) + " " + message_; + } else { + message_ = rate; + } + } + + fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", + name.ToString().c_str(), + (finish - start_) * 1e6 / done_, + (message_.empty() ? "" : " "), + message_.c_str()); + if (FLAGS_histogram) { + fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); + } + fflush(stdout); + } + + public: + enum Order { + SEQUENTIAL, + RANDOM + }; + enum DBState { + FRESH, + EXISTING + }; + + Benchmark() + : db_(NULL), + db_num_(0), + num_(FLAGS_num), + reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), + bytes_(0), + rand_(301) { + std::vector files; + std::string test_dir; + Env::Default()->GetTestDirectory(&test_dir); + Env::Default()->GetChildren(test_dir, &files); + if (!FLAGS_use_existing_db) { + for (int i = 0; i < files.size(); i++) { + if (Slice(files[i]).starts_with("dbbench_sqlite3")) { + std::string file_name(test_dir); + file_name += "/"; + file_name += files[i]; + Env::Default()->DeleteFile(file_name.c_str()); + } + } + } + } + + ~Benchmark() { + int status = sqlite3_close(db_); + ErrorCheck(status); + } + + void Run() { + PrintHeader(); + Open(); + + const char* benchmarks = FLAGS_benchmarks; + while (benchmarks != NULL) { + const char* sep = strchr(benchmarks, ','); + Slice name; + if (sep == NULL) { + name = benchmarks; + benchmarks = NULL; + } else { + name = Slice(benchmarks, sep - benchmarks); + benchmarks = sep + 1; + } + + bytes_ = 0; + Start(); + + bool known = true; + bool write_sync = false; + if (name == Slice("fillseq")) { + Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillseqbatch")) { + Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1000); + WalCheckpoint(db_); + } else if (name == Slice("fillrandom")) { + Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillrandbatch")) { + Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1000); + WalCheckpoint(db_); + } else if (name == Slice("overwrite")) { + Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("overwritebatch")) { + Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1000); + WalCheckpoint(db_); + } else if (name == Slice("fillrandsync")) { + write_sync = true; + Write(write_sync, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillseqsync")) { + write_sync = true; + Write(write_sync, SEQUENTIAL, FRESH, num_ / 100, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillrand100K")) { + Write(write_sync, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillseq100K")) { + Write(write_sync, SEQUENTIAL, FRESH, num_ / 1000, 100 * 1000, 1); + WalCheckpoint(db_); + } else if (name == Slice("readseq")) { + ReadSequential(); + } else if (name == Slice("readrandom")) { + Read(RANDOM, 1); + } else if (name == Slice("readrand100K")) { + int n = reads_; + reads_ /= 1000; + Read(RANDOM, 1); + reads_ = n; + } else { + known = false; + if (name != Slice()) { // No error message for empty name + fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str()); + } + } + if (known) { + Stop(name); + } + } + } + + void Open() { + assert(db_ == NULL); + + int status; + char file_name[100]; + char* err_msg = NULL; + db_num_++; + + // Open database + std::string tmp_dir; + Env::Default()->GetTestDirectory(&tmp_dir); + snprintf(file_name, sizeof(file_name), + "%s/dbbench_sqlite3-%d.db", + tmp_dir.c_str(), + db_num_); + status = sqlite3_open(file_name, &db_); + if (status) { + fprintf(stderr, "open error: %s\n", sqlite3_errmsg(db_)); + exit(1); + } + + // Change SQLite cache size + char cache_size[100]; + snprintf(cache_size, sizeof(cache_size), "PRAGMA cache_size = %d", + FLAGS_num_pages); + status = sqlite3_exec(db_, cache_size, NULL, NULL, &err_msg); + ExecErrorCheck(status, err_msg); + + // FLAGS_page_size is defaulted to 1024 + if (FLAGS_page_size != 1024) { + char page_size[100]; + snprintf(page_size, sizeof(page_size), "PRAGMA page_size = %d", + FLAGS_page_size); + status = sqlite3_exec(db_, page_size, NULL, NULL, &err_msg); + ExecErrorCheck(status, err_msg); + } + + // Change journal mode to WAL if WAL enabled flag is on + if (FLAGS_WAL_enabled) { + std::string WAL_stmt = "PRAGMA journal_mode = WAL"; + + // LevelDB's default cache size is a combined 4 MB + std::string WAL_checkpoint = "PRAGMA wal_autocheckpoint = 4096"; + status = sqlite3_exec(db_, WAL_stmt.c_str(), NULL, NULL, &err_msg); + ExecErrorCheck(status, err_msg); + status = sqlite3_exec(db_, WAL_checkpoint.c_str(), NULL, NULL, &err_msg); + ExecErrorCheck(status, err_msg); + } + + // Change locking mode to exclusive and create tables/index for database + std::string locking_stmt = "PRAGMA locking_mode = EXCLUSIVE"; + std::string create_stmt = + "CREATE TABLE test (key blob, value blob, PRIMARY KEY(key))"; + std::string stmt_array[] = { locking_stmt, create_stmt }; + int stmt_array_length = sizeof(stmt_array) / sizeof(std::string); + for (int i = 0; i < stmt_array_length; i++) { + status = sqlite3_exec(db_, stmt_array[i].c_str(), NULL, NULL, &err_msg); + ExecErrorCheck(status, err_msg); + } + } + + void Write(bool write_sync, Order order, DBState state, + int num_entries, int value_size, int entries_per_batch) { + // Create new database if state == FRESH + if (state == FRESH) { + if (FLAGS_use_existing_db) { + message_ = "skipping (--use_existing_db is true)"; + return; + } + sqlite3_close(db_); + db_ = NULL; + Open(); + Start(); + } + + if (num_entries != num_) { + char msg[100]; + snprintf(msg, sizeof(msg), "(%d ops)", num_entries); + message_ = msg; + } + + char* err_msg = NULL; + int status; + + sqlite3_stmt *replace_stmt, *begin_trans_stmt, *end_trans_stmt; + std::string replace_str = "REPLACE INTO test (key, value) VALUES (?, ?)"; + std::string begin_trans_str = "BEGIN TRANSACTION;"; + std::string end_trans_str = "END TRANSACTION;"; + + // Check for synchronous flag in options + std::string sync_stmt = (write_sync) ? "PRAGMA synchronous = FULL" : + "PRAGMA synchronous = OFF"; + status = sqlite3_exec(db_, sync_stmt.c_str(), NULL, NULL, &err_msg); + ExecErrorCheck(status, err_msg); + + // Preparing sqlite3 statements + status = sqlite3_prepare_v2(db_, replace_str.c_str(), -1, + &replace_stmt, NULL); + ErrorCheck(status); + status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1, + &begin_trans_stmt, NULL); + ErrorCheck(status); + status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, + &end_trans_stmt, NULL); + ErrorCheck(status); + + bool transaction = (entries_per_batch > 1); + for (int i = 0; i < num_entries; i += entries_per_batch) { + // Begin write transaction + if (FLAGS_transaction && transaction) { + status = sqlite3_step(begin_trans_stmt); + StepErrorCheck(status); + status = sqlite3_reset(begin_trans_stmt); + ErrorCheck(status); + } + + // Create and execute SQL statements + for (int j = 0; j < entries_per_batch; j++) { + const char* value = gen_.Generate(value_size).data(); + + // Create values for key-value pair + const int k = (order == SEQUENTIAL) ? i + j : + (rand_.Next() % num_entries); + char key[100]; + snprintf(key, sizeof(key), "%016d", k); + + // Bind KV values into replace_stmt + status = sqlite3_bind_blob(replace_stmt, 1, key, 16, SQLITE_STATIC); + ErrorCheck(status); + status = sqlite3_bind_blob(replace_stmt, 2, value, + value_size, SQLITE_STATIC); + ErrorCheck(status); + + // Execute replace_stmt + bytes_ += value_size + strlen(key); + status = sqlite3_step(replace_stmt); + StepErrorCheck(status); + + // Reset SQLite statement for another use + status = sqlite3_clear_bindings(replace_stmt); + ErrorCheck(status); + status = sqlite3_reset(replace_stmt); + ErrorCheck(status); + + FinishedSingleOp(); + } + + // End write transaction + if (FLAGS_transaction && transaction) { + status = sqlite3_step(end_trans_stmt); + StepErrorCheck(status); + status = sqlite3_reset(end_trans_stmt); + ErrorCheck(status); + } + } + + status = sqlite3_finalize(replace_stmt); + ErrorCheck(status); + status = sqlite3_finalize(begin_trans_stmt); + ErrorCheck(status); + status = sqlite3_finalize(end_trans_stmt); + ErrorCheck(status); + } + + void Read(Order order, int entries_per_batch) { + int status; + sqlite3_stmt *read_stmt, *begin_trans_stmt, *end_trans_stmt; + + std::string read_str = "SELECT * FROM test WHERE key = ?"; + std::string begin_trans_str = "BEGIN TRANSACTION;"; + std::string end_trans_str = "END TRANSACTION;"; + + // Preparing sqlite3 statements + status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1, + &begin_trans_stmt, NULL); + ErrorCheck(status); + status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, + &end_trans_stmt, NULL); + ErrorCheck(status); + status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &read_stmt, NULL); + ErrorCheck(status); + + bool transaction = (entries_per_batch > 1); + for (int i = 0; i < reads_; i += entries_per_batch) { + // Begin read transaction + if (FLAGS_transaction && transaction) { + status = sqlite3_step(begin_trans_stmt); + StepErrorCheck(status); + status = sqlite3_reset(begin_trans_stmt); + ErrorCheck(status); + } + + // Create and execute SQL statements + for (int j = 0; j < entries_per_batch; j++) { + // Create key value + char key[100]; + int k = (order == SEQUENTIAL) ? i + j : (rand_.Next() % reads_); + snprintf(key, sizeof(key), "%016d", k); + + // Bind key value into read_stmt + status = sqlite3_bind_blob(read_stmt, 1, key, 16, SQLITE_STATIC); + ErrorCheck(status); + + // Execute read statement + while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {} + StepErrorCheck(status); + + // Reset SQLite statement for another use + status = sqlite3_clear_bindings(read_stmt); + ErrorCheck(status); + status = sqlite3_reset(read_stmt); + ErrorCheck(status); + FinishedSingleOp(); + } + + // End read transaction + if (FLAGS_transaction && transaction) { + status = sqlite3_step(end_trans_stmt); + StepErrorCheck(status); + status = sqlite3_reset(end_trans_stmt); + ErrorCheck(status); + } + } + + status = sqlite3_finalize(read_stmt); + ErrorCheck(status); + status = sqlite3_finalize(begin_trans_stmt); + ErrorCheck(status); + status = sqlite3_finalize(end_trans_stmt); + ErrorCheck(status); + } + + void ReadSequential() { + int status; + sqlite3_stmt *pStmt; + std::string read_str = "SELECT * FROM test ORDER BY key"; + + status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &pStmt, NULL); + ErrorCheck(status); + for (int i = 0; i < reads_ && SQLITE_ROW == sqlite3_step(pStmt); i++) { + bytes_ += sqlite3_column_bytes(pStmt, 1) + sqlite3_column_bytes(pStmt, 2); + FinishedSingleOp(); + } + + status = sqlite3_finalize(pStmt); + ErrorCheck(status); + } + +}; + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + std::string default_db_path; + for (int i = 1; i < argc; i++) { + double d; + int n; + char junk; + if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) { + FLAGS_benchmarks = argv[i] + strlen("--benchmarks="); + } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_histogram = n; + } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) { + FLAGS_compression_ratio = d; + } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_use_existing_db = n; + } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { + FLAGS_num = n; + } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { + FLAGS_reads = n; + } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { + FLAGS_value_size = n; + } else if (leveldb::Slice(argv[i]) == leveldb::Slice("--no_transaction")) { + FLAGS_transaction = false; + } else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) { + FLAGS_page_size = n; + } else if (sscanf(argv[i], "--num_pages=%d%c", &n, &junk) == 1) { + FLAGS_num_pages = n; + } else if (sscanf(argv[i], "--WAL_enabled=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_WAL_enabled = n; + } else if (strncmp(argv[i], "--db=", 5) == 0) { + FLAGS_db = argv[i] + 5; + } else { + fprintf(stderr, "Invalid flag '%s'\n", argv[i]); + exit(1); + } + } + + // Choose a location for the test database if none given with --db= + if (FLAGS_db == NULL) { + leveldb::Env::Default()->GetTestDirectory(&default_db_path); + default_db_path += "/dbbench"; + FLAGS_db = default_db_path.c_str(); + } + + leveldb::Benchmark benchmark; + benchmark.Run(); + return 0; +} diff --git a/Subtrees/hyperleveldb/doc/bench/db_bench_tree_db.cc b/Subtrees/hyperleveldb/doc/bench/db_bench_tree_db.cc new file mode 100644 index 0000000000..3f6a2acea2 --- /dev/null +++ b/Subtrees/hyperleveldb/doc/bench/db_bench_tree_db.cc @@ -0,0 +1,528 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include "util/histogram.h" +#include "util/random.h" +#include "util/testutil.h" + +// Comma-separated list of operations to run in the specified order +// Actual benchmarks: +// +// fillseq -- write N values in sequential key order in async mode +// fillrandom -- write N values in random key order in async mode +// overwrite -- overwrite N values in random key order in async mode +// fillseqsync -- write N/100 values in sequential key order in sync mode +// fillrandsync -- write N/100 values in random key order in sync mode +// fillrand100K -- write N/1000 100K values in random order in async mode +// fillseq100K -- write N/1000 100K values in seq order in async mode +// readseq -- read N times sequentially +// readseq100K -- read N/1000 100K values in sequential order in async mode +// readrand100K -- read N/1000 100K values in sequential order in async mode +// readrandom -- read N times in random order +static const char* FLAGS_benchmarks = + "fillseq," + "fillseqsync," + "fillrandsync," + "fillrandom," + "overwrite," + "readrandom," + "readseq," + "fillrand100K," + "fillseq100K," + "readseq100K," + "readrand100K," + ; + +// Number of key/values to place in database +static int FLAGS_num = 1000000; + +// Number of read operations to do. If negative, do FLAGS_num reads. +static int FLAGS_reads = -1; + +// Size of each value +static int FLAGS_value_size = 100; + +// Arrange to generate values that shrink to this fraction of +// their original size after compression +static double FLAGS_compression_ratio = 0.5; + +// Print histogram of operation timings +static bool FLAGS_histogram = false; + +// Cache size. Default 4 MB +static int FLAGS_cache_size = 4194304; + +// Page size. Default 1 KB +static int FLAGS_page_size = 1024; + +// If true, do not destroy the existing database. If you set this +// flag and also specify a benchmark that wants a fresh database, that +// benchmark will fail. +static bool FLAGS_use_existing_db = false; + +// Compression flag. If true, compression is on. If false, compression +// is off. +static bool FLAGS_compression = true; + +// Use the db with the following name. +static const char* FLAGS_db = NULL; + +inline +static void DBSynchronize(kyotocabinet::TreeDB* db_) +{ + // Synchronize will flush writes to disk + if (!db_->synchronize()) { + fprintf(stderr, "synchronize error: %s\n", db_->error().name()); + } +} + +namespace hyperleveldb { + +// Helper for quickly generating random data. +namespace { +class RandomGenerator { + private: + std::string data_; + int pos_; + + public: + RandomGenerator() { + // We use a limited amount of data over and over again and ensure + // that it is larger than the compression window (32KB), and also + // large enough to serve all typical value sizes we want to write. + Random rnd(301); + std::string piece; + while (data_.size() < 1048576) { + // Add a short fragment that is as compressible as specified + // by FLAGS_compression_ratio. + test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece); + data_.append(piece); + } + pos_ = 0; + } + + Slice Generate(int len) { + if (pos_ + len > data_.size()) { + pos_ = 0; + assert(len < data_.size()); + } + pos_ += len; + return Slice(data_.data() + pos_ - len, len); + } +}; + +static Slice TrimSpace(Slice s) { + int start = 0; + while (start < s.size() && isspace(s[start])) { + start++; + } + int limit = s.size(); + while (limit > start && isspace(s[limit-1])) { + limit--; + } + return Slice(s.data() + start, limit - start); +} + +} // namespace + +class Benchmark { + private: + kyotocabinet::TreeDB* db_; + int db_num_; + int num_; + int reads_; + double start_; + double last_op_finish_; + int64_t bytes_; + std::string message_; + Histogram hist_; + RandomGenerator gen_; + Random rand_; + kyotocabinet::LZOCompressor comp_; + + // State kept for progress messages + int done_; + int next_report_; // When to report next + + void PrintHeader() { + const int kKeySize = 16; + PrintEnvironment(); + fprintf(stdout, "Keys: %d bytes each\n", kKeySize); + fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n", + FLAGS_value_size, + static_cast(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); + fprintf(stdout, "Entries: %d\n", num_); + fprintf(stdout, "RawSize: %.1f MB (estimated)\n", + ((static_cast(kKeySize + FLAGS_value_size) * num_) + / 1048576.0)); + fprintf(stdout, "FileSize: %.1f MB (estimated)\n", + (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) + / 1048576.0)); + PrintWarnings(); + fprintf(stdout, "------------------------------------------------\n"); + } + + void PrintWarnings() { +#if defined(__GNUC__) && !defined(__OPTIMIZE__) + fprintf(stdout, + "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n" + ); +#endif +#ifndef NDEBUG + fprintf(stdout, + "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); +#endif + } + + void PrintEnvironment() { + fprintf(stderr, "Kyoto Cabinet: version %s, lib ver %d, lib rev %d\n", + kyotocabinet::VERSION, kyotocabinet::LIBVER, kyotocabinet::LIBREV); + +#if defined(__linux) + time_t now = time(NULL); + fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline + + FILE* cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo != NULL) { + char line[1000]; + int num_cpus = 0; + std::string cpu_type; + std::string cache_size; + while (fgets(line, sizeof(line), cpuinfo) != NULL) { + const char* sep = strchr(line, ':'); + if (sep == NULL) { + continue; + } + Slice key = TrimSpace(Slice(line, sep - 1 - line)); + Slice val = TrimSpace(Slice(sep + 1)); + if (key == "model name") { + ++num_cpus; + cpu_type = val.ToString(); + } else if (key == "cache size") { + cache_size = val.ToString(); + } + } + fclose(cpuinfo); + fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str()); + fprintf(stderr, "CPUCache: %s\n", cache_size.c_str()); + } +#endif + } + + void Start() { + start_ = Env::Default()->NowMicros() * 1e-6; + bytes_ = 0; + message_.clear(); + last_op_finish_ = start_; + hist_.Clear(); + done_ = 0; + next_report_ = 100; + } + + void FinishedSingleOp() { + if (FLAGS_histogram) { + double now = Env::Default()->NowMicros() * 1e-6; + double micros = (now - last_op_finish_) * 1e6; + hist_.Add(micros); + if (micros > 20000) { + fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); + fflush(stderr); + } + last_op_finish_ = now; + } + + done_++; + if (done_ >= next_report_) { + if (next_report_ < 1000) next_report_ += 100; + else if (next_report_ < 5000) next_report_ += 500; + else if (next_report_ < 10000) next_report_ += 1000; + else if (next_report_ < 50000) next_report_ += 5000; + else if (next_report_ < 100000) next_report_ += 10000; + else if (next_report_ < 500000) next_report_ += 50000; + else next_report_ += 100000; + fprintf(stderr, "... finished %d ops%30s\r", done_, ""); + fflush(stderr); + } + } + + void Stop(const Slice& name) { + double finish = Env::Default()->NowMicros() * 1e-6; + + // Pretend at least one op was done in case we are running a benchmark + // that does not call FinishedSingleOp(). + if (done_ < 1) done_ = 1; + + if (bytes_ > 0) { + char rate[100]; + snprintf(rate, sizeof(rate), "%6.1f MB/s", + (bytes_ / 1048576.0) / (finish - start_)); + if (!message_.empty()) { + message_ = std::string(rate) + " " + message_; + } else { + message_ = rate; + } + } + + fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", + name.ToString().c_str(), + (finish - start_) * 1e6 / done_, + (message_.empty() ? "" : " "), + message_.c_str()); + if (FLAGS_histogram) { + fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); + } + fflush(stdout); + } + + public: + enum Order { + SEQUENTIAL, + RANDOM + }; + enum DBState { + FRESH, + EXISTING + }; + + Benchmark() + : db_(NULL), + num_(FLAGS_num), + reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), + bytes_(0), + rand_(301) { + std::vector files; + std::string test_dir; + Env::Default()->GetTestDirectory(&test_dir); + Env::Default()->GetChildren(test_dir.c_str(), &files); + if (!FLAGS_use_existing_db) { + for (int i = 0; i < files.size(); i++) { + if (Slice(files[i]).starts_with("dbbench_polyDB")) { + std::string file_name(test_dir); + file_name += "/"; + file_name += files[i]; + Env::Default()->DeleteFile(file_name.c_str()); + } + } + } + } + + ~Benchmark() { + if (!db_->close()) { + fprintf(stderr, "close error: %s\n", db_->error().name()); + } + } + + void Run() { + PrintHeader(); + Open(false); + + const char* benchmarks = FLAGS_benchmarks; + while (benchmarks != NULL) { + const char* sep = strchr(benchmarks, ','); + Slice name; + if (sep == NULL) { + name = benchmarks; + benchmarks = NULL; + } else { + name = Slice(benchmarks, sep - benchmarks); + benchmarks = sep + 1; + } + + Start(); + + bool known = true; + bool write_sync = false; + if (name == Slice("fillseq")) { + Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1); + + } else if (name == Slice("fillrandom")) { + Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1); + DBSynchronize(db_); + } else if (name == Slice("overwrite")) { + Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1); + DBSynchronize(db_); + } else if (name == Slice("fillrandsync")) { + write_sync = true; + Write(write_sync, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1); + DBSynchronize(db_); + } else if (name == Slice("fillseqsync")) { + write_sync = true; + Write(write_sync, SEQUENTIAL, FRESH, num_ / 100, FLAGS_value_size, 1); + DBSynchronize(db_); + } else if (name == Slice("fillrand100K")) { + Write(write_sync, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1); + DBSynchronize(db_); + } else if (name == Slice("fillseq100K")) { + Write(write_sync, SEQUENTIAL, FRESH, num_ / 1000, 100 * 1000, 1); + DBSynchronize(db_); + } else if (name == Slice("readseq")) { + ReadSequential(); + } else if (name == Slice("readrandom")) { + ReadRandom(); + } else if (name == Slice("readrand100K")) { + int n = reads_; + reads_ /= 1000; + ReadRandom(); + reads_ = n; + } else if (name == Slice("readseq100K")) { + int n = reads_; + reads_ /= 1000; + ReadSequential(); + reads_ = n; + } else { + known = false; + if (name != Slice()) { // No error message for empty name + fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str()); + } + } + if (known) { + Stop(name); + } + } + } + + private: + void Open(bool sync) { + assert(db_ == NULL); + + // Initialize db_ + db_ = new kyotocabinet::TreeDB(); + char file_name[100]; + db_num_++; + std::string test_dir; + Env::Default()->GetTestDirectory(&test_dir); + snprintf(file_name, sizeof(file_name), + "%s/dbbench_polyDB-%d.kct", + test_dir.c_str(), + db_num_); + + // Create tuning options and open the database + int open_options = kyotocabinet::PolyDB::OWRITER | + kyotocabinet::PolyDB::OCREATE; + int tune_options = kyotocabinet::TreeDB::TSMALL | + kyotocabinet::TreeDB::TLINEAR; + if (FLAGS_compression) { + tune_options |= kyotocabinet::TreeDB::TCOMPRESS; + db_->tune_compressor(&comp_); + } + db_->tune_options(tune_options); + db_->tune_page_cache(FLAGS_cache_size); + db_->tune_page(FLAGS_page_size); + db_->tune_map(256LL<<20); + if (sync) { + open_options |= kyotocabinet::PolyDB::OAUTOSYNC; + } + if (!db_->open(file_name, open_options)) { + fprintf(stderr, "open error: %s\n", db_->error().name()); + } + } + + void Write(bool sync, Order order, DBState state, + int num_entries, int value_size, int entries_per_batch) { + // Create new database if state == FRESH + if (state == FRESH) { + if (FLAGS_use_existing_db) { + message_ = "skipping (--use_existing_db is true)"; + return; + } + delete db_; + db_ = NULL; + Open(sync); + Start(); // Do not count time taken to destroy/open + } + + if (num_entries != num_) { + char msg[100]; + snprintf(msg, sizeof(msg), "(%d ops)", num_entries); + message_ = msg; + } + + // Write to database + for (int i = 0; i < num_entries; i++) + { + const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % num_entries); + char key[100]; + snprintf(key, sizeof(key), "%016d", k); + bytes_ += value_size + strlen(key); + std::string cpp_key = key; + if (!db_->set(cpp_key, gen_.Generate(value_size).ToString())) { + fprintf(stderr, "set error: %s\n", db_->error().name()); + } + FinishedSingleOp(); + } + } + + void ReadSequential() { + kyotocabinet::DB::Cursor* cur = db_->cursor(); + cur->jump(); + std::string ckey, cvalue; + while (cur->get(&ckey, &cvalue, true)) { + bytes_ += ckey.size() + cvalue.size(); + FinishedSingleOp(); + } + delete cur; + } + + void ReadRandom() { + std::string value; + for (int i = 0; i < reads_; i++) { + char key[100]; + const int k = rand_.Next() % reads_; + snprintf(key, sizeof(key), "%016d", k); + db_->get(key, &value); + FinishedSingleOp(); + } + } +}; + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + std::string default_db_path; + for (int i = 1; i < argc; i++) { + double d; + int n; + char junk; + if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) { + FLAGS_benchmarks = argv[i] + strlen("--benchmarks="); + } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) { + FLAGS_compression_ratio = d; + } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_histogram = n; + } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { + FLAGS_num = n; + } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { + FLAGS_reads = n; + } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { + FLAGS_value_size = n; + } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) { + FLAGS_cache_size = n; + } else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) { + FLAGS_page_size = n; + } else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_compression = (n == 1) ? true : false; + } else if (strncmp(argv[i], "--db=", 5) == 0) { + FLAGS_db = argv[i] + 5; + } else { + fprintf(stderr, "Invalid flag '%s'\n", argv[i]); + exit(1); + } + } + + // Choose a location for the test database if none given with --db= + if (FLAGS_db == NULL) { + leveldb::Env::Default()->GetTestDirectory(&default_db_path); + default_db_path += "/dbbench"; + FLAGS_db = default_db_path.c_str(); + } + + leveldb::Benchmark benchmark; + benchmark.Run(); + return 0; +} diff --git a/Subtrees/hyperleveldb/doc/benchmark.html b/Subtrees/hyperleveldb/doc/benchmark.html new file mode 100644 index 0000000000..c4639772c1 --- /dev/null +++ b/Subtrees/hyperleveldb/doc/benchmark.html @@ -0,0 +1,459 @@ + + + +LevelDB Benchmarks + + + + +

LevelDB Benchmarks

+

Google, July 2011

+
+ +

In order to test LevelDB's performance, we benchmark it against other well-established database implementations. We compare LevelDB (revision 39) against SQLite3 (version 3.7.6.3) and Kyoto Cabinet's (version 1.2.67) TreeDB (a B+Tree based key-value store). We would like to acknowledge Scott Hess and Mikio Hirabayashi for their suggestions and contributions to the SQLite3 and Kyoto Cabinet benchmarks, respectively.

+ +

Benchmarks were all performed on a six-core Intel(R) Xeon(R) CPU X5650 @ 2.67GHz, with 12288 KB of total L3 cache and 12 GB of DDR3 RAM at 1333 MHz. (Note that LevelDB uses at most two CPUs since the benchmarks are single threaded: one to run the benchmark, and one for background compactions.) We ran the benchmarks on two machines (with identical processors), one with an Ext3 file system and one with an Ext4 file system. The machine with the Ext3 file system has a SATA Hitachi HDS721050CLA362 hard drive. The machine with the Ext4 file system has a SATA Samsung HD502HJ hard drive. Both hard drives spin at 7200 RPM and have hard drive write-caching enabled (using `hdparm -W 1 [device]`). The numbers reported below are the median of three measurements.

+ +

Benchmark Source Code

+

We wrote benchmark tools for SQLite and Kyoto TreeDB based on LevelDB's db_bench. The code for each of the benchmarks resides here:

+ + +

Custom Build Specifications

+
    +
  • LevelDB: LevelDB was compiled with the tcmalloc library and the Snappy compression library (revision 33). Assertions were disabled.
  • +
  • TreeDB: TreeDB was compiled using the LZO compression library (version 2.03). Furthermore, we enabled the TSMALL and TLINEAR options when opening the database in order to reduce the footprint of each record.
  • +
  • SQLite: We tuned SQLite's performance, by setting its locking mode to exclusive. We also enabled SQLite's write-ahead logging.
  • +
+ +

1. Baseline Performance

+

This section gives the baseline performance of all the +databases. Following sections show how performance changes as various +parameters are varied. For the baseline:

+
    +
  • Each database is allowed 4 MB of cache memory.
  • +
  • Databases are opened in asynchronous write mode. + (LevelDB's sync option, TreeDB's OAUTOSYNC option, and + SQLite3's synchronous options are all turned off). I.e., + every write is pushed to the operating system, but the + benchmark does not wait for the write to reach the disk.
  • +
  • Keys are 16 bytes each.
  • +
  • Value are 100 bytes each (with enough redundancy so that + a simple compressor shrinks them to 50% of their original + size).
  • +
  • Sequential reads/writes traverse the key space in increasing order.
  • +
  • Random reads/writes traverse the key space in random order.
  • +
+ +

A. Sequential Reads

+ + + + + + + + + + +
LevelDB4,030,000 ops/sec
 
Kyoto TreeDB1,010,000 ops/sec
 
SQLite3383,000 ops/sec
 
+

B. Random Reads

+ + + + + + + + + + +
LevelDB129,000 ops/sec
 
Kyoto TreeDB151,000 ops/sec
 
SQLite3134,000 ops/sec
 
+

C. Sequential Writes

+ + + + + + + + + + +
LevelDB779,000 ops/sec
 
Kyoto TreeDB342,000 ops/sec
 
SQLite348,600 ops/sec
 
+

D. Random Writes

+ + + + + + + + + + +
LevelDB164,000 ops/sec
 
Kyoto TreeDB88,500 ops/sec
 
SQLite39,860 ops/sec
 
+ +

LevelDB outperforms both SQLite3 and TreeDB in sequential and random write operations and sequential read operations. Kyoto Cabinet has the fastest random read operations.

+ +

2. Write Performance under Different Configurations

+

A. Large Values

+

For this benchmark, we start with an empty database, and write 100,000 byte values (~50% compressible). To keep the benchmark running time reasonable, we stop after writing 1000 values.

+

Sequential Writes

+ + + + + + + + + + +
LevelDB1,100 ops/sec
 
Kyoto TreeDB1,000 ops/sec
 
SQLite31,600 ops/sec
 
+

Random Writes

+ + + + + + + + + + +
LevelDB480 ops/sec
 
Kyoto TreeDB1,100 ops/sec
 
SQLite31,600 ops/sec
 
+

LevelDB doesn't perform as well with large values of 100,000 bytes each. This is because LevelDB writes keys and values at least twice: first time to the transaction log, and second time (during a compaction) to a sorted file. +With larger values, LevelDB's per-operation efficiency is swamped by the +cost of extra copies of large values.

+

B. Batch Writes

+

A batch write is a set of writes that are applied atomically to the underlying database. A single batch of N writes may be significantly faster than N individual writes. The following benchmark writes one thousand batches where each batch contains one thousand 100-byte values. TreeDB does not support batch writes and is omitted from this benchmark.

+

Sequential Writes

+ + + + + + + + + +
LevelDB840,000 entries/sec
 
(1.08x baseline)
SQLite3124,000 entries/sec
 
(2.55x baseline)
+

Random Writes

+ + + + + + + + + +
LevelDB221,000 entries/sec
 
(1.35x baseline)
SQLite322,000 entries/sec
 
(2.23x baseline)
+ +

Because of the way LevelDB persistent storage is organized, batches of +random writes are not much slower (only a factor of 4x) than batches +of sequential writes.

+ +

C. Synchronous Writes

+

In the following benchmark, we enable the synchronous writing modes +of all of the databases. Since this change significantly slows down the +benchmark, we stop after 10,000 writes. For synchronous write tests, we've +disabled hard drive write-caching (using `hdparm -W 0 [device]`).

+
    +
  • For LevelDB, we set WriteOptions.sync = true.
  • +
  • In TreeDB, we enabled TreeDB's OAUTOSYNC option.
  • +
  • For SQLite3, we set "PRAGMA synchronous = FULL".
  • +
+

Sequential Writes

+ + + + + + + + + + + + + +
LevelDB100 ops/sec
 
(0.003x baseline)
Kyoto TreeDB7 ops/sec
 
(0.0004x baseline)
SQLite388 ops/sec
 
(0.002x baseline)
+

Random Writes

+ + + + + + + + + + + + + +
LevelDB100 ops/sec
 
(0.015x baseline)
Kyoto TreeDB8 ops/sec
 
(0.001x baseline)
SQLite388 ops/sec
 
(0.009x baseline)
+ +

Also see the ext4 performance numbers below +since synchronous writes behave significantly differently +on ext3 and ext4.

+ +

D. Turning Compression Off

+ +

In the baseline measurements, LevelDB and TreeDB were using +light-weight compression +(Snappy for LevelDB, +and LZO for +TreeDB). SQLite3, by default does not use compression. The +experiments below show what happens when compression is disabled in +all of the databases (the SQLite3 numbers are just a copy of +its baseline measurements):

+ +

Sequential Writes

+ + + + + + + + + + + + + +
LevelDB594,000 ops/sec
 
(0.76x baseline)
Kyoto TreeDB485,000 ops/sec
 
(1.42x baseline)
SQLite348,600 ops/sec
 
(1.00x baseline)
+

Random Writes

+ + + + + + + + + + + + + +
LevelDB135,000 ops/sec
 
(0.82x baseline)
Kyoto TreeDB159,000 ops/sec
 
(1.80x baseline)
SQLite39,860 ops/sec
 
(1.00x baseline)
+ +

LevelDB's write performance is better with compression than without +since compression decreases the amount of data that has to be written +to disk. Therefore LevelDB users can leave compression enabled in +most scenarios without having worry about a tradeoff between space +usage and performance. TreeDB's performance on the other hand is +better without compression than with compression. Presumably this is +because TreeDB's compression library (LZO) is more expensive than +LevelDB's compression library (Snappy).

+ +

E. Using More Memory

+

We increased the overall cache size for each database to 128 MB. For LevelDB, we partitioned 128 MB into a 120 MB write buffer and 8 MB of cache (up from 2 MB of write buffer and 2 MB of cache). For SQLite3, we kept the page size at 1024 bytes, but increased the number of pages to 131,072 (up from 4096). For TreeDB, we also kept the page size at 1024 bytes, but increased the cache size to 128 MB (up from 4 MB).

+

Sequential Writes

+ + + + + + + + + + + + + +
LevelDB812,000 ops/sec
 
(1.04x baseline)
Kyoto TreeDB321,000 ops/sec
 
(0.94x baseline)
SQLite348,500 ops/sec
 
(1.00x baseline)
+

Random Writes

+ + + + + + + + + + + + + +
LevelDB355,000 ops/sec
 
(2.16x baseline)
Kyoto TreeDB284,000 ops/sec
 
(3.21x baseline)
SQLite39,670 ops/sec
 
(0.98x baseline)
+ +

SQLite's performance does not change substantially when compared to +the baseline, but the random write performance for both LevelDB and +TreeDB increases significantly. LevelDB's performance improves +because a larger write buffer reduces the need to merge sorted files +(since it creates a smaller number of larger sorted files). TreeDB's +performance goes up because the entire database is available in memory +for fast in-place updates.

+ +

3. Read Performance under Different Configurations

+

A. Larger Caches

+

We increased the overall memory usage to 128 MB for each database. +For LevelDB, we allocated 8 MB to LevelDB's write buffer and 120 MB +to LevelDB's cache. The other databases don't differentiate between a +write buffer and a cache, so we simply set their cache size to 128 +MB.

+

Sequential Reads

+ + + + + + + + + + + + + +
LevelDB5,210,000 ops/sec
 
(1.29x baseline)
Kyoto TreeDB1,070,000 ops/sec
 
(1.06x baseline)
SQLite3609,000 ops/sec
 
(1.59x baseline)
+ +

Random Reads

+ + + + + + + + + + + + + +
LevelDB190,000 ops/sec
 
(1.47x baseline)
Kyoto TreeDB463,000 ops/sec
 
(3.07x baseline)
SQLite3186,000 ops/sec
 
(1.39x baseline)
+ +

As expected, the read performance of all of the databases increases +when the caches are enlarged. In particular, TreeDB seems to make +very effective use of a cache that is large enough to hold the entire +database.

+ +

B. No Compression Reads

+

For this benchmark, we populated a database with 1 million entries consisting of 16 byte keys and 100 byte values. We compiled LevelDB and Kyoto Cabinet without compression support, so results that are read out from the database are already uncompressed. We've listed the SQLite3 baseline read performance as a point of comparison.

+

Sequential Reads

+ + + + + + + + + + + + + +
LevelDB4,880,000 ops/sec
 
(1.21x baseline)
Kyoto TreeDB1,230,000 ops/sec
 
(3.60x baseline)
SQLite3383,000 ops/sec
 
(1.00x baseline)
+

Random Reads

+ + + + + + + + + + + + + +
LevelDB149,000 ops/sec
 
(1.16x baseline)
Kyoto TreeDB175,000 ops/sec
 
(1.16x baseline)
SQLite3134,000 ops/sec
 
(1.00x baseline)
+ +

Performance of both LevelDB and TreeDB improves a small amount when +compression is disabled. Note however that under different workloads, +performance may very well be better with compression if it allows more +of the working set to fit in memory.

+ +

Note about Ext4 Filesystems

+

The preceding numbers are for an ext3 file system. Synchronous writes are much slower under ext4 (LevelDB drops to ~31 writes / second and TreeDB drops to ~5 writes / second; SQLite3's synchronous writes do not noticeably drop) due to ext4's different handling of fsync / msync calls. Even LevelDB's asynchronous write performance drops somewhat since it spreads its storage across multiple files and issues fsync calls when switching to a new file.

+ +

Acknowledgements

+

Jeff Dean and Sanjay Ghemawat wrote LevelDB. Kevin Tseng wrote and compiled these benchmarks. Mikio Hirabayashi, Scott Hess, and Gabor Cselle provided help and advice.

+ + diff --git a/Subtrees/hyperleveldb/doc/doc.css b/Subtrees/hyperleveldb/doc/doc.css new file mode 100644 index 0000000000..700c564e43 --- /dev/null +++ b/Subtrees/hyperleveldb/doc/doc.css @@ -0,0 +1,89 @@ +body { + margin-left: 0.5in; + margin-right: 0.5in; + background: white; + color: black; +} + +h1 { + margin-left: -0.2in; + font-size: 14pt; +} +h2 { + margin-left: -0in; + font-size: 12pt; +} +h3 { + margin-left: -0in; +} +h4 { + margin-left: -0in; +} +hr { + margin-left: -0in; +} + +/* Definition lists: definition term bold */ +dt { + font-weight: bold; +} + +address { + text-align: center; +} +code,samp,var { + color: blue; +} +kbd { + color: #600000; +} +div.note p { + float: right; + width: 3in; + margin-right: 0%; + padding: 1px; + border: 2px solid #6060a0; + background-color: #fffff0; +} + +ul { + margin-top: -0em; + margin-bottom: -0em; +} + +ol { + margin-top: -0em; + margin-bottom: -0em; +} + +UL.nobullets { + list-style-type: none; + list-style-image: none; + margin-left: -1em; +} + +p { + margin: 1em 0 1em 0; + padding: 0 0 0 0; +} + +pre { + line-height: 1.3em; + padding: 0.4em 0 0.8em 0; + margin: 0 0 0 0; + border: 0 0 0 0; + color: blue; +} + +.datatable { + margin-left: auto; + margin-right: auto; + margin-top: 2em; + margin-bottom: 2em; + border: 1px solid; +} + +.datatable td,th { + padding: 0 0.5em 0 0.5em; + text-align: right; +} diff --git a/Subtrees/hyperleveldb/doc/impl.html b/Subtrees/hyperleveldb/doc/impl.html new file mode 100644 index 0000000000..e870795d23 --- /dev/null +++ b/Subtrees/hyperleveldb/doc/impl.html @@ -0,0 +1,213 @@ + + + + +Leveldb file layout and compactions + + + + +

Files

+ +The implementation of leveldb is similar in spirit to the +representation of a single + +Bigtable tablet (section 5.3). +However the organization of the files that make up the representation +is somewhat different and is explained below. + +

+Each database is represented by a set of files stored in a directory. +There are several different types of files as documented below: +

+

Log files

+

+A log file (*.log) stores a sequence of recent updates. Each update +is appended to the current log file. When the log file reaches a +pre-determined size (approximately 4MB by default), it is converted +to a sorted table (see below) and a new log file is created for future +updates. +

+A copy of the current log file is kept in an in-memory structure (the +memtable). This copy is consulted on every read so that read +operations reflect all logged updates. +

+

Sorted tables

+

+A sorted table (*.sst) stores a sequence of entries sorted by key. +Each entry is either a value for the key, or a deletion marker for the +key. (Deletion markers are kept around to hide obsolete values +present in older sorted tables). +

+The set of sorted tables are organized into a sequence of levels. The +sorted table generated from a log file is placed in a special young +level (also called level-0). When the number of young files exceeds a +certain threshold (currently four), all of the young files are merged +together with all of the overlapping level-1 files to produce a +sequence of new level-1 files (we create a new level-1 file for every +2MB of data.) +

+Files in the young level may contain overlapping keys. However files +in other levels have distinct non-overlapping key ranges. Consider +level number L where L >= 1. When the combined size of files in +level-L exceeds (10^L) MB (i.e., 10MB for level-1, 100MB for level-2, +...), one file in level-L, and all of the overlapping files in +level-(L+1) are merged to form a set of new files for level-(L+1). +These merges have the effect of gradually migrating new updates from +the young level to the largest level using only bulk reads and writes +(i.e., minimizing expensive seeks). + +

Manifest

+

+A MANIFEST file lists the set of sorted tables that make up each +level, the corresponding key ranges, and other important metadata. +A new MANIFEST file (with a new number embedded in the file name) +is created whenever the database is reopened. The MANIFEST file is +formatted as a log, and changes made to the serving state (as files +are added or removed) are appended to this log. +

+

Current

+

+CURRENT is a simple text file that contains the name of the latest +MANIFEST file. +

+

Info logs

+

+Informational messages are printed to files named LOG and LOG.old. +

+

Others

+

+Other files used for miscellaneous purposes may also be present +(LOCK, *.dbtmp). + +

Level 0

+When the log file grows above a certain size (1MB by default): +
    +
  • Create a brand new memtable and log file and direct future updates here +
  • In the background: +
      +
    • Write the contents of the previous memtable to an sstable +
    • Discard the memtable +
    • Delete the old log file and the old memtable +
    • Add the new sstable to the young (level-0) level. +
    +
+ +

Compactions

+ +

+When the size of level L exceeds its limit, we compact it in a +background thread. The compaction picks a file from level L and all +overlapping files from the next level L+1. Note that if a level-L +file overlaps only part of a level-(L+1) file, the entire file at +level-(L+1) is used as an input to the compaction and will be +discarded after the compaction. Aside: because level-0 is special +(files in it may overlap each other), we treat compactions from +level-0 to level-1 specially: a level-0 compaction may pick more than +one level-0 file in case some of these files overlap each other. + +

+A compaction merges the contents of the picked files to produce a +sequence of level-(L+1) files. We switch to producing a new +level-(L+1) file after the current output file has reached the target +file size (2MB). We also switch to a new output file when the key +range of the current output file has grown enough to overlap more then +ten level-(L+2) files. This last rule ensures that a later compaction +of a level-(L+1) file will not pick up too much data from level-(L+2). + +

+The old files are discarded and the new files are added to the serving +state. + +

+Compactions for a particular level rotate through the key space. In +more detail, for each level L, we remember the ending key of the last +compaction at level L. The next compaction for level L will pick the +first file that starts after this key (wrapping around to the +beginning of the key space if there is no such file). + +

+Compactions drop overwritten values. They also drop deletion markers +if there are no higher numbered levels that contain a file whose range +overlaps the current key. + +

Timing

+ +Level-0 compactions will read up to four 1MB files from level-0, and +at worst all the level-1 files (10MB). I.e., we will read 14MB and +write 14MB. + +

+Other than the special level-0 compactions, we will pick one 2MB file +from level L. In the worst case, this will overlap ~ 12 files from +level L+1 (10 because level-(L+1) is ten times the size of level-L, +and another two at the boundaries since the file ranges at level-L +will usually not be aligned with the file ranges at level-L+1). The +compaction will therefore read 26MB and write 26MB. Assuming a disk +IO rate of 100MB/s (ballpark range for modern drives), the worst +compaction cost will be approximately 0.5 second. + +

+If we throttle the background writing to something small, say 10% of +the full 100MB/s speed, a compaction may take up to 5 seconds. If the +user is writing at 10MB/s, we might build up lots of level-0 files +(~50 to hold the 5*10MB). This may signficantly increase the cost of +reads due to the overhead of merging more files together on every +read. + +

+Solution 1: To reduce this problem, we might want to increase the log +switching threshold when the number of level-0 files is large. Though +the downside is that the larger this threshold, the more memory we will +need to hold the corresponding memtable. + +

+Solution 2: We might want to decrease write rate artificially when the +number of level-0 files goes up. + +

+Solution 3: We work on reducing the cost of very wide merges. +Perhaps most of the level-0 files will have their blocks sitting +uncompressed in the cache and we will only need to worry about the +O(N) complexity in the merging iterator. + +

Number of files

+ +Instead of always making 2MB files, we could make larger files for +larger levels to reduce the total file count, though at the expense of +more bursty compactions. Alternatively, we could shard the set of +files into multiple directories. + +

+An experiment on an ext3 filesystem on Feb 04, 2011 shows +the following timings to do 100K file opens in directories with +varying number of files: + + + + + +
Files in directoryMicroseconds to open a file
10009
1000010
10000016
+So maybe even the sharding is not necessary on modern filesystems? + +

Recovery

+ +
    +
  • Read CURRENT to find name of the latest committed MANIFEST +
  • Read the named MANIFEST file +
  • Clean up stale files +
  • We could open all sstables here, but it is probably better to be lazy... +
  • Convert log chunk to a new level-0 sstable +
  • Start directing new writes to a new log file with recovered sequence# +
+ +

Garbage collection of files

+ +DeleteObsoleteFiles() is called at the end of every +compaction and at the end of recovery. It finds the names of all +files in the database. It deletes all log files that are not the +current log file. It deletes all table files that are not referenced +from some level and are not the output of an active compaction. + + + diff --git a/Subtrees/hyperleveldb/doc/index.html b/Subtrees/hyperleveldb/doc/index.html new file mode 100644 index 0000000000..3ed0ed9d9e --- /dev/null +++ b/Subtrees/hyperleveldb/doc/index.html @@ -0,0 +1,549 @@ + + + + +Leveldb + + + +

Leveldb

+
Jeff Dean, Sanjay Ghemawat
+

+The leveldb library provides a persistent key value store. Keys and +values are arbitrary byte arrays. The keys are ordered within the key +value store according to a user-specified comparator function. + +

+

Opening A Database

+

+A leveldb database has a name which corresponds to a file system +directory. All of the contents of database are stored in this +directory. The following example shows how to open a database, +creating it if necessary: +

+

+  #include <assert>
+  #include "leveldb/db.h"
+
+  leveldb::DB* db;
+  leveldb::Options options;
+  options.create_if_missing = true;
+  leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
+  assert(status.ok());
+  ...
+
+If you want to raise an error if the database already exists, add +the following line before the leveldb::DB::Open call: +
+  options.error_if_exists = true;
+
+

Status

+

+You may have noticed the leveldb::Status type above. Values of this +type are returned by most functions in leveldb that may encounter an +error. You can check if such a result is ok, and also print an +associated error message: +

+

+   leveldb::Status s = ...;
+   if (!s.ok()) cerr << s.ToString() << endl;
+
+

Closing A Database

+

+When you are done with a database, just delete the database object. +Example: +

+

+  ... open the db as described above ...
+  ... do something with db ...
+  delete db;
+
+

Reads And Writes

+

+The database provides Put, Delete, and Get methods to +modify/query the database. For example, the following code +moves the value stored under key1 to key2. +

+  std::string value;
+  leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
+  if (s.ok()) s = db->Put(leveldb::WriteOptions(), key2, value);
+  if (s.ok()) s = db->Delete(leveldb::WriteOptions(), key1);
+
+ +

Atomic Updates

+

+Note that if the process dies after the Put of key2 but before the +delete of key1, the same value may be left stored under multiple keys. +Such problems can be avoided by using the WriteBatch class to +atomically apply a set of updates: +

+

+  #include "leveldb/write_batch.h"
+  ...
+  std::string value;
+  leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
+  if (s.ok()) {
+    leveldb::WriteBatch batch;
+    batch.Delete(key1);
+    batch.Put(key2, value);
+    s = db->Write(leveldb::WriteOptions(), &batch);
+  }
+
+The WriteBatch holds a sequence of edits to be made to the database, +and these edits within the batch are applied in order. Note that we +called Delete before Put so that if key1 is identical to key2, +we do not end up erroneously dropping the value entirely. +

+Apart from its atomicity benefits, WriteBatch may also be used to +speed up bulk updates by placing lots of individual mutations into the +same batch. + +

Synchronous Writes

+By default, each write to leveldb is asynchronous: it +returns after pushing the write from the process into the operating +system. The transfer from operating system memory to the underlying +persistent storage happens asynchronously. The sync flag +can be turned on for a particular write to make the write operation +not return until the data being written has been pushed all the way to +persistent storage. (On Posix systems, this is implemented by calling +either fsync(...) or fdatasync(...) or +msync(..., MS_SYNC) before the write operation returns.) +
+  leveldb::WriteOptions write_options;
+  write_options.sync = true;
+  db->Put(write_options, ...);
+
+Asynchronous writes are often more than a thousand times as fast as +synchronous writes. The downside of asynchronous writes is that a +crash of the machine may cause the last few updates to be lost. Note +that a crash of just the writing process (i.e., not a reboot) will not +cause any loss since even when sync is false, an update +is pushed from the process memory into the operating system before it +is considered done. + +

+Asynchronous writes can often be used safely. For example, when +loading a large amount of data into the database you can handle lost +updates by restarting the bulk load after a crash. A hybrid scheme is +also possible where every Nth write is synchronous, and in the event +of a crash, the bulk load is restarted just after the last synchronous +write finished by the previous run. (The synchronous write can update +a marker that describes where to restart on a crash.) + +

+WriteBatch provides an alternative to asynchronous writes. +Multiple updates may be placed in the same WriteBatch and +applied together using a synchronous write (i.e., +write_options.sync is set to true). The extra cost of +the synchronous write will be amortized across all of the writes in +the batch. + +

+

Concurrency

+

+A database may only be opened by one process at a time. +The leveldb implementation acquires a lock from the +operating system to prevent misuse. Within a single process, the +same leveldb::DB object may be safely shared by multiple +concurrent threads. I.e., different threads may write into or fetch +iterators or call Get on the same database without any +external synchronization (the leveldb implementation will +automatically do the required synchronization). However other objects +(like Iterator and WriteBatch) may require external synchronization. +If two threads share such an object, they must protect access to it +using their own locking protocol. More details are available in +the public header files. +

+

Iteration

+

+The following example demonstrates how to print all key,value pairs +in a database. +

+

+  leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions());
+  for (it->SeekToFirst(); it->Valid(); it->Next()) {
+    cout << it->key().ToString() << ": "  << it->value().ToString() << endl;
+  }
+  assert(it->status().ok());  // Check for any errors found during the scan
+  delete it;
+
+The following variation shows how to process just the keys in the +range [start,limit): +

+

+  for (it->Seek(start);
+       it->Valid() && it->key().ToString() < limit;
+       it->Next()) {
+    ...
+  }
+
+You can also process entries in reverse order. (Caveat: reverse +iteration may be somewhat slower than forward iteration.) +

+

+  for (it->SeekToLast(); it->Valid(); it->Prev()) {
+    ...
+  }
+
+

Snapshots

+

+Snapshots provide consistent read-only views over the entire state of +the key-value store. ReadOptions::snapshot may be non-NULL to indicate +that a read should operate on a particular version of the DB state. +If ReadOptions::snapshot is NULL, the read will operate on an +implicit snapshot of the current state. +

+Snapshots are created by the DB::GetSnapshot() method: +

+

+  leveldb::ReadOptions options;
+  options.snapshot = db->GetSnapshot();
+  ... apply some updates to db ...
+  leveldb::Iterator* iter = db->NewIterator(options);
+  ... read using iter to view the state when the snapshot was created ...
+  delete iter;
+  db->ReleaseSnapshot(options.snapshot);
+
+Note that when a snapshot is no longer needed, it should be released +using the DB::ReleaseSnapshot interface. This allows the +implementation to get rid of state that was being maintained just to +support reading as of that snapshot. +

Slice

+

+The return value of the it->key() and it->value() calls above +are instances of the leveldb::Slice type. Slice is a simple +structure that contains a length and a pointer to an external byte +array. Returning a Slice is a cheaper alternative to returning a +std::string since we do not need to copy potentially large keys and +values. In addition, leveldb methods do not return null-terminated +C-style strings since leveldb keys and values are allowed to +contain '\0' bytes. +

+C++ strings and null-terminated C-style strings can be easily converted +to a Slice: +

+

+   leveldb::Slice s1 = "hello";
+
+   std::string str("world");
+   leveldb::Slice s2 = str;
+
+A Slice can be easily converted back to a C++ string: +
+   std::string str = s1.ToString();
+   assert(str == std::string("hello"));
+
+Be careful when using Slices since it is up to the caller to ensure that +the external byte array into which the Slice points remains live while +the Slice is in use. For example, the following is buggy: +

+

+   leveldb::Slice slice;
+   if (...) {
+     std::string str = ...;
+     slice = str;
+   }
+   Use(slice);
+
+When the if statement goes out of scope, str will be destroyed and the +backing storage for slice will disappear. +

+

Comparators

+

+The preceding examples used the default ordering function for key, +which orders bytes lexicographically. You can however supply a custom +comparator when opening a database. For example, suppose each +database key consists of two numbers and we should sort by the first +number, breaking ties by the second number. First, define a proper +subclass of leveldb::Comparator that expresses these rules: +

+

+  class TwoPartComparator : public leveldb::Comparator {
+   public:
+    // Three-way comparison function:
+    //   if a < b: negative result
+    //   if a > b: positive result
+    //   else: zero result
+    int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const {
+      int a1, a2, b1, b2;
+      ParseKey(a, &a1, &a2);
+      ParseKey(b, &b1, &b2);
+      if (a1 < b1) return -1;
+      if (a1 > b1) return +1;
+      if (a2 < b2) return -1;
+      if (a2 > b2) return +1;
+      return 0;
+    }
+
+    // Ignore the following methods for now:
+    const char* Name() const { return "TwoPartComparator"; }
+    void FindShortestSeparator(std::string*, const leveldb::Slice&) const { }
+    void FindShortSuccessor(std::string*) const { }
+  };
+
+Now create a database using this custom comparator: +

+

+  TwoPartComparator cmp;
+  leveldb::DB* db;
+  leveldb::Options options;
+  options.create_if_missing = true;
+  options.comparator = &cmp;
+  leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
+  ...
+
+

Backwards compatibility

+

+The result of the comparator's Name method is attached to the +database when it is created, and is checked on every subsequent +database open. If the name changes, the leveldb::DB::Open call will +fail. Therefore, change the name if and only if the new key format +and comparison function are incompatible with existing databases, and +it is ok to discard the contents of all existing databases. +

+You can however still gradually evolve your key format over time with +a little bit of pre-planning. For example, you could store a version +number at the end of each key (one byte should suffice for most uses). +When you wish to switch to a new key format (e.g., adding an optional +third part to the keys processed by TwoPartComparator), +(a) keep the same comparator name (b) increment the version number +for new keys (c) change the comparator function so it uses the +version numbers found in the keys to decide how to interpret them. +

+

Performance

+

+Performance can be tuned by changing the default values of the +types defined in include/leveldb/options.h. + +

+

Block size

+

+leveldb groups adjacent keys together into the same block and such a +block is the unit of transfer to and from persistent storage. The +default block size is approximately 4096 uncompressed bytes. +Applications that mostly do bulk scans over the contents of the +database may wish to increase this size. Applications that do a lot +of point reads of small values may wish to switch to a smaller block +size if performance measurements indicate an improvement. There isn't +much benefit in using blocks smaller than one kilobyte, or larger than +a few megabytes. Also note that compression will be more effective +with larger block sizes. +

+

Compression

+

+Each block is individually compressed before being written to +persistent storage. Compression is on by default since the default +compression method is very fast, and is automatically disabled for +uncompressible data. In rare cases, applications may want to disable +compression entirely, but should only do so if benchmarks show a +performance improvement: +

+

+  leveldb::Options options;
+  options.compression = leveldb::kNoCompression;
+  ... leveldb::DB::Open(options, name, ...) ....
+
+

Cache

+

+The contents of the database are stored in a set of files in the +filesystem and each file stores a sequence of compressed blocks. If +options.cache is non-NULL, it is used to cache frequently used +uncompressed block contents. +

+

+  #include "leveldb/cache.h"
+
+  leveldb::Options options;
+  options.cache = leveldb::NewLRUCache(100 * 1048576);  // 100MB cache
+  leveldb::DB* db;
+  leveldb::DB::Open(options, name, &db);
+  ... use the db ...
+  delete db
+  delete options.cache;
+
+Note that the cache holds uncompressed data, and therefore it should +be sized according to application level data sizes, without any +reduction from compression. (Caching of compressed blocks is left to +the operating system buffer cache, or any custom Env +implementation provided by the client.) +

+When performing a bulk read, the application may wish to disable +caching so that the data processed by the bulk read does not end up +displacing most of the cached contents. A per-iterator option can be +used to achieve this: +

+

+  leveldb::ReadOptions options;
+  options.fill_cache = false;
+  leveldb::Iterator* it = db->NewIterator(options);
+  for (it->SeekToFirst(); it->Valid(); it->Next()) {
+    ...
+  }
+
+

Key Layout

+

+Note that the unit of disk transfer and caching is a block. Adjacent +keys (according to the database sort order) will usually be placed in +the same block. Therefore the application can improve its performance +by placing keys that are accessed together near each other and placing +infrequently used keys in a separate region of the key space. +

+For example, suppose we are implementing a simple file system on top +of leveldb. The types of entries we might wish to store are: +

+

+   filename -> permission-bits, length, list of file_block_ids
+   file_block_id -> data
+
+We might want to prefix filename keys with one letter (say '/') and the +file_block_id keys with a different letter (say '0') so that scans +over just the metadata do not force us to fetch and cache bulky file +contents. +

+

Filters

+

+Because of the way leveldb data is organized on disk, +a single Get() call may involve multiple reads from disk. +The optional FilterPolicy mechanism can be used to reduce +the number of disk reads substantially. +

+   leveldb::Options options;
+   options.filter_policy = NewBloomFilterPolicy(10);
+   leveldb::DB* db;
+   leveldb::DB::Open(options, "/tmp/testdb", &db);
+   ... use the database ...
+   delete db;
+   delete options.filter_policy;
+
+The preceding code associates a +Bloom filter +based filtering policy with the database. Bloom filter based +filtering relies on keeping some number of bits of data in memory per +key (in this case 10 bits per key since that is the argument we passed +to NewBloomFilterPolicy). This filter will reduce the number of unnecessary +disk reads needed for Get() calls by a factor of +approximately a 100. Increasing the bits per key will lead to a +larger reduction at the cost of more memory usage. We recommend that +applications whose working set does not fit in memory and that do a +lot of random reads set a filter policy. +

+If you are using a custom comparator, you should ensure that the filter +policy you are using is compatible with your comparator. For example, +consider a comparator that ignores trailing spaces when comparing keys. +NewBloomFilterPolicy must not be used with such a comparator. +Instead, the application should provide a custom filter policy that +also ignores trailing spaces. For example: +

+  class CustomFilterPolicy : public leveldb::FilterPolicy {
+   private:
+    FilterPolicy* builtin_policy_;
+   public:
+    CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) { }
+    ~CustomFilterPolicy() { delete builtin_policy_; }
+
+    const char* Name() const { return "IgnoreTrailingSpacesFilter"; }
+
+    void CreateFilter(const Slice* keys, int n, std::string* dst) const {
+      // Use builtin bloom filter code after removing trailing spaces
+      std::vector<Slice> trimmed(n);
+      for (int i = 0; i < n; i++) {
+        trimmed[i] = RemoveTrailingSpaces(keys[i]);
+      }
+      return builtin_policy_->CreateFilter(&trimmed[i], n, dst);
+    }
+
+    bool KeyMayMatch(const Slice& key, const Slice& filter) const {
+      // Use builtin bloom filter code after removing trailing spaces
+      return builtin_policy_->KeyMayMatch(RemoveTrailingSpaces(key), filter);
+    }
+  };
+
+

+Advanced applications may provide a filter policy that does not use +a bloom filter but uses some other mechanism for summarizing a set +of keys. See leveldb/filter_policy.h for detail. +

+

Checksums

+

+leveldb associates checksums with all data it stores in the file system. +There are two separate controls provided over how aggressively these +checksums are verified: +

+

    +
  • ReadOptions::verify_checksums may be set to true to force + checksum verification of all data that is read from the file system on + behalf of a particular read. By default, no such verification is + done. +

    +

  • Options::paranoid_checks may be set to true before opening a + database to make the database implementation raise an error as soon as + it detects an internal corruption. Depending on which portion of the + database has been corrupted, the error may be raised when the database + is opened, or later by another database operation. By default, + paranoid checking is off so that the database can be used even if + parts of its persistent storage have been corrupted. +

    + If a database is corrupted (perhaps it cannot be opened when + paranoid checking is turned on), the leveldb::RepairDB function + may be used to recover as much of the data as possible +

    +

+

Approximate Sizes

+

+The GetApproximateSizes method can used to get the approximate +number of bytes of file system space used by one or more key ranges. +

+

+   leveldb::Range ranges[2];
+   ranges[0] = leveldb::Range("a", "c");
+   ranges[1] = leveldb::Range("x", "z");
+   uint64_t sizes[2];
+   leveldb::Status s = db->GetApproximateSizes(ranges, 2, sizes);
+
+The preceding call will set sizes[0] to the approximate number of +bytes of file system space used by the key range [a..c) and +sizes[1] to the approximate number of bytes used by the key range +[x..z). +

+

Environment

+

+All file operations (and other operating system calls) issued by the +leveldb implementation are routed through a leveldb::Env object. +Sophisticated clients may wish to provide their own Env +implementation to get better control. For example, an application may +introduce artificial delays in the file IO paths to limit the impact +of leveldb on other activities in the system. +

+

+  class SlowEnv : public leveldb::Env {
+    .. implementation of the Env interface ...
+  };
+
+  SlowEnv env;
+  leveldb::Options options;
+  options.env = &env;
+  Status s = leveldb::DB::Open(options, ...);
+
+

Porting

+

+leveldb may be ported to a new platform by providing platform +specific implementations of the types/methods/functions exported by +leveldb/port/port.h. See leveldb/port/port_example.h for more +details. +

+In addition, the new platform may need a new default leveldb::Env +implementation. See leveldb/util/env_posix.h for an example. + +

Other Information

+ +

+Details about the leveldb implementation may be found in +the following documents: +

+ + + diff --git a/Subtrees/hyperleveldb/doc/log_format.txt b/Subtrees/hyperleveldb/doc/log_format.txt new file mode 100644 index 0000000000..5228f624de --- /dev/null +++ b/Subtrees/hyperleveldb/doc/log_format.txt @@ -0,0 +1,75 @@ +The log file contents are a sequence of 32KB blocks. The only +exception is that the tail of the file may contain a partial block. + +Each block consists of a sequence of records: + block := record* trailer? + record := + checksum: uint32 // crc32c of type and data[] ; little-endian + length: uint16 // little-endian + type: uint8 // One of FULL, FIRST, MIDDLE, LAST + data: uint8[length] + +A record never starts within the last six bytes of a block (since it +won't fit). Any leftover bytes here form the trailer, which must +consist entirely of zero bytes and must be skipped by readers. + +Aside: if exactly seven bytes are left in the current block, and a new +non-zero length record is added, the writer must emit a FIRST record +(which contains zero bytes of user data) to fill up the trailing seven +bytes of the block and then emit all of the user data in subsequent +blocks. + +More types may be added in the future. Some Readers may skip record +types they do not understand, others may report that some data was +skipped. + +FULL == 1 +FIRST == 2 +MIDDLE == 3 +LAST == 4 + +The FULL record contains the contents of an entire user record. + +FIRST, MIDDLE, LAST are types used for user records that have been +split into multiple fragments (typically because of block boundaries). +FIRST is the type of the first fragment of a user record, LAST is the +type of the last fragment of a user record, and MID is the type of all +interior fragments of a user record. + +Example: consider a sequence of user records: + A: length 1000 + B: length 97270 + C: length 8000 +A will be stored as a FULL record in the first block. + +B will be split into three fragments: first fragment occupies the rest +of the first block, second fragment occupies the entirety of the +second block, and the third fragment occupies a prefix of the third +block. This will leave six bytes free in the third block, which will +be left empty as the trailer. + +C will be stored as a FULL record in the fourth block. + +=================== + +Some benefits over the recordio format: + +(1) We do not need any heuristics for resyncing - just go to next +block boundary and scan. If there is a corruption, skip to the next +block. As a side-benefit, we do not get confused when part of the +contents of one log file are embedded as a record inside another log +file. + +(2) Splitting at approximate boundaries (e.g., for mapreduce) is +simple: find the next block boundary and skip records until we +hit a FULL or FIRST record. + +(3) We do not need extra buffering for large records. + +Some downsides compared to recordio format: + +(1) No packing of tiny records. This could be fixed by adding a new +record type, so it is a shortcoming of the current implementation, +not necessarily the format. + +(2) No compression. Again, this could be fixed by adding new record types. diff --git a/Subtrees/hyperleveldb/doc/table_format.txt b/Subtrees/hyperleveldb/doc/table_format.txt new file mode 100644 index 0000000000..ca8f9b4460 --- /dev/null +++ b/Subtrees/hyperleveldb/doc/table_format.txt @@ -0,0 +1,104 @@ +File format +=========== + + + [data block 1] + [data block 2] + ... + [data block N] + [meta block 1] + ... + [meta block K] + [metaindex block] + [index block] + [Footer] (fixed size; starts at file_size - sizeof(Footer)) + + +The file contains internal pointers. Each such pointer is called +a BlockHandle and contains the following information: + offset: varint64 + size: varint64 +See https://developers.google.com/protocol-buffers/docs/encoding#varints +for an explanation of varint64 format. + +(1) The sequence of key/value pairs in the file are stored in sorted +order and partitioned into a sequence of data blocks. These blocks +come one after another at the beginning of the file. Each data block +is formatted according to the code in block_builder.cc, and then +optionally compressed. + +(2) After the data blocks we store a bunch of meta blocks. The +supported meta block types are described below. More meta block types +may be added in the future. Each meta block is again formatted using +block_builder.cc and then optionally compressed. + +(3) A "metaindex" block. It contains one entry for every other meta +block where the key is the name of the meta block and the value is a +BlockHandle pointing to that meta block. + +(4) An "index" block. This block contains one entry per data block, +where the key is a string >= last key in that data block and before +the first key in the successive data block. The value is the +BlockHandle for the data block. + +(6) At the very end of the file is a fixed length footer that contains +the BlockHandle of the metaindex and index blocks as well as a magic number. + metaindex_handle: char[p]; // Block handle for metaindex + index_handle: char[q]; // Block handle for index + padding: char[40-p-q]; // zeroed bytes to make fixed length + // (40==2*BlockHandle::kMaxEncodedLength) + magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) + +"filter" Meta Block +------------------- + +If a "FilterPolicy" was specified when the database was opened, a +filter block is stored in each table. The "metaindex" block contains +an entry that maps from "filter." to the BlockHandle for the filter +block where "" is the string returned by the filter policy's +"Name()" method. + +The filter block stores a sequence of filters, where filter i contains +the output of FilterPolicy::CreateFilter() on all keys that are stored +in a block whose file offset falls within the range + + [ i*base ... (i+1)*base-1 ] + +Currently, "base" is 2KB. So for example, if blocks X and Y start in +the range [ 0KB .. 2KB-1 ], all of the keys in X and Y will be +converted to a filter by calling FilterPolicy::CreateFilter(), and the +resulting filter will be stored as the first filter in the filter +block. + +The filter block is formatted as follows: + + [filter 0] + [filter 1] + [filter 2] + ... + [filter N-1] + + [offset of filter 0] : 4 bytes + [offset of filter 1] : 4 bytes + [offset of filter 2] : 4 bytes + ... + [offset of filter N-1] : 4 bytes + + [offset of beginning of offset array] : 4 bytes + lg(base) : 1 byte + +The offset array at the end of the filter block allows efficient +mapping from a data block offset to the corresponding filter. + +"stats" Meta Block +------------------ + +This meta block contains a bunch of stats. The key is the name +of the statistic. The value contains the statistic. +TODO(postrelease): record following stats. + data size + index size + key size (uncompressed) + value size (uncompressed) + number of entries + number of data blocks diff --git a/Subtrees/hyperleveldb/helpers/memenv/memenv.cc b/Subtrees/hyperleveldb/helpers/memenv/memenv.cc new file mode 100644 index 0000000000..48964a8b53 --- /dev/null +++ b/Subtrees/hyperleveldb/helpers/memenv/memenv.cc @@ -0,0 +1,383 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "memenv.h" + +#include "../../hyperleveldb/env.h" +#include "../../hyperleveldb/status.h" +#include "../../port/port.h" +#include "../../util/mutexlock.h" +#include +#include +#include +#include + +namespace hyperleveldb { + +namespace { + +class FileState { + public: + // FileStates are reference counted. The initial reference count is zero + // and the caller must call Ref() at least once. + FileState() : refs_(0), size_(0) {} + + // Increase the reference count. + void Ref() { + MutexLock lock(&refs_mutex_); + ++refs_; + } + + // Decrease the reference count. Delete if this is the last reference. + void Unref() { + bool do_delete = false; + + { + MutexLock lock(&refs_mutex_); + --refs_; + assert(refs_ >= 0); + if (refs_ <= 0) { + do_delete = true; + } + } + + if (do_delete) { + delete this; + } + } + + uint64_t Size() const { return size_; } + + Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { + if (offset > size_) { + return Status::IOError("Offset greater than file size."); + } + const uint64_t available = size_ - offset; + if (n > available) { + n = available; + } + if (n == 0) { + *result = Slice(); + return Status::OK(); + } + + size_t block = offset / kBlockSize; + size_t block_offset = offset % kBlockSize; + + if (n <= kBlockSize - block_offset) { + // The requested bytes are all in the first block. + *result = Slice(blocks_[block] + block_offset, n); + return Status::OK(); + } + + size_t bytes_to_copy = n; + char* dst = scratch; + + while (bytes_to_copy > 0) { + size_t avail = kBlockSize - block_offset; + if (avail > bytes_to_copy) { + avail = bytes_to_copy; + } + memcpy(dst, blocks_[block] + block_offset, avail); + + bytes_to_copy -= avail; + dst += avail; + block++; + block_offset = 0; + } + + *result = Slice(scratch, n); + return Status::OK(); + } + + Status Append(const Slice& data) { + const char* src = data.data(); + size_t src_len = data.size(); + + while (src_len > 0) { + size_t avail; + size_t offset = size_ % kBlockSize; + + if (offset != 0) { + // There is some room in the last block. + avail = kBlockSize - offset; + } else { + // No room in the last block; push new one. + blocks_.push_back(new char[kBlockSize]); + avail = kBlockSize; + } + + if (avail > src_len) { + avail = src_len; + } + memcpy(blocks_.back() + offset, src, avail); + src_len -= avail; + src += avail; + size_ += avail; + } + + return Status::OK(); + } + + private: + // Private since only Unref() should be used to delete it. + ~FileState() { + for (std::vector::iterator i = blocks_.begin(); i != blocks_.end(); + ++i) { + delete [] *i; + } + } + + // No copying allowed. + FileState(const FileState&); + void operator=(const FileState&); + + port::Mutex refs_mutex_; + int refs_; // Protected by refs_mutex_; + + // The following fields are not protected by any mutex. They are only mutable + // while the file is being written, and concurrent access is not allowed + // to writable files. + std::vector blocks_; + uint64_t size_; + + enum { kBlockSize = 8 * 1024 }; +}; + +class SequentialFileImpl : public SequentialFile { + public: + explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) { + file_->Ref(); + } + + ~SequentialFileImpl() { + file_->Unref(); + } + + virtual Status Read(size_t n, Slice* result, char* scratch) { + Status s = file_->Read(pos_, n, result, scratch); + if (s.ok()) { + pos_ += result->size(); + } + return s; + } + + virtual Status Skip(uint64_t n) { + if (pos_ > file_->Size()) { + return Status::IOError("pos_ > file_->Size()"); + } + const size_t available = file_->Size() - pos_; + if (n > available) { + n = available; + } + pos_ += n; + return Status::OK(); + } + + private: + FileState* file_; + size_t pos_; +}; + +class RandomAccessFileImpl : public RandomAccessFile { + public: + explicit RandomAccessFileImpl(FileState* file) : file_(file) { + file_->Ref(); + } + + ~RandomAccessFileImpl() { + file_->Unref(); + } + + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const { + return file_->Read(offset, n, result, scratch); + } + + private: + FileState* file_; +}; + +class WritableFileImpl : public WritableFile { + public: + WritableFileImpl(FileState* file) : file_(file) { + file_->Ref(); + } + + ~WritableFileImpl() { + file_->Unref(); + } + + virtual Status Append(const Slice& data) { + return file_->Append(data); + } + + virtual Status Close() { return Status::OK(); } + virtual Status Sync() { return Status::OK(); } + + private: + FileState* file_; +}; + +class NoOpLogger : public Logger { + public: + virtual void Logv(const char* format, va_list ap) { } +}; + +class InMemoryEnv : public EnvWrapper { + public: + explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { } + + virtual ~InMemoryEnv() { + for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ + i->second->Unref(); + } + } + + // Partial implementation of the Env interface. + virtual Status NewSequentialFile(const std::string& fname, + SequentialFile** result) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + *result = NULL; + return Status::IOError(fname, "File not found"); + } + + *result = new SequentialFileImpl(file_map_[fname]); + return Status::OK(); + } + + virtual Status NewRandomAccessFile(const std::string& fname, + RandomAccessFile** result) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + *result = NULL; + return Status::IOError(fname, "File not found"); + } + + *result = new RandomAccessFileImpl(file_map_[fname]); + return Status::OK(); + } + + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) != file_map_.end()) { + DeleteFileInternal(fname); + } + + FileState* file = new FileState(); + file->Ref(); + file_map_[fname] = file; + + *result = new WritableFileImpl(file); + return Status::OK(); + } + + virtual bool FileExists(const std::string& fname) { + MutexLock lock(&mutex_); + return file_map_.find(fname) != file_map_.end(); + } + + virtual Status GetChildren(const std::string& dir, + std::vector* result) { + MutexLock lock(&mutex_); + result->clear(); + + for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ + const std::string& filename = i->first; + + if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' && + Slice(filename).starts_with(Slice(dir))) { + result->push_back(filename.substr(dir.size() + 1)); + } + } + + return Status::OK(); + } + + void DeleteFileInternal(const std::string& fname) { + if (file_map_.find(fname) == file_map_.end()) { + return; + } + + file_map_[fname]->Unref(); + file_map_.erase(fname); + } + + virtual Status DeleteFile(const std::string& fname) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + return Status::IOError(fname, "File not found"); + } + + DeleteFileInternal(fname); + return Status::OK(); + } + + virtual Status CreateDir(const std::string& dirname) { + return Status::OK(); + } + + virtual Status DeleteDir(const std::string& dirname) { + return Status::OK(); + } + + virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + return Status::IOError(fname, "File not found"); + } + + *file_size = file_map_[fname]->Size(); + return Status::OK(); + } + + virtual Status RenameFile(const std::string& src, + const std::string& target) { + MutexLock lock(&mutex_); + if (file_map_.find(src) == file_map_.end()) { + return Status::IOError(src, "File not found"); + } + + DeleteFileInternal(target); + file_map_[target] = file_map_[src]; + file_map_.erase(src); + return Status::OK(); + } + + virtual Status LockFile(const std::string& fname, FileLock** lock) { + *lock = new FileLock; + return Status::OK(); + } + + virtual Status UnlockFile(FileLock* lock) { + delete lock; + return Status::OK(); + } + + virtual Status GetTestDirectory(std::string* path) { + *path = "/test"; + return Status::OK(); + } + + virtual Status NewLogger(const std::string& fname, Logger** result) { + *result = new NoOpLogger; + return Status::OK(); + } + + private: + // Map from filenames to FileState objects, representing a simple file system. + typedef std::map FileSystem; + port::Mutex mutex_; + FileSystem file_map_; // Protected by mutex_. +}; + +} // namespace + +Env* NewMemEnv(Env* base_env) { + return new InMemoryEnv(base_env); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/helpers/memenv/memenv.h b/Subtrees/hyperleveldb/helpers/memenv/memenv.h new file mode 100644 index 0000000000..8bf898dba3 --- /dev/null +++ b/Subtrees/hyperleveldb/helpers/memenv/memenv.h @@ -0,0 +1,20 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_HELPERS_MEMENV_MEMENV_H_ +#define STORAGE_HYPERLEVELDB_HELPERS_MEMENV_MEMENV_H_ + +namespace hyperleveldb { + +class Env; + +// Returns a new environment that stores its data in memory and delegates +// all non-file-storage tasks to base_env. The caller must delete the result +// when it is no longer needed. +// *base_env must remain live while the result is in use. +Env* NewMemEnv(Env* base_env); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_HELPERS_MEMENV_MEMENV_H_ diff --git a/Subtrees/hyperleveldb/helpers/memenv/memenv_test.cc b/Subtrees/hyperleveldb/helpers/memenv/memenv_test.cc new file mode 100644 index 0000000000..bd71aa7ac0 --- /dev/null +++ b/Subtrees/hyperleveldb/helpers/memenv/memenv_test.cc @@ -0,0 +1,232 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "memenv.h" + +#include "../../db/db_impl.h" +#include "../../hyperleveldb/db.h" +#include "../../hyperleveldb/env.h" +#include "../../util/testharness.h" +#include +#include + +namespace hyperleveldb { + +class MemEnvTest { + public: + Env* env_; + + MemEnvTest() + : env_(NewMemEnv(Env::Default())) { + } + ~MemEnvTest() { + delete env_; + } +}; + +TEST(MemEnvTest, Basics) { + uint64_t file_size; + WritableFile* writable_file; + std::vector children; + + ASSERT_OK(env_->CreateDir("/dir")); + + // Check that the directory is empty. + ASSERT_TRUE(!env_->FileExists("/dir/non_existent")); + ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok()); + ASSERT_OK(env_->GetChildren("/dir", &children)); + ASSERT_EQ(0, children.size()); + + // Create a file. + ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + delete writable_file; + + // Check that the file exists. + ASSERT_TRUE(env_->FileExists("/dir/f")); + ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(0, file_size); + ASSERT_OK(env_->GetChildren("/dir", &children)); + ASSERT_EQ(1, children.size()); + ASSERT_EQ("f", children[0]); + + // Write to the file. + ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_OK(writable_file->Append("abc")); + delete writable_file; + + // Check for expected size. + ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(3, file_size); + + // Check that renaming works. + ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); + ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g")); + ASSERT_TRUE(!env_->FileExists("/dir/f")); + ASSERT_TRUE(env_->FileExists("/dir/g")); + ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); + ASSERT_EQ(3, file_size); + + // Check that opening non-existent file fails. + SequentialFile* seq_file; + RandomAccessFile* rand_file; + ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok()); + ASSERT_TRUE(!seq_file); + ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok()); + ASSERT_TRUE(!rand_file); + + // Check that deleting works. + ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok()); + ASSERT_OK(env_->DeleteFile("/dir/g")); + ASSERT_TRUE(!env_->FileExists("/dir/g")); + ASSERT_OK(env_->GetChildren("/dir", &children)); + ASSERT_EQ(0, children.size()); + ASSERT_OK(env_->DeleteDir("/dir")); +} + +TEST(MemEnvTest, ReadWrite) { + WritableFile* writable_file; + SequentialFile* seq_file; + RandomAccessFile* rand_file; + Slice result; + char scratch[100]; + + ASSERT_OK(env_->CreateDir("/dir")); + + ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_OK(writable_file->Append("hello ")); + ASSERT_OK(writable_file->Append("world")); + delete writable_file; + + // Read sequentially. + ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file)); + ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello". + ASSERT_EQ(0, result.compare("hello")); + ASSERT_OK(seq_file->Skip(1)); + ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world". + ASSERT_EQ(0, result.compare("world")); + ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF. + ASSERT_EQ(0, result.size()); + ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file. + ASSERT_OK(seq_file->Read(1000, &result, scratch)); + ASSERT_EQ(0, result.size()); + delete seq_file; + + // Random reads. + ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file)); + ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world". + ASSERT_EQ(0, result.compare("world")); + ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello". + ASSERT_EQ(0, result.compare("hello")); + ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d". + ASSERT_EQ(0, result.compare("d")); + + // Too high offset. + ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok()); + delete rand_file; +} + +TEST(MemEnvTest, Locks) { + FileLock* lock; + + // These are no-ops, but we test they return success. + ASSERT_OK(env_->LockFile("some file", &lock)); + ASSERT_OK(env_->UnlockFile(lock)); +} + +TEST(MemEnvTest, Misc) { + std::string test_dir; + ASSERT_OK(env_->GetTestDirectory(&test_dir)); + ASSERT_TRUE(!test_dir.empty()); + + WritableFile* writable_file; + ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file)); + + // These are no-ops, but we test they return success. + ASSERT_OK(writable_file->Sync()); + ASSERT_OK(writable_file->Flush()); + ASSERT_OK(writable_file->Close()); + delete writable_file; +} + +TEST(MemEnvTest, LargeWrite) { + const size_t kWriteSize = 300 * 1024; + char* scratch = new char[kWriteSize * 2]; + + std::string write_data; + for (size_t i = 0; i < kWriteSize; ++i) { + write_data.append(1, static_cast(i)); + } + + WritableFile* writable_file; + ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_OK(writable_file->Append("foo")); + ASSERT_OK(writable_file->Append(write_data)); + delete writable_file; + + SequentialFile* seq_file; + Slice result; + ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file)); + ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo". + ASSERT_EQ(0, result.compare("foo")); + + size_t read = 0; + std::string read_data; + while (read < kWriteSize) { + ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch)); + read_data.append(result.data(), result.size()); + read += result.size(); + } + ASSERT_TRUE(write_data == read_data); + delete seq_file; + delete [] scratch; +} + +TEST(MemEnvTest, DBTest) { + Options options; + options.create_if_missing = true; + options.env = env_; + DB* db; + + const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")}; + const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")}; + + ASSERT_OK(DB::Open(options, "/dir/db", &db)); + for (size_t i = 0; i < 3; ++i) { + ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i])); + } + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + Iterator* iterator = db->NewIterator(ReadOptions()); + iterator->SeekToFirst(); + for (size_t i = 0; i < 3; ++i) { + ASSERT_TRUE(iterator->Valid()); + ASSERT_TRUE(keys[i] == iterator->key()); + ASSERT_TRUE(vals[i] == iterator->value()); + iterator->Next(); + } + ASSERT_TRUE(!iterator->Valid()); + delete iterator; + + DBImpl* dbi = reinterpret_cast(db); + ASSERT_OK(dbi->TEST_CompactMemTable()); + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + delete db; +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/hyperleveldb/c.h b/Subtrees/hyperleveldb/hyperleveldb/c.h new file mode 100644 index 0000000000..50675feafa --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/c.h @@ -0,0 +1,291 @@ +/* Copyright (c) 2011 The LevelDB Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. See the AUTHORS file for names of contributors. + + C bindings for leveldb. May be useful as a stable ABI that can be + used by programs that keep leveldb in a shared library, or for + a JNI api. + + Does not support: + . getters for the option types + . custom comparators that implement key shortening + . capturing post-write-snapshot + . custom iter, db, env, cache implementations using just the C bindings + + Some conventions: + + (1) We expose just opaque struct pointers and functions to clients. + This allows us to change internal representations without having to + recompile clients. + + (2) For simplicity, there is no equivalent to the Slice type. Instead, + the caller has to pass the pointer and length as separate + arguments. + + (3) Errors are represented by a null-terminated c string. NULL + means no error. All operations that can raise an error are passed + a "char** errptr" as the last argument. One of the following must + be true on entry: + *errptr == NULL + *errptr points to a malloc()ed null-terminated error message + (On Windows, *errptr must have been malloc()-ed by this library.) + On success, a leveldb routine leaves *errptr unchanged. + On failure, leveldb frees the old value of *errptr and + set *errptr to a malloc()ed error message. + + (4) Bools have the type unsigned char (0 == false; rest == true) + + (5) All of the pointer arguments must be non-NULL. +*/ + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_C_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_C_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* Exported types */ + +typedef struct leveldb_t leveldb_t; +typedef struct leveldb_cache_t leveldb_cache_t; +typedef struct leveldb_comparator_t leveldb_comparator_t; +typedef struct leveldb_env_t leveldb_env_t; +typedef struct leveldb_filelock_t leveldb_filelock_t; +typedef struct leveldb_filterpolicy_t leveldb_filterpolicy_t; +typedef struct leveldb_iterator_t leveldb_iterator_t; +typedef struct leveldb_logger_t leveldb_logger_t; +typedef struct leveldb_options_t leveldb_options_t; +typedef struct leveldb_randomfile_t leveldb_randomfile_t; +typedef struct leveldb_readoptions_t leveldb_readoptions_t; +typedef struct leveldb_seqfile_t leveldb_seqfile_t; +typedef struct leveldb_snapshot_t leveldb_snapshot_t; +typedef struct leveldb_writablefile_t leveldb_writablefile_t; +typedef struct leveldb_writebatch_t leveldb_writebatch_t; +typedef struct leveldb_writeoptions_t leveldb_writeoptions_t; + +/* DB operations */ + +extern leveldb_t* leveldb_open( + const leveldb_options_t* options, + const char* name, + char** errptr); + +extern void leveldb_close(leveldb_t* db); + +extern void leveldb_put( + leveldb_t* db, + const leveldb_writeoptions_t* options, + const char* key, size_t keylen, + const char* val, size_t vallen, + char** errptr); + +extern void leveldb_delete( + leveldb_t* db, + const leveldb_writeoptions_t* options, + const char* key, size_t keylen, + char** errptr); + +extern void leveldb_write( + leveldb_t* db, + const leveldb_writeoptions_t* options, + leveldb_writebatch_t* batch, + char** errptr); + +/* Returns NULL if not found. A malloc()ed array otherwise. + Stores the length of the array in *vallen. */ +extern char* leveldb_get( + leveldb_t* db, + const leveldb_readoptions_t* options, + const char* key, size_t keylen, + size_t* vallen, + char** errptr); + +extern leveldb_iterator_t* leveldb_create_iterator( + leveldb_t* db, + const leveldb_readoptions_t* options); + +extern const leveldb_snapshot_t* leveldb_create_snapshot( + leveldb_t* db); + +extern void leveldb_release_snapshot( + leveldb_t* db, + const leveldb_snapshot_t* snapshot); + +/* Returns NULL if property name is unknown. + Else returns a pointer to a malloc()-ed null-terminated value. */ +extern char* leveldb_property_value( + leveldb_t* db, + const char* propname); + +extern void leveldb_approximate_sizes( + leveldb_t* db, + int num_ranges, + const char* const* range_start_key, const size_t* range_start_key_len, + const char* const* range_limit_key, const size_t* range_limit_key_len, + uint64_t* sizes); + +extern void leveldb_compact_range( + leveldb_t* db, + const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len); + +/* Management operations */ + +extern void leveldb_destroy_db( + const leveldb_options_t* options, + const char* name, + char** errptr); + +extern void leveldb_repair_db( + const leveldb_options_t* options, + const char* name, + char** errptr); + +/* Iterator */ + +extern void leveldb_iter_destroy(leveldb_iterator_t*); +extern unsigned char leveldb_iter_valid(const leveldb_iterator_t*); +extern void leveldb_iter_seek_to_first(leveldb_iterator_t*); +extern void leveldb_iter_seek_to_last(leveldb_iterator_t*); +extern void leveldb_iter_seek(leveldb_iterator_t*, const char* k, size_t klen); +extern void leveldb_iter_next(leveldb_iterator_t*); +extern void leveldb_iter_prev(leveldb_iterator_t*); +extern const char* leveldb_iter_key(const leveldb_iterator_t*, size_t* klen); +extern const char* leveldb_iter_value(const leveldb_iterator_t*, size_t* vlen); +extern void leveldb_iter_get_error(const leveldb_iterator_t*, char** errptr); + +/* Write batch */ + +extern leveldb_writebatch_t* leveldb_writebatch_create(); +extern void leveldb_writebatch_destroy(leveldb_writebatch_t*); +extern void leveldb_writebatch_clear(leveldb_writebatch_t*); +extern void leveldb_writebatch_put( + leveldb_writebatch_t*, + const char* key, size_t klen, + const char* val, size_t vlen); +extern void leveldb_writebatch_delete( + leveldb_writebatch_t*, + const char* key, size_t klen); +extern void leveldb_writebatch_iterate( + leveldb_writebatch_t*, + void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); + +/* Options */ + +extern leveldb_options_t* leveldb_options_create(); +extern void leveldb_options_destroy(leveldb_options_t*); +extern void leveldb_options_set_comparator( + leveldb_options_t*, + leveldb_comparator_t*); +extern void leveldb_options_set_filter_policy( + leveldb_options_t*, + leveldb_filterpolicy_t*); +extern void leveldb_options_set_create_if_missing( + leveldb_options_t*, unsigned char); +extern void leveldb_options_set_error_if_exists( + leveldb_options_t*, unsigned char); +extern void leveldb_options_set_paranoid_checks( + leveldb_options_t*, unsigned char); +extern void leveldb_options_set_env(leveldb_options_t*, leveldb_env_t*); +extern void leveldb_options_set_info_log(leveldb_options_t*, leveldb_logger_t*); +extern void leveldb_options_set_write_buffer_size(leveldb_options_t*, size_t); +extern void leveldb_options_set_max_open_files(leveldb_options_t*, int); +extern void leveldb_options_set_cache(leveldb_options_t*, leveldb_cache_t*); +extern void leveldb_options_set_block_size(leveldb_options_t*, size_t); +extern void leveldb_options_set_block_restart_interval(leveldb_options_t*, int); + +enum { + leveldb_no_compression = 0, + leveldb_snappy_compression = 1 +}; +extern void leveldb_options_set_compression(leveldb_options_t*, int); + +/* Comparator */ + +extern leveldb_comparator_t* leveldb_comparator_create( + void* state, + void (*destructor)(void*), + int (*compare)( + void*, + const char* a, size_t alen, + const char* b, size_t blen), + const char* (*name)(void*)); +extern void leveldb_comparator_destroy(leveldb_comparator_t*); + +/* Filter policy */ + +extern leveldb_filterpolicy_t* leveldb_filterpolicy_create( + void* state, + void (*destructor)(void*), + char* (*create_filter)( + void*, + const char* const* key_array, const size_t* key_length_array, + int num_keys, + size_t* filter_length), + unsigned char (*key_may_match)( + void*, + const char* key, size_t length, + const char* filter, size_t filter_length), + const char* (*name)(void*)); +extern void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t*); + +extern leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom( + int bits_per_key); + +/* Read options */ + +extern leveldb_readoptions_t* leveldb_readoptions_create(); +extern void leveldb_readoptions_destroy(leveldb_readoptions_t*); +extern void leveldb_readoptions_set_verify_checksums( + leveldb_readoptions_t*, + unsigned char); +extern void leveldb_readoptions_set_fill_cache( + leveldb_readoptions_t*, unsigned char); +extern void leveldb_readoptions_set_snapshot( + leveldb_readoptions_t*, + const leveldb_snapshot_t*); + +/* Write options */ + +extern leveldb_writeoptions_t* leveldb_writeoptions_create(); +extern void leveldb_writeoptions_destroy(leveldb_writeoptions_t*); +extern void leveldb_writeoptions_set_sync( + leveldb_writeoptions_t*, unsigned char); + +/* Cache */ + +extern leveldb_cache_t* leveldb_cache_create_lru(size_t capacity); +extern void leveldb_cache_destroy(leveldb_cache_t* cache); + +/* Env */ + +extern leveldb_env_t* leveldb_create_default_env(); +extern void leveldb_env_destroy(leveldb_env_t*); + +/* Utility */ + +/* Calls free(ptr). + REQUIRES: ptr was malloc()-ed and returned by one of the routines + in this file. Note that in certain cases (typically on Windows), you + may need to call this routine instead of free(ptr) to dispose of + malloc()-ed memory returned by this library. */ +extern void leveldb_free(void* ptr); + +/* Return the major version number for this release. */ +extern int leveldb_major_version(); + +/* Return the minor version number for this release. */ +extern int leveldb_minor_version(); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif + +#endif /* STORAGE_HYPERLEVELDB_INCLUDE_C_H_ */ diff --git a/Subtrees/hyperleveldb/hyperleveldb/cache.h b/Subtrees/hyperleveldb/hyperleveldb/cache.h new file mode 100644 index 0000000000..f97fc4de2a --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/cache.h @@ -0,0 +1,99 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A Cache is an interface that maps keys to values. It has internal +// synchronization and may be safely accessed concurrently from +// multiple threads. It may automatically evict entries to make room +// for new entries. Values have a specified charge against the cache +// capacity. For example, a cache where the values are variable +// length strings, may use the length of the string as the charge for +// the string. +// +// A builtin cache implementation with a least-recently-used eviction +// policy is provided. Clients may use their own implementations if +// they want something more sophisticated (like scan-resistance, a +// custom eviction policy, variable cache sizing, etc.) + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_CACHE_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_CACHE_H_ + +#include +#include "slice.h" + +namespace hyperleveldb { + +class Cache; + +// Create a new cache with a fixed size capacity. This implementation +// of Cache uses a least-recently-used eviction policy. +extern Cache* NewLRUCache(size_t capacity); + +class Cache { + public: + Cache() { } + + // Destroys all existing entries by calling the "deleter" + // function that was passed to the constructor. + virtual ~Cache(); + + // Opaque handle to an entry stored in the cache. + struct Handle { }; + + // Insert a mapping from key->value into the cache and assign it + // the specified charge against the total cache capacity. + // + // Returns a handle that corresponds to the mapping. The caller + // must call this->Release(handle) when the returned mapping is no + // longer needed. + // + // When the inserted entry is no longer needed, the key and + // value will be passed to "deleter". + virtual Handle* Insert(const Slice& key, void* value, size_t charge, + void (*deleter)(const Slice& key, void* value)) = 0; + + // If the cache has no mapping for "key", returns NULL. + // + // Else return a handle that corresponds to the mapping. The caller + // must call this->Release(handle) when the returned mapping is no + // longer needed. + virtual Handle* Lookup(const Slice& key) = 0; + + // Release a mapping returned by a previous Lookup(). + // REQUIRES: handle must not have been released yet. + // REQUIRES: handle must have been returned by a method on *this. + virtual void Release(Handle* handle) = 0; + + // Return the value encapsulated in a handle returned by a + // successful Lookup(). + // REQUIRES: handle must not have been released yet. + // REQUIRES: handle must have been returned by a method on *this. + virtual void* Value(Handle* handle) = 0; + + // If the cache contains entry for key, erase it. Note that the + // underlying entry will be kept around until all existing handles + // to it have been released. + virtual void Erase(const Slice& key) = 0; + + // Return a new numeric id. May be used by multiple clients who are + // sharing the same cache to partition the key space. Typically the + // client will allocate a new id at startup and prepend the id to + // its cache keys. + virtual uint64_t NewId() = 0; + + private: + void LRU_Remove(Handle* e); + void LRU_Append(Handle* e); + void Unref(Handle* e); + + struct Rep; + Rep* rep_; + + // No copying allowed + Cache(const Cache&); + void operator=(const Cache&); +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_CACHE_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/comparator.h b/Subtrees/hyperleveldb/hyperleveldb/comparator.h new file mode 100644 index 0000000000..5c056aee12 --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/comparator.h @@ -0,0 +1,63 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_COMPARATOR_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_COMPARATOR_H_ + +#include + +namespace hyperleveldb { + +class Slice; + +// A Comparator object provides a total order across slices that are +// used as keys in an sstable or a database. A Comparator implementation +// must be thread-safe since leveldb may invoke its methods concurrently +// from multiple threads. +class Comparator { + public: + virtual ~Comparator(); + + // Three-way comparison. Returns value: + // < 0 iff "a" < "b", + // == 0 iff "a" == "b", + // > 0 iff "a" > "b" + virtual int Compare(const Slice& a, const Slice& b) const = 0; + + // The name of the comparator. Used to check for comparator + // mismatches (i.e., a DB created with one comparator is + // accessed using a different comparator. + // + // The client of this package should switch to a new name whenever + // the comparator implementation changes in a way that will cause + // the relative ordering of any two keys to change. + // + // Names starting with "leveldb." are reserved and should not be used + // by any clients of this package. + virtual const char* Name() const = 0; + + // Advanced functions: these are used to reduce the space requirements + // for internal data structures like index blocks. + + // If *start < limit, changes *start to a short string in [start,limit). + // Simple comparator implementations may return with *start unchanged, + // i.e., an implementation of this method that does nothing is correct. + virtual void FindShortestSeparator( + std::string* start, + const Slice& limit) const = 0; + + // Changes *key to a short string >= *key. + // Simple comparator implementations may return with *key unchanged, + // i.e., an implementation of this method that does nothing is correct. + virtual void FindShortSuccessor(std::string* key) const = 0; +}; + +// Return a builtin comparator that uses lexicographic byte-wise +// ordering. The result remains the property of this module and +// must not be deleted. +extern const Comparator* BytewiseComparator(); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_COMPARATOR_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/db.h b/Subtrees/hyperleveldb/hyperleveldb/db.h new file mode 100644 index 0000000000..5f573ae5ef --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/db.h @@ -0,0 +1,161 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_DB_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_DB_H_ + +#include +#include +#include "iterator.h" +#include "options.h" + +namespace hyperleveldb { + +// Update Makefile if you change these +static const int kMajorVersion = 1; +static const int kMinorVersion = 11; + +struct Options; +struct ReadOptions; +struct WriteOptions; +class WriteBatch; + +// Abstract handle to particular state of a DB. +// A Snapshot is an immutable object and can therefore be safely +// accessed from multiple threads without any external synchronization. +class Snapshot { + protected: + virtual ~Snapshot(); +}; + +// A range of keys +struct Range { + Slice start; // Included in the range + Slice limit; // Not included in the range + + Range() { } + Range(const Slice& s, const Slice& l) : start(s), limit(l) { } +}; + +// A DB is a persistent ordered map from keys to values. +// A DB is safe for concurrent access from multiple threads without +// any external synchronization. +class DB { + public: + // Open the database with the specified "name". + // Stores a pointer to a heap-allocated database in *dbptr and returns + // OK on success. + // Stores NULL in *dbptr and returns a non-OK status on error. + // Caller should delete *dbptr when it is no longer needed. + static Status Open(const Options& options, + const std::string& name, + DB** dbptr); + + DB() { } + virtual ~DB(); + + // Set the database entry for "key" to "value". Returns OK on success, + // and a non-OK status on error. + // Note: consider setting options.sync = true. + virtual Status Put(const WriteOptions& options, + const Slice& key, + const Slice& value) = 0; + + // Remove the database entry (if any) for "key". Returns OK on + // success, and a non-OK status on error. It is not an error if "key" + // did not exist in the database. + // Note: consider setting options.sync = true. + virtual Status Delete(const WriteOptions& options, const Slice& key) = 0; + + // Apply the specified updates to the database. + // Returns OK on success, non-OK on failure. + // Note: consider setting options.sync = true. + virtual Status Write(const WriteOptions& options, WriteBatch* updates) = 0; + + // If the database contains an entry for "key" store the + // corresponding value in *value and return OK. + // + // If there is no entry for "key" leave *value unchanged and return + // a status for which Status::IsNotFound() returns true. + // + // May return some other Status on an error. + virtual Status Get(const ReadOptions& options, + const Slice& key, std::string* value) = 0; + + // Return a heap-allocated iterator over the contents of the database. + // The result of NewIterator() is initially invalid (caller must + // call one of the Seek methods on the iterator before using it). + // + // Caller should delete the iterator when it is no longer needed. + // The returned iterator should be deleted before this db is deleted. + virtual Iterator* NewIterator(const ReadOptions& options) = 0; + + // Return a handle to the current DB state. Iterators created with + // this handle will all observe a stable snapshot of the current DB + // state. The caller must call ReleaseSnapshot(result) when the + // snapshot is no longer needed. + virtual const Snapshot* GetSnapshot() = 0; + + // Release a previously acquired snapshot. The caller must not + // use "snapshot" after this call. + virtual void ReleaseSnapshot(const Snapshot* snapshot) = 0; + + // DB implementations can export properties about their state + // via this method. If "property" is a valid property understood by this + // DB implementation, fills "*value" with its current value and returns + // true. Otherwise returns false. + // + // + // Valid property names include: + // + // "leveldb.num-files-at-level" - return the number of files at level , + // where is an ASCII representation of a level number (e.g. "0"). + // "leveldb.stats" - returns a multi-line string that describes statistics + // about the internal operation of the DB. + // "leveldb.sstables" - returns a multi-line string that describes all + // of the sstables that make up the db contents. + virtual bool GetProperty(const Slice& property, std::string* value) = 0; + + // For each i in [0,n-1], store in "sizes[i]", the approximate + // file system space used by keys in "[range[i].start .. range[i].limit)". + // + // Note that the returned sizes measure file system space usage, so + // if the user data compresses by a factor of ten, the returned + // sizes will be one-tenth the size of the corresponding user data size. + // + // The results may not include the sizes of recently written data. + virtual void GetApproximateSizes(const Range* range, int n, + uint64_t* sizes) = 0; + + // Compact the underlying storage for the key range [*begin,*end]. + // In particular, deleted and overwritten versions are discarded, + // and the data is rearranged to reduce the cost of operations + // needed to access the data. This operation should typically only + // be invoked by users who understand the underlying implementation. + // + // begin==NULL is treated as a key before all keys in the database. + // end==NULL is treated as a key after all keys in the database. + // Therefore the following call will compact the entire database: + // db->CompactRange(NULL, NULL); + virtual void CompactRange(const Slice* begin, const Slice* end) = 0; + + private: + // No copying allowed + DB(const DB&); + void operator=(const DB&); +}; + +// Destroy the contents of the specified database. +// Be very careful using this method. +Status DestroyDB(const std::string& name, const Options& options); + +// If a DB cannot be opened, you may attempt to call this method to +// resurrect as much of the contents of the database as possible. +// Some data may be lost, so be careful when calling this function +// on a database that contains important information. +Status RepairDB(const std::string& dbname, const Options& options); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_DB_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/env.h b/Subtrees/hyperleveldb/hyperleveldb/env.h new file mode 100644 index 0000000000..7ff2aacb75 --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/env.h @@ -0,0 +1,337 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// An Env is an interface used by the leveldb implementation to access +// operating system functionality like the filesystem etc. Callers +// may wish to provide a custom Env object when opening a database to +// get fine gain control; e.g., to rate limit file system operations. +// +// All Env implementations are safe for concurrent access from +// multiple threads without any external synchronization. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_ENV_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_ENV_H_ + +#include +#include +#include +#include +#include "status.h" + +namespace hyperleveldb { + +class FileLock; +class Logger; +class RandomAccessFile; +class SequentialFile; +class Slice; +class WritableFile; + +class Env { + public: + Env() { } + virtual ~Env(); + + // Return a default environment suitable for the current operating + // system. Sophisticated users may wish to provide their own Env + // implementation instead of relying on this default environment. + // + // The result of Default() belongs to leveldb and must never be deleted. + static Env* Default(); + + // Create a brand new sequentially-readable file with the specified name. + // On success, stores a pointer to the new file in *result and returns OK. + // On failure stores NULL in *result and returns non-OK. If the file does + // not exist, returns a non-OK status. + // + // The returned file will only be accessed by one thread at a time. + virtual Status NewSequentialFile(const std::string& fname, + SequentialFile** result) = 0; + + // Create a brand new random access read-only file with the + // specified name. On success, stores a pointer to the new file in + // *result and returns OK. On failure stores NULL in *result and + // returns non-OK. If the file does not exist, returns a non-OK + // status. + // + // The returned file may be concurrently accessed by multiple threads. + virtual Status NewRandomAccessFile(const std::string& fname, + RandomAccessFile** result) = 0; + + // Create an object that writes to a new file with the specified + // name. Deletes any existing file with the same name and creates a + // new file. On success, stores a pointer to the new file in + // *result and returns OK. On failure stores NULL in *result and + // returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result) = 0; + + // Returns true iff the named file exists. + virtual bool FileExists(const std::string& fname) = 0; + + // Store in *result the names of the children of the specified directory. + // The names are relative to "dir". + // Original contents of *results are dropped. + virtual Status GetChildren(const std::string& dir, + std::vector* result) = 0; + + // Delete the named file. + virtual Status DeleteFile(const std::string& fname) = 0; + + // Create the specified directory. + virtual Status CreateDir(const std::string& dirname) = 0; + + // Delete the specified directory. + virtual Status DeleteDir(const std::string& dirname) = 0; + + // Store the size of fname in *file_size. + virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) = 0; + + // Rename file src to target. + virtual Status RenameFile(const std::string& src, + const std::string& target) = 0; + + // Lock the specified file. Used to prevent concurrent access to + // the same db by multiple processes. On failure, stores NULL in + // *lock and returns non-OK. + // + // On success, stores a pointer to the object that represents the + // acquired lock in *lock and returns OK. The caller should call + // UnlockFile(*lock) to release the lock. If the process exits, + // the lock will be automatically released. + // + // If somebody else already holds the lock, finishes immediately + // with a failure. I.e., this call does not wait for existing locks + // to go away. + // + // May create the named file if it does not already exist. + virtual Status LockFile(const std::string& fname, FileLock** lock) = 0; + + // Release the lock acquired by a previous successful call to LockFile. + // REQUIRES: lock was returned by a successful LockFile() call + // REQUIRES: lock has not already been unlocked. + virtual Status UnlockFile(FileLock* lock) = 0; + + // Arrange to run "(*function)(arg)" once in a background thread. + // + // "function" may run in an unspecified thread. Multiple functions + // added to the same Env may run concurrently in different threads. + // I.e., the caller may not assume that background work items are + // serialized. + virtual void Schedule( + void (*function)(void* arg), + void* arg) = 0; + + // Start a new thread, invoking "function(arg)" within the new thread. + // When "function(arg)" returns, the thread will be destroyed. + virtual void StartThread(void (*function)(void* arg), void* arg) = 0; + + // *path is set to a temporary directory that can be used for testing. It may + // or many not have just been created. The directory may or may not differ + // between runs of the same process, but subsequent calls will return the + // same directory. + virtual Status GetTestDirectory(std::string* path) = 0; + + // Create and return a log file for storing informational messages. + virtual Status NewLogger(const std::string& fname, Logger** result) = 0; + + // Returns the number of micro-seconds since some fixed point in time. Only + // useful for computing deltas of time. + virtual uint64_t NowMicros() = 0; + + // Sleep/delay the thread for the perscribed number of micro-seconds. + virtual void SleepForMicroseconds(int micros) = 0; + + private: + // No copying allowed + Env(const Env&); + void operator=(const Env&); +}; + +// A file abstraction for reading sequentially through a file +class SequentialFile { + public: + SequentialFile() { } + virtual ~SequentialFile(); + + // Read up to "n" bytes from the file. "scratch[0..n-1]" may be + // written by this routine. Sets "*result" to the data that was + // read (including if fewer than "n" bytes were successfully read). + // May set "*result" to point at data in "scratch[0..n-1]", so + // "scratch[0..n-1]" must be live when "*result" is used. + // If an error was encountered, returns a non-OK status. + // + // REQUIRES: External synchronization + virtual Status Read(size_t n, Slice* result, char* scratch) = 0; + + // Skip "n" bytes from the file. This is guaranteed to be no + // slower that reading the same data, but may be faster. + // + // If end of file is reached, skipping will stop at the end of the + // file, and Skip will return OK. + // + // REQUIRES: External synchronization + virtual Status Skip(uint64_t n) = 0; + + private: + // No copying allowed + SequentialFile(const SequentialFile&); + void operator=(const SequentialFile&); +}; + +// A file abstraction for randomly reading the contents of a file. +class RandomAccessFile { + public: + RandomAccessFile() { } + virtual ~RandomAccessFile(); + + // Read up to "n" bytes from the file starting at "offset". + // "scratch[0..n-1]" may be written by this routine. Sets "*result" + // to the data that was read (including if fewer than "n" bytes were + // successfully read). May set "*result" to point at data in + // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when + // "*result" is used. If an error was encountered, returns a non-OK + // status. + // + // Safe for concurrent use by multiple threads. + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const = 0; + + private: + // No copying allowed + RandomAccessFile(const RandomAccessFile&); + void operator=(const RandomAccessFile&); +}; + +// A file abstraction for sequential writing. The implementation +// must provide buffering since callers may append small fragments +// at a time to the file. +class WritableFile { + public: + WritableFile() { } + virtual ~WritableFile(); + + // Allows concurrent writers + // REQUIRES: The range of data falling in [offset, offset + data.size()) must + // only be written once. + virtual Status WriteAt(uint64_t offset, const Slice& data) = 0; + // REQUIRES: external synchronization + virtual Status Append(const Slice& data) = 0; + virtual Status Close() = 0; + virtual Status Sync() = 0; + + private: + // No copying allowed + WritableFile(const WritableFile&); + void operator=(const WritableFile&); +}; + +// An interface for writing log messages. +class Logger { + public: + Logger() { } + virtual ~Logger(); + + // Write an entry to the log file with the specified format. + virtual void Logv(const char* format, va_list ap) = 0; + + private: + // No copying allowed + Logger(const Logger&); + void operator=(const Logger&); +}; + + +// Identifies a locked file. +class FileLock { + public: + FileLock() { } + virtual ~FileLock(); + private: + // No copying allowed + FileLock(const FileLock&); + void operator=(const FileLock&); +}; + +// Log the specified data to *info_log if info_log is non-NULL. +extern void Log(Logger* info_log, const char* format, ...) +# if defined(__GNUC__) || defined(__clang__) + __attribute__((__format__ (__printf__, 2, 3))) +# endif + ; + +// A utility routine: write "data" to the named file. +extern Status WriteStringToFile(Env* env, const Slice& data, + const std::string& fname); + +// A utility routine: read contents of named file into *data +extern Status ReadFileToString(Env* env, const std::string& fname, + std::string* data); + +// An implementation of Env that forwards all calls to another Env. +// May be useful to clients who wish to override just part of the +// functionality of another Env. +class EnvWrapper : public Env { + public: + // Initialize an EnvWrapper that delegates all calls to *t + explicit EnvWrapper(Env* t) : target_(t) { } + virtual ~EnvWrapper(); + + // Return the target to which this Env forwards all calls + Env* target() const { return target_; } + + // The following text is boilerplate that forwards all methods to target() + Status NewSequentialFile(const std::string& f, SequentialFile** r) { + return target_->NewSequentialFile(f, r); + } + Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) { + return target_->NewRandomAccessFile(f, r); + } + Status NewWritableFile(const std::string& f, WritableFile** r) { + return target_->NewWritableFile(f, r); + } + bool FileExists(const std::string& f) { return target_->FileExists(f); } + Status GetChildren(const std::string& dir, std::vector* r) { + return target_->GetChildren(dir, r); + } + Status DeleteFile(const std::string& f) { return target_->DeleteFile(f); } + Status CreateDir(const std::string& d) { return target_->CreateDir(d); } + Status DeleteDir(const std::string& d) { return target_->DeleteDir(d); } + Status GetFileSize(const std::string& f, uint64_t* s) { + return target_->GetFileSize(f, s); + } + Status RenameFile(const std::string& s, const std::string& t) { + return target_->RenameFile(s, t); + } + Status LockFile(const std::string& f, FileLock** l) { + return target_->LockFile(f, l); + } + Status UnlockFile(FileLock* l) { return target_->UnlockFile(l); } + void Schedule(void (*f)(void*), void* a) { + return target_->Schedule(f, a); + } + void StartThread(void (*f)(void*), void* a) { + return target_->StartThread(f, a); + } + virtual Status GetTestDirectory(std::string* path) { + return target_->GetTestDirectory(path); + } + virtual Status NewLogger(const std::string& fname, Logger** result) { + return target_->NewLogger(fname, result); + } + uint64_t NowMicros() { + return target_->NowMicros(); + } + void SleepForMicroseconds(int micros) { + target_->SleepForMicroseconds(micros); + } + private: + Env* target_; +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_ENV_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/filter_policy.h b/Subtrees/hyperleveldb/hyperleveldb/filter_policy.h new file mode 100644 index 0000000000..6707922816 --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/filter_policy.h @@ -0,0 +1,70 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A database can be configured with a custom FilterPolicy object. +// This object is responsible for creating a small filter from a set +// of keys. These filters are stored in leveldb and are consulted +// automatically by leveldb to decide whether or not to read some +// information from disk. In many cases, a filter can cut down the +// number of disk seeks form a handful to a single disk seek per +// DB::Get() call. +// +// Most people will want to use the builtin bloom filter support (see +// NewBloomFilterPolicy() below). + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_FILTER_POLICY_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_FILTER_POLICY_H_ + +#include + +namespace hyperleveldb { + +class Slice; + +class FilterPolicy { + public: + virtual ~FilterPolicy(); + + // Return the name of this policy. Note that if the filter encoding + // changes in an incompatible way, the name returned by this method + // must be changed. Otherwise, old incompatible filters may be + // passed to methods of this type. + virtual const char* Name() const = 0; + + // keys[0,n-1] contains a list of keys (potentially with duplicates) + // that are ordered according to the user supplied comparator. + // Append a filter that summarizes keys[0,n-1] to *dst. + // + // Warning: do not change the initial contents of *dst. Instead, + // append the newly constructed filter to *dst. + virtual void CreateFilter(const Slice* keys, int n, std::string* dst) + const = 0; + + // "filter" contains the data appended by a preceding call to + // CreateFilter() on this class. This method must return true if + // the key was in the list of keys passed to CreateFilter(). + // This method may return true or false if the key was not on the + // list, but it should aim to return false with a high probability. + virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const = 0; +}; + +// Return a new filter policy that uses a bloom filter with approximately +// the specified number of bits per key. A good value for bits_per_key +// is 10, which yields a filter with ~ 1% false positive rate. +// +// Callers must delete the result after any database that is using the +// result has been closed. +// +// Note: if you are using a custom comparator that ignores some parts +// of the keys being compared, you must not use NewBloomFilterPolicy() +// and must provide your own FilterPolicy that also ignores the +// corresponding parts of the keys. For example, if the comparator +// ignores trailing spaces, it would be incorrect to use a +// FilterPolicy (like NewBloomFilterPolicy) that does not ignore +// trailing spaces in keys. +extern const FilterPolicy* NewBloomFilterPolicy(int bits_per_key); + +} + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_FILTER_POLICY_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/iterator.h b/Subtrees/hyperleveldb/hyperleveldb/iterator.h new file mode 100644 index 0000000000..c4d982bf9c --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/iterator.h @@ -0,0 +1,100 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// An iterator yields a sequence of key/value pairs from a source. +// The following class defines the interface. Multiple implementations +// are provided by this library. In particular, iterators are provided +// to access the contents of a Table or a DB. +// +// Multiple threads can invoke const methods on an Iterator without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Iterator must use +// external synchronization. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_ITERATOR_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_ITERATOR_H_ + +#include "slice.h" +#include "status.h" + +namespace hyperleveldb { + +class Iterator { + public: + Iterator(); + virtual ~Iterator(); + + // An iterator is either positioned at a key/value pair, or + // not valid. This method returns true iff the iterator is valid. + virtual bool Valid() const = 0; + + // Position at the first key in the source. The iterator is Valid() + // after this call iff the source is not empty. + virtual void SeekToFirst() = 0; + + // Position at the last key in the source. The iterator is + // Valid() after this call iff the source is not empty. + virtual void SeekToLast() = 0; + + // Position at the first key in the source that at or past target + // The iterator is Valid() after this call iff the source contains + // an entry that comes at or past target. + virtual void Seek(const Slice& target) = 0; + + // Moves to the next entry in the source. After this call, Valid() is + // true iff the iterator was not positioned at the last entry in the source. + // REQUIRES: Valid() + virtual void Next() = 0; + + // Moves to the previous entry in the source. After this call, Valid() is + // true iff the iterator was not positioned at the first entry in source. + // REQUIRES: Valid() + virtual void Prev() = 0; + + // Return the key for the current entry. The underlying storage for + // the returned slice is valid only until the next modification of + // the iterator. + // REQUIRES: Valid() + virtual Slice key() const = 0; + + // Return the value for the current entry. The underlying storage for + // the returned slice is valid only until the next modification of + // the iterator. + // REQUIRES: !AtEnd() && !AtStart() + virtual Slice value() const = 0; + + // If an error has occurred, return it. Else return an ok status. + virtual Status status() const = 0; + + // Clients are allowed to register function/arg1/arg2 triples that + // will be invoked when this iterator is destroyed. + // + // Note that unlike all of the preceding methods, this method is + // not abstract and therefore clients should not override it. + typedef void (*CleanupFunction)(void* arg1, void* arg2); + void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2); + + private: + struct Cleanup { + CleanupFunction function; + void* arg1; + void* arg2; + Cleanup* next; + }; + Cleanup cleanup_; + + // No copying allowed + Iterator(const Iterator&); + void operator=(const Iterator&); +}; + +// Return an empty iterator (yields nothing). +extern Iterator* NewEmptyIterator(); + +// Return an empty iterator with the specified status. +extern Iterator* NewErrorIterator(const Status& status); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_ITERATOR_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/options.h b/Subtrees/hyperleveldb/hyperleveldb/options.h new file mode 100644 index 0000000000..29c2b0209c --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/options.h @@ -0,0 +1,195 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_OPTIONS_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_OPTIONS_H_ + +#include + +namespace hyperleveldb { + +class Cache; +class Comparator; +class Env; +class FilterPolicy; +class Logger; +class Snapshot; + +// DB contents are stored in a set of blocks, each of which holds a +// sequence of key,value pairs. Each block may be compressed before +// being stored in a file. The following enum describes which +// compression method (if any) is used to compress a block. +enum CompressionType { + // NOTE: do not change the values of existing entries, as these are + // part of the persistent format on disk. + kNoCompression = 0x0, + kSnappyCompression = 0x1 +}; + +// Options to control the behavior of a database (passed to DB::Open) +struct Options { + // ------------------- + // Parameters that affect behavior + + // Comparator used to define the order of keys in the table. + // Default: a comparator that uses lexicographic byte-wise ordering + // + // REQUIRES: The client must ensure that the comparator supplied + // here has the same name and orders keys *exactly* the same as the + // comparator provided to previous open calls on the same DB. + const Comparator* comparator; + + // If true, the database will be created if it is missing. + // Default: false + bool create_if_missing; + + // If true, an error is raised if the database already exists. + // Default: false + bool error_if_exists; + + // If true, the implementation will do aggressive checking of the + // data it is processing and will stop early if it detects any + // errors. This may have unforeseen ramifications: for example, a + // corruption of one DB entry may cause a large number of entries to + // become unreadable or for the entire DB to become unopenable. + // Default: false + bool paranoid_checks; + + // Use the specified object to interact with the environment, + // e.g. to read/write files, schedule background work, etc. + // Default: Env::Default() + Env* env; + + // Any internal progress/error information generated by the db will + // be written to info_log if it is non-NULL, or to a file stored + // in the same directory as the DB contents if info_log is NULL. + // Default: NULL + Logger* info_log; + + // ------------------- + // Parameters that affect performance + + // Amount of data to build up in memory (backed by an unsorted log + // on disk) before converting to a sorted on-disk file. + // + // Larger values increase performance, especially during bulk loads. + // Up to two write buffers may be held in memory at the same time, + // so you may wish to adjust this parameter to control memory usage. + // Also, a larger write buffer will result in a longer recovery time + // the next time the database is opened. + // + // Default: 4MB + size_t write_buffer_size; + + // Number of open files that can be used by the DB. You may need to + // increase this if your database has a large working set (budget + // one open file per 2MB of working set). + // + // Default: 1000 + int max_open_files; + + // Control over blocks (user data is stored in a set of blocks, and + // a block is the unit of reading from disk). + + // If non-NULL, use the specified cache for blocks. + // If NULL, leveldb will automatically create and use an 8MB internal cache. + // Default: NULL + Cache* block_cache; + + // Approximate size of user data packed per block. Note that the + // block size specified here corresponds to uncompressed data. The + // actual size of the unit read from disk may be smaller if + // compression is enabled. This parameter can be changed dynamically. + // + // Default: 4K + size_t block_size; + + // Number of keys between restart points for delta encoding of keys. + // This parameter can be changed dynamically. Most clients should + // leave this parameter alone. + // + // Default: 16 + int block_restart_interval; + + // Compress blocks using the specified compression algorithm. This + // parameter can be changed dynamically. + // + // Default: kSnappyCompression, which gives lightweight but fast + // compression. + // + // Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz: + // ~200-500MB/s compression + // ~400-800MB/s decompression + // Note that these speeds are significantly faster than most + // persistent storage speeds, and therefore it is typically never + // worth switching to kNoCompression. Even if the input data is + // incompressible, the kSnappyCompression implementation will + // efficiently detect that and will switch to uncompressed mode. + CompressionType compression; + + // If non-NULL, use the specified filter policy to reduce disk reads. + // Many applications will benefit from passing the result of + // NewBloomFilterPolicy() here. + // + // Default: NULL + const FilterPolicy* filter_policy; + + // Create an Options object with default values for all fields. + Options(); +}; + +// Options that control read operations +struct ReadOptions { + // If true, all data read from underlying storage will be + // verified against corresponding checksums. + // Default: false + bool verify_checksums; + + // Should the data read for this iteration be cached in memory? + // Callers may wish to set this field to false for bulk scans. + // Default: true + bool fill_cache; + + // If "snapshot" is non-NULL, read as of the supplied snapshot + // (which must belong to the DB that is being read and which must + // not have been released). If "snapshot" is NULL, use an impliicit + // snapshot of the state at the beginning of this read operation. + // Default: NULL + const Snapshot* snapshot; + + ReadOptions() + : verify_checksums(false), + fill_cache(true), + snapshot(NULL) { + } +}; + +// Options that control write operations +struct WriteOptions { + // If true, the write will be flushed from the operating system + // buffer cache (by calling WritableFile::Sync()) before the write + // is considered complete. If this flag is true, writes will be + // slower. + // + // If this flag is false, and the machine crashes, some recent + // writes may be lost. Note that if it is just the process that + // crashes (i.e., the machine does not reboot), no writes will be + // lost even if sync==false. + // + // In other words, a DB write with sync==false has similar + // crash semantics as the "write()" system call. A DB write + // with sync==true has similar crash semantics to a "write()" + // system call followed by "fsync()". + // + // Default: false + bool sync; + + WriteOptions() + : sync(false) { + } +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_OPTIONS_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/slice.h b/Subtrees/hyperleveldb/hyperleveldb/slice.h new file mode 100644 index 0000000000..4dc2ac2391 --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/slice.h @@ -0,0 +1,109 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Slice is a simple structure containing a pointer into some external +// storage and a size. The user of a Slice must ensure that the slice +// is not used after the corresponding external storage has been +// deallocated. +// +// Multiple threads can invoke const methods on a Slice without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Slice must use +// external synchronization. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_SLICE_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_SLICE_H_ + +#include +#include +#include +#include + +namespace hyperleveldb { + +class Slice { + public: + // Create an empty slice. + Slice() : data_(""), size_(0) { } + + // Create a slice that refers to d[0,n-1]. + Slice(const char* d, size_t n) : data_(d), size_(n) { } + + // Create a slice that refers to the contents of "s" + Slice(const std::string& s) : data_(s.data()), size_(s.size()) { } + + // Create a slice that refers to s[0,strlen(s)-1] + Slice(const char* s) : data_(s), size_(strlen(s)) { } + + // Return a pointer to the beginning of the referenced data + const char* data() const { return data_; } + + // Return the length (in bytes) of the referenced data + size_t size() const { return size_; } + + // Return true iff the length of the referenced data is zero + bool empty() const { return size_ == 0; } + + // Return the ith byte in the referenced data. + // REQUIRES: n < size() + char operator[](size_t n) const { + assert(n < size()); + return data_[n]; + } + + // Change this slice to refer to an empty array + void clear() { data_ = ""; size_ = 0; } + + // Drop the first "n" bytes from this slice. + void remove_prefix(size_t n) { + assert(n <= size()); + data_ += n; + size_ -= n; + } + + // Return a string that contains the copy of the referenced data. + std::string ToString() const { return std::string(data_, size_); } + + // Three-way comparison. Returns value: + // < 0 iff "*this" < "b", + // == 0 iff "*this" == "b", + // > 0 iff "*this" > "b" + int compare(const Slice& b) const; + + // Return true iff "x" is a prefix of "*this" + bool starts_with(const Slice& x) const { + return ((size_ >= x.size_) && + (memcmp(data_, x.data_, x.size_) == 0)); + } + + private: + const char* data_; + size_t size_; + + // Intentionally copyable +}; + +inline bool operator==(const Slice& x, const Slice& y) { + return ((x.size() == y.size()) && + (memcmp(x.data(), y.data(), x.size()) == 0)); +} + +inline bool operator!=(const Slice& x, const Slice& y) { + return !(x == y); +} + +inline int Slice::compare(const Slice& b) const { + const int min_len = (size_ < b.size_) ? size_ : b.size_; + int r = memcmp(data_, b.data_, min_len); + if (r == 0) { + if (size_ < b.size_) r = -1; + else if (size_ > b.size_) r = +1; + } + return r; +} + +} // namespace hyperleveldb + + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_SLICE_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/status.h b/Subtrees/hyperleveldb/hyperleveldb/status.h new file mode 100644 index 0000000000..2d091e575d --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/status.h @@ -0,0 +1,106 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A Status encapsulates the result of an operation. It may indicate success, +// or it may indicate an error with an associated error message. +// +// Multiple threads can invoke const methods on a Status without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Status must use +// external synchronization. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_STATUS_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_STATUS_H_ + +#include +#include "slice.h" + +namespace hyperleveldb { + +class Status { + public: + // Create a success status. + Status() : state_(NULL) { } + ~Status() { delete[] state_; } + + // Copy the specified status. + Status(const Status& s); + void operator=(const Status& s); + + // Return a success status. + static Status OK() { return Status(); } + + // Return error status of an appropriate type. + static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kNotFound, msg, msg2); + } + static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kCorruption, msg, msg2); + } + static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kNotSupported, msg, msg2); + } + static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kInvalidArgument, msg, msg2); + } + static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kIOError, msg, msg2); + } + + // Returns true iff the status indicates success. + bool ok() const { return (state_ == NULL); } + + // Returns true iff the status indicates a NotFound error. + bool IsNotFound() const { return code() == kNotFound; } + + // Returns true iff the status indicates a Corruption error. + bool IsCorruption() const { return code() == kCorruption; } + + // Returns true iff the status indicates an IOError. + bool IsIOError() const { return code() == kIOError; } + + // Return a string representation of this status suitable for printing. + // Returns the string "OK" for success. + std::string ToString() const; + + private: + // OK status has a NULL state_. Otherwise, state_ is a new[] array + // of the following form: + // state_[0..3] == length of message + // state_[4] == code + // state_[5..] == message + const char* state_; + + enum Code { + kOk = 0, + kNotFound = 1, + kCorruption = 2, + kNotSupported = 3, + kInvalidArgument = 4, + kIOError = 5 + }; + + Code code() const { + return (state_ == NULL) ? kOk : static_cast(state_[4]); + } + + Status(Code code, const Slice& msg, const Slice& msg2); + static const char* CopyState(const char* s); +}; + +inline Status::Status(const Status& s) { + state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); +} +inline void Status::operator=(const Status& s) { + // The following condition catches both aliasing (when this == &s), + // and the common case where both s and *this are ok. + if (state_ != s.state_) { + delete[] state_; + state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); + } +} + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_STATUS_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/table.h b/Subtrees/hyperleveldb/hyperleveldb/table.h new file mode 100644 index 0000000000..e29399eed3 --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/table.h @@ -0,0 +1,85 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_TABLE_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_TABLE_H_ + +#include +#include "iterator.h" + +namespace hyperleveldb { + +class Block; +class BlockHandle; +class Footer; +struct Options; +class RandomAccessFile; +struct ReadOptions; +class TableCache; + +// A Table is a sorted map from strings to strings. Tables are +// immutable and persistent. A Table may be safely accessed from +// multiple threads without external synchronization. +class Table { + public: + // Attempt to open the table that is stored in bytes [0..file_size) + // of "file", and read the metadata entries necessary to allow + // retrieving data from the table. + // + // If successful, returns ok and sets "*table" to the newly opened + // table. The client should delete "*table" when no longer needed. + // If there was an error while initializing the table, sets "*table" + // to NULL and returns a non-ok status. Does not take ownership of + // "*source", but the client must ensure that "source" remains live + // for the duration of the returned table's lifetime. + // + // *file must remain live while this Table is in use. + static Status Open(const Options& options, + RandomAccessFile* file, + uint64_t file_size, + Table** table); + + ~Table(); + + // Returns a new iterator over the table contents. + // The result of NewIterator() is initially invalid (caller must + // call one of the Seek methods on the iterator before using it). + Iterator* NewIterator(const ReadOptions&) const; + + // Given a key, return an approximate byte offset in the file where + // the data for that key begins (or would begin if the key were + // present in the file). The returned value is in terms of file + // bytes, and so includes effects like compression of the underlying data. + // E.g., the approximate offset of the last key in the table will + // be close to the file length. + uint64_t ApproximateOffsetOf(const Slice& key) const; + + private: + struct Rep; + Rep* rep_; + + explicit Table(Rep* rep) { rep_ = rep; } + static Iterator* BlockReader(void*, const ReadOptions&, const Slice&); + + // Calls (*handle_result)(arg, ...) with the entry found after a call + // to Seek(key). May not make such a call if filter policy says + // that key is not present. + friend class TableCache; + Status InternalGet( + const ReadOptions&, const Slice& key, + void* arg, + void (*handle_result)(void* arg, const Slice& k, const Slice& v)); + + + void ReadMeta(const Footer& footer); + void ReadFilter(const Slice& filter_handle_value); + + // No copying allowed + Table(const Table&); + void operator=(const Table&); +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_TABLE_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/table_builder.h b/Subtrees/hyperleveldb/hyperleveldb/table_builder.h new file mode 100644 index 0000000000..5d40c07d9b --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/table_builder.h @@ -0,0 +1,92 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// TableBuilder provides the interface used to build a Table +// (an immutable and sorted map from keys to values). +// +// Multiple threads can invoke const methods on a TableBuilder without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same TableBuilder must use +// external synchronization. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_TABLE_BUILDER_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_TABLE_BUILDER_H_ + +#include +#include "options.h" +#include "status.h" + +namespace hyperleveldb { + +class BlockBuilder; +class BlockHandle; +class WritableFile; + +class TableBuilder { + public: + // Create a builder that will store the contents of the table it is + // building in *file. Does not close the file. It is up to the + // caller to close the file after calling Finish(). + TableBuilder(const Options& options, WritableFile* file); + + // REQUIRES: Either Finish() or Abandon() has been called. + ~TableBuilder(); + + // Change the options used by this builder. Note: only some of the + // option fields can be changed after construction. If a field is + // not allowed to change dynamically and its value in the structure + // passed to the constructor is different from its value in the + // structure passed to this method, this method will return an error + // without changing any fields. + Status ChangeOptions(const Options& options); + + // Add key,value to the table being constructed. + // REQUIRES: key is after any previously added key according to comparator. + // REQUIRES: Finish(), Abandon() have not been called + void Add(const Slice& key, const Slice& value); + + // Advanced operation: flush any buffered key/value pairs to file. + // Can be used to ensure that two adjacent entries never live in + // the same data block. Most clients should not need to use this method. + // REQUIRES: Finish(), Abandon() have not been called + void Flush(); + + // Return non-ok iff some error has been detected. + Status status() const; + + // Finish building the table. Stops using the file passed to the + // constructor after this function returns. + // REQUIRES: Finish(), Abandon() have not been called + Status Finish(); + + // Indicate that the contents of this builder should be abandoned. Stops + // using the file passed to the constructor after this function returns. + // If the caller is not going to call Finish(), it must call Abandon() + // before destroying this builder. + // REQUIRES: Finish(), Abandon() have not been called + void Abandon(); + + // Number of calls to Add() so far. + uint64_t NumEntries() const; + + // Size of the file generated so far. If invoked after a successful + // Finish() call, returns the size of the final generated file. + uint64_t FileSize() const; + + private: + bool ok() const { return status().ok(); } + void WriteBlock(BlockBuilder* block, BlockHandle* handle); + void WriteRawBlock(const Slice& data, CompressionType, BlockHandle* handle); + + struct Rep; + Rep* rep_; + + // No copying allowed + TableBuilder(const TableBuilder&); + void operator=(const TableBuilder&); +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_TABLE_BUILDER_H_ diff --git a/Subtrees/hyperleveldb/hyperleveldb/write_batch.h b/Subtrees/hyperleveldb/hyperleveldb/write_batch.h new file mode 100644 index 0000000000..42564cfab6 --- /dev/null +++ b/Subtrees/hyperleveldb/hyperleveldb/write_batch.h @@ -0,0 +1,64 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// WriteBatch holds a collection of updates to apply atomically to a DB. +// +// The updates are applied in the order in which they are added +// to the WriteBatch. For example, the value of "key" will be "v3" +// after the following batch is written: +// +// batch.Put("key", "v1"); +// batch.Delete("key"); +// batch.Put("key", "v2"); +// batch.Put("key", "v3"); +// +// Multiple threads can invoke const methods on a WriteBatch without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same WriteBatch must use +// external synchronization. + +#ifndef STORAGE_HYPERLEVELDB_INCLUDE_WRITE_BATCH_H_ +#define STORAGE_HYPERLEVELDB_INCLUDE_WRITE_BATCH_H_ + +#include +#include "status.h" + +namespace hyperleveldb { + +class Slice; + +class WriteBatch { + public: + WriteBatch(); + ~WriteBatch(); + + // Store the mapping "key->value" in the database. + void Put(const Slice& key, const Slice& value); + + // If the database contains a mapping for "key", erase it. Else do nothing. + void Delete(const Slice& key); + + // Clear all updates buffered in this batch. + void Clear(); + + // Support for iterating over the contents of a batch. + class Handler { + public: + virtual ~Handler(); + virtual void Put(const Slice& key, const Slice& value) = 0; + virtual void Delete(const Slice& key) = 0; + }; + Status Iterate(Handler* handler) const; + + private: + friend class WriteBatchInternal; + + std::string rep_; // See comment in write_batch.cc for the format of rep_ + + // Intentionally copyable +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_INCLUDE_WRITE_BATCH_H_ diff --git a/Subtrees/hyperleveldb/issues/issue178_test.cc b/Subtrees/hyperleveldb/issues/issue178_test.cc new file mode 100644 index 0000000000..05dc01af47 --- /dev/null +++ b/Subtrees/hyperleveldb/issues/issue178_test.cc @@ -0,0 +1,88 @@ +// Test for issue 178: a manual compaction causes deleted data to reappear. +#include +#include +#include + +#include "hyperleveldb/db.h" +#include "hyperleveldb/write_batch.h" +#include "util/testharness.h" + +namespace { + +const int kNumKeys = 1100000; + +std::string Key1(int i) { + char buf[100]; + snprintf(buf, sizeof(buf), "my_key_%d", i); + return buf; +} + +std::string Key2(int i) { + return Key1(i) + "_xxx"; +} + +class Issue178 { }; + +TEST(Issue178, Test) { + // Get rid of any state from an old run. + std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test"; + DestroyDB(dbpath, leveldb::Options()); + + // Open database. Disable compression since it affects the creation + // of layers and the code below is trying to test against a very + // specific scenario. + leveldb::DB* db; + leveldb::Options db_options; + db_options.create_if_missing = true; + db_options.compression = leveldb::kNoCompression; + ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db)); + + // create first key range + leveldb::WriteBatch batch; + for (size_t i = 0; i < kNumKeys; i++) { + batch.Put(Key1(i), "value for range 1 key"); + } + ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // create second key range + batch.Clear(); + for (size_t i = 0; i < kNumKeys; i++) { + batch.Put(Key2(i), "value for range 2 key"); + } + ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // delete second key range + batch.Clear(); + for (size_t i = 0; i < kNumKeys; i++) { + batch.Delete(Key2(i)); + } + ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // compact database + std::string start_key = Key1(0); + std::string end_key = Key1(kNumKeys - 1); + leveldb::Slice least(start_key.data(), start_key.size()); + leveldb::Slice greatest(end_key.data(), end_key.size()); + + // commenting out the line below causes the example to work correctly + db->CompactRange(&least, &greatest); + + // count the keys + leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions()); + size_t num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + num_keys++; + } + delete iter; + ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys"; + + // close database + delete db; + DestroyDB(dbpath, leveldb::Options()); +} + +} // anonymous namespace + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/libhyperleveldb.pc.in b/Subtrees/hyperleveldb/libhyperleveldb.pc.in new file mode 100644 index 0000000000..912226726c --- /dev/null +++ b/Subtrees/hyperleveldb/libhyperleveldb.pc.in @@ -0,0 +1,12 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: HyperLevelDB +Description: A fast and lightweight key-value storage library +Version: @VERSION@ + +Requires: +Libs: -L${libdir} -lhyperleveldb +Cflags: -I${includedir} diff --git a/Subtrees/hyperleveldb/m4/anal_warnings_cxx.m4 b/Subtrees/hyperleveldb/m4/anal_warnings_cxx.m4 new file mode 100644 index 0000000000..e6b7cfbbb3 --- /dev/null +++ b/Subtrees/hyperleveldb/m4/anal_warnings_cxx.m4 @@ -0,0 +1,118 @@ +# Copyright (c) 2012, Robert Escriva +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of this project nor the names of its contributors may +# be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This macro enables many compiler warnings for C++ that generally catch bugs in +# code. It offers the "--enable-wanal-cxxflags" option which defaults to "no". + +AC_DEFUN([ANAL_WARNINGS_CXX], + [WANAL_CXXFLAGS="" + AC_ARG_ENABLE([wanal-cxxflags], [AS_HELP_STRING([--enable-wanal-cxxflags], + [enable many warnings @<:@default: no@:>@])], + [wanal_cxxflags=${enableval}], [wanal_cxxflags=no]) + if test x"${wanal_cxxflags}" = xyes; then + AX_CHECK_COMPILE_FLAG([-pedantic],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -pedantic"],,) + AX_CHECK_COMPILE_FLAG([-Wall],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wall"],,) + AX_CHECK_COMPILE_FLAG([-Wextra],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wextra"],,) + AX_CHECK_COMPILE_FLAG([-Wabi],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wabi"],,) + AX_CHECK_COMPILE_FLAG([-Waddress],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Waddress"],,) + #AX_CHECK_COMPILE_FLAG([-Waggregate-return],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Waggregate-return"],,) + AX_CHECK_COMPILE_FLAG([-Warray-bounds],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Warray-bounds"],,) + AX_CHECK_COMPILE_FLAG([-Wc++0x-compat],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wc++0x-compat"],,) + AX_CHECK_COMPILE_FLAG([-Wcast-align],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wcast-align"],,) + AX_CHECK_COMPILE_FLAG([-Wcast-qual],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wcast-qual"],,) + AX_CHECK_COMPILE_FLAG([-Wchar-subscripts],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wchar-subscripts"],,) + AX_CHECK_COMPILE_FLAG([-Wclobbered],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wclobbered"],,) + AX_CHECK_COMPILE_FLAG([-Wcomment],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wcomment"],,) + AX_CHECK_COMPILE_FLAG([-Wconversion],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wconversion"],,) + AX_CHECK_COMPILE_FLAG([-Wctor-dtor-privacy],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wctor-dtor-privacy"],,) + AX_CHECK_COMPILE_FLAG([-Wdisabled-optimization],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wdisabled-optimization"],,) + AX_CHECK_COMPILE_FLAG([-Weffc++],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Weffc++"],,) + AX_CHECK_COMPILE_FLAG([-Wempty-body],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wempty-body"],,) + AX_CHECK_COMPILE_FLAG([-Wenum-compare],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wenum-compare"],,) + AX_CHECK_COMPILE_FLAG([-Wfloat-equal],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wfloat-equal"],,) + AX_CHECK_COMPILE_FLAG([-Wformat],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wformat"],,) + AX_CHECK_COMPILE_FLAG([-Wformat=2],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wformat=2"],,) + AX_CHECK_COMPILE_FLAG([-Wformat-nonliteral],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wformat-nonliteral"],,) + AX_CHECK_COMPILE_FLAG([-Wformat-security],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wformat-security"],,) + AX_CHECK_COMPILE_FLAG([-Wformat-y2k],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wformat-y2k"],,) + AX_CHECK_COMPILE_FLAG([-Wframe-larger-than=8192],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wframe-larger-than=8192"],,) + AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wignored-qualifiers"],,) + AX_CHECK_COMPILE_FLAG([-Wimplicit],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wimplicit"],,) + AX_CHECK_COMPILE_FLAG([-Winit-self],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Winit-self"],,) + AX_CHECK_COMPILE_FLAG([-Winline],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Winline"],,) + AX_CHECK_COMPILE_FLAG([-Wlarger-than=4096],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wlarger-than=4096"],,) + AX_CHECK_COMPILE_FLAG([-Wlogical-op],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wlogical-op"],,) + AX_CHECK_COMPILE_FLAG([-Wmain],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wmain"],,) + AX_CHECK_COMPILE_FLAG([-Wmissing-braces],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wmissing-braces"],,) + AX_CHECK_COMPILE_FLAG([-Wmissing-declarations],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wmissing-declarations"],,) + AX_CHECK_COMPILE_FLAG([-Wmissing-field-initializers],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wmissing-field-initializers"],,) + AX_CHECK_COMPILE_FLAG([-Wmissing-format-attribute],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wmissing-format-attribute"],,) + AX_CHECK_COMPILE_FLAG([-Wmissing-include-dirs],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wmissing-include-dirs"],,) + AX_CHECK_COMPILE_FLAG([-Wmissing-noreturn],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wmissing-noreturn"],,) + AX_CHECK_COMPILE_FLAG([-Wnon-virtual-dtor],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wnon-virtual-dtor"],,) + AX_CHECK_COMPILE_FLAG([-Wold-style-cast],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wold-style-cast"],,) + AX_CHECK_COMPILE_FLAG([-Woverlength-strings],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Woverlength-strings"],,) + AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Woverloaded-virtual"],,) + AX_CHECK_COMPILE_FLAG([-Wpacked],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wpacked"],,) + AX_CHECK_COMPILE_FLAG([-Wpacked-bitfield-compat],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wpacked-bitfield-compat"],,) + AX_CHECK_COMPILE_FLAG([-Wpadded],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wpadded"],,) + AX_CHECK_COMPILE_FLAG([-Wparentheses],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wparentheses"],,) + AX_CHECK_COMPILE_FLAG([-Wpointer-arith],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wpointer-arith"],,) + AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wredundant-decls"],,) + AX_CHECK_COMPILE_FLAG([-Wreorder],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wreorder"],,) + AX_CHECK_COMPILE_FLAG([-Wreturn-type],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wreturn-type"],,) + AX_CHECK_COMPILE_FLAG([-Wsequence-point],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wsequence-point"],,) + AX_CHECK_COMPILE_FLAG([-Wshadow],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wshadow"],,) + AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wsign-compare"],,) + AX_CHECK_COMPILE_FLAG([-Wsign-conversion],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wsign-conversion"],,) + AX_CHECK_COMPILE_FLAG([-Wsign-promo],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wsign-promo"],,) + AX_CHECK_COMPILE_FLAG([-Wstack-protector],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wstack-protector"],,) + AX_CHECK_COMPILE_FLAG([-Wstrict-aliasing],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wstrict-aliasing"],,) + AX_CHECK_COMPILE_FLAG([-Wstrict-aliasing=3],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wstrict-aliasing=3"],,) + AX_CHECK_COMPILE_FLAG([-Wstrict-null-sentinel],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wstrict-null-sentinel"],,) + AX_CHECK_COMPILE_FLAG([-Wstrict-overflow],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wstrict-overflow"],,) + AX_CHECK_COMPILE_FLAG([-Wstrict-overflow=4],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wstrict-overflow=4"],,) + AX_CHECK_COMPILE_FLAG([-Wswitch],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wswitch"],,) + AX_CHECK_COMPILE_FLAG([-Wswitch-default],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wswitch-default"],,) + AX_CHECK_COMPILE_FLAG([-Wswitch-enum],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wswitch-enum"],,) + AX_CHECK_COMPILE_FLAG([-Wtrigraphs],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wtrigraphs"],,) + AX_CHECK_COMPILE_FLAG([-Wtype-limits],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wtype-limits"],,) + AX_CHECK_COMPILE_FLAG([-Wundef],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wundef"],,) + AX_CHECK_COMPILE_FLAG([-Wuninitialized],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wuninitialized"],,) + #AX_CHECK_COMPILE_FLAG([-Wunreachable-code],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wunreachable-code"],,) + AX_CHECK_COMPILE_FLAG([-Wunsafe-loop-optimizations],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wunsafe-loop-optimizations"],,) + AX_CHECK_COMPILE_FLAG([-Wunused],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wunused"],,) + AX_CHECK_COMPILE_FLAG([-Wunused-function],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wunused-function"],,) + AX_CHECK_COMPILE_FLAG([-Wunused-label],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wunused-label"],,) + AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wunused-parameter"],,) + AX_CHECK_COMPILE_FLAG([-Wunused-value],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wunused-value"],,) + AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wunused-variable"],,) + AX_CHECK_COMPILE_FLAG([-Wvolatile-register-var],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wvolatile-register-var"],,) + AX_CHECK_COMPILE_FLAG([-Wwrite-strings],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wwrite-strings"],,) + AX_CHECK_COMPILE_FLAG([-Wno-long-long],[WANAL_CXXFLAGS="${WANAL_CXXFLAGS} -Wno-long-long"],,) + fi + AC_SUBST([WANAL_CXXFLAGS], [${WANAL_CXXFLAGS}]) +]) diff --git a/Subtrees/hyperleveldb/m4/ax_check_compile_flag.m4 b/Subtrees/hyperleveldb/m4/ax_check_compile_flag.m4 new file mode 100644 index 0000000000..c3a8d695a1 --- /dev/null +++ b/Subtrees/hyperleveldb/m4/ax_check_compile_flag.m4 @@ -0,0 +1,72 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2011 Maarten Bosmans +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 2 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/Subtrees/hyperleveldb/m4/ax_check_link_flag.m4 b/Subtrees/hyperleveldb/m4/ax_check_link_flag.m4 new file mode 100644 index 0000000000..e2d0d363e4 --- /dev/null +++ b/Subtrees/hyperleveldb/m4/ax_check_link_flag.m4 @@ -0,0 +1,71 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_LINK_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the linker or gives an error. +# (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the linker's default flags +# when the check is done. The check is thus made with the flags: "LDFLAGS +# EXTRA-FLAGS FLAG". This can for example be used to force the linker to +# issue an error when a bad flag is given. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,COMPILE}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2011 Maarten Bosmans +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 2 + +AC_DEFUN([AX_CHECK_LINK_FLAG], +[AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_ldflags_$4_$1])dnl +AC_CACHE_CHECK([whether the linker accepts $1], CACHEVAR, [ + ax_check_save_flags=$LDFLAGS + LDFLAGS="$LDFLAGS $4 $1" + AC_LINK_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + LDFLAGS=$ax_check_save_flags]) +AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_LINK_FLAGS diff --git a/Subtrees/hyperleveldb/m4/ax_check_preproc_flag.m4 b/Subtrees/hyperleveldb/m4/ax_check_preproc_flag.m4 new file mode 100644 index 0000000000..b1cfef6b86 --- /dev/null +++ b/Subtrees/hyperleveldb/m4/ax_check_preproc_flag.m4 @@ -0,0 +1,72 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_PREPROC_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's +# preprocessor or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the preprocessor's default +# flags when the check is done. The check is thus made with the flags: +# "CPPFLAGS EXTRA-FLAGS FLAG". This can for example be used to force the +# preprocessor to issue an error when a bad flag is given. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{COMPILE,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2011 Maarten Bosmans +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 2 + +AC_DEFUN([AX_CHECK_PREPROC_FLAG], +[AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]cppflags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG preprocessor accepts $1], CACHEVAR, [ + ax_check_save_flags=$CPPFLAGS + CPPFLAGS="$CPPFLAGS $4 $1" + AC_PREPROC_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + CPPFLAGS=$ax_check_save_flags]) +AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_PREPROC_FLAGS diff --git a/Subtrees/hyperleveldb/port/README b/Subtrees/hyperleveldb/port/README new file mode 100644 index 0000000000..422563e25c --- /dev/null +++ b/Subtrees/hyperleveldb/port/README @@ -0,0 +1,10 @@ +This directory contains interfaces and implementations that isolate the +rest of the package from platform details. + +Code in the rest of the package includes "port.h" from this directory. +"port.h" in turn includes a platform specific "port_.h" file +that provides the platform specific implementation. + +See port_posix.h for an example of what must be provided in a platform +specific header file. + diff --git a/Subtrees/hyperleveldb/port/atomic_pointer.h b/Subtrees/hyperleveldb/port/atomic_pointer.h new file mode 100644 index 0000000000..57af05c1c4 --- /dev/null +++ b/Subtrees/hyperleveldb/port/atomic_pointer.h @@ -0,0 +1,224 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// AtomicPointer provides storage for a lock-free pointer. +// Platform-dependent implementation of AtomicPointer: +// - If the platform provides a cheap barrier, we use it with raw pointers +// - If cstdatomic is present (on newer versions of gcc, it is), we use +// a cstdatomic-based AtomicPointer. However we prefer the memory +// barrier based version, because at least on a gcc 4.4 32-bit build +// on linux, we have encountered a buggy +// implementation. Also, some implementations are much +// slower than a memory-barrier based implementation (~16ns for +// based acquire-load vs. ~1ns for a barrier based +// acquire-load). +// This code is based on atomicops-internals-* in Google's perftools: +// http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase + +#ifndef PORT_ATOMIC_POINTER_H_ +#define PORT_ATOMIC_POINTER_H_ + +#include +#ifdef LEVELDB_CSTDATOMIC_PRESENT +#include +#endif +#ifdef OS_WIN +#include +#endif +#ifdef OS_MACOSX +#include +#endif + +#if defined(_M_X64) || defined(__x86_64__) +#define ARCH_CPU_X86_FAMILY 1 +#elif defined(_M_IX86) || defined(__i386__) || defined(__i386) +#define ARCH_CPU_X86_FAMILY 1 +#elif defined(__ARMEL__) +#define ARCH_CPU_ARM_FAMILY 1 +#elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__) +#define ARCH_CPU_PPC_FAMILY 1 +#endif + +namespace hyperleveldb { +namespace port { + +// Define MemoryBarrier() if available +// Windows on x86 +#if defined(OS_WIN) && defined(COMPILER_MSVC) && defined(ARCH_CPU_X86_FAMILY) +// windows.h already provides a MemoryBarrier(void) macro +// http://msdn.microsoft.com/en-us/library/ms684208(v=vs.85).aspx +#define LEVELDB_HAVE_MEMORY_BARRIER + +// Gcc on x86 +#elif defined(ARCH_CPU_X86_FAMILY) && defined(__GNUC__) +inline void MemoryBarrier() { + // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on + // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. + __asm__ __volatile__("" : : : "memory"); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + +// Sun Studio +#elif defined(ARCH_CPU_X86_FAMILY) && defined(__SUNPRO_CC) +inline void MemoryBarrier() { + // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on + // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. + asm volatile("" : : : "memory"); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + +// Mac OS +#elif defined(OS_MACOSX) +inline void MemoryBarrier() { + OSMemoryBarrier(); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + +// ARM Linux +#elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__) +typedef void (*LinuxKernelMemoryBarrierFunc)(void); +// The Linux ARM kernel provides a highly optimized device-specific memory +// barrier function at a fixed memory address that is mapped in every +// user-level process. +// +// This beats using CPU-specific instructions which are, on single-core +// devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more +// than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking +// shows that the extra function call cost is completely negligible on +// multi-core devices. +// +inline void MemoryBarrier() { + (*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)(); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + +// PPC +#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__) +inline void MemoryBarrier() { + // TODO for some powerpc expert: is there a cheaper suitable variant? + // Perhaps by having separate barriers for acquire and release ops. + asm volatile("sync" : : : "memory"); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + +#endif + +// AtomicPointer built using platform-specific MemoryBarrier() +#if defined(LEVELDB_HAVE_MEMORY_BARRIER) +class AtomicPointer { + private: + void* rep_; + public: + AtomicPointer() { } + explicit AtomicPointer(void* p) : rep_(p) {} + inline void* NoBarrier_Load() const { return rep_; } + inline void NoBarrier_Store(void* v) { rep_ = v; } + inline void* Acquire_Load() const { + void* result = rep_; + MemoryBarrier(); + return result; + } + inline void Release_Store(void* v) { + MemoryBarrier(); + rep_ = v; + } +}; + +// AtomicPointer based on +#elif defined(LEVELDB_CSTDATOMIC_PRESENT) +class AtomicPointer { + private: + std::atomic rep_; + public: + AtomicPointer() { } + explicit AtomicPointer(void* v) : rep_(v) { } + inline void* Acquire_Load() const { + return rep_.load(std::memory_order_acquire); + } + inline void Release_Store(void* v) { + rep_.store(v, std::memory_order_release); + } + inline void* NoBarrier_Load() const { + return rep_.load(std::memory_order_relaxed); + } + inline void NoBarrier_Store(void* v) { + rep_.store(v, std::memory_order_relaxed); + } +}; + +// Atomic pointer based on sparc memory barriers +#elif defined(__sparcv9) && defined(__GNUC__) +class AtomicPointer { + private: + void* rep_; + public: + AtomicPointer() { } + explicit AtomicPointer(void* v) : rep_(v) { } + inline void* Acquire_Load() const { + void* val; + __asm__ __volatile__ ( + "ldx [%[rep_]], %[val] \n\t" + "membar #LoadLoad|#LoadStore \n\t" + : [val] "=r" (val) + : [rep_] "r" (&rep_) + : "memory"); + return val; + } + inline void Release_Store(void* v) { + __asm__ __volatile__ ( + "membar #LoadStore|#StoreStore \n\t" + "stx %[v], [%[rep_]] \n\t" + : + : [rep_] "r" (&rep_), [v] "r" (v) + : "memory"); + } + inline void* NoBarrier_Load() const { return rep_; } + inline void NoBarrier_Store(void* v) { rep_ = v; } +}; + +// Atomic pointer based on ia64 acq/rel +#elif defined(__ia64) && defined(__GNUC__) +class AtomicPointer { + private: + void* rep_; + public: + AtomicPointer() { } + explicit AtomicPointer(void* v) : rep_(v) { } + inline void* Acquire_Load() const { + void* val ; + __asm__ __volatile__ ( + "ld8.acq %[val] = [%[rep_]] \n\t" + : [val] "=r" (val) + : [rep_] "r" (&rep_) + : "memory" + ); + return val; + } + inline void Release_Store(void* v) { + __asm__ __volatile__ ( + "st8.rel [%[rep_]] = %[v] \n\t" + : + : [rep_] "r" (&rep_), [v] "r" (v) + : "memory" + ); + } + inline void* NoBarrier_Load() const { return rep_; } + inline void NoBarrier_Store(void* v) { rep_ = v; } +}; + +// We have neither MemoryBarrier(), nor +#else +#error Please implement AtomicPointer for this platform. + +#endif + +#undef LEVELDB_HAVE_MEMORY_BARRIER +#undef ARCH_CPU_X86_FAMILY +#undef ARCH_CPU_ARM_FAMILY +#undef ARCH_CPU_PPC_FAMILY + +} // namespace port +} // namespace hyperleveldb + +#endif // PORT_ATOMIC_POINTER_H_ diff --git a/Subtrees/hyperleveldb/port/port.h b/Subtrees/hyperleveldb/port/port.h new file mode 100644 index 0000000000..6e81f240e8 --- /dev/null +++ b/Subtrees/hyperleveldb/port/port.h @@ -0,0 +1,19 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_PORT_PORT_H_ +#define STORAGE_HYPERLEVELDB_PORT_PORT_H_ + +#include + +// Include the appropriate platform specific file below. If you are +// porting to a new platform, see "port_example.h" for documentation +// of what the new port_.h file must provide. +#if defined(LEVELDB_PLATFORM_POSIX) +# include "port_posix.h" +#elif defined(LEVELDB_PLATFORM_CHROMIUM) +# include "port_chromium.h" +#endif + +#endif // STORAGE_HYPERLEVELDB_PORT_PORT_H_ diff --git a/Subtrees/hyperleveldb/port/port_example.h b/Subtrees/hyperleveldb/port/port_example.h new file mode 100644 index 0000000000..bf289c0014 --- /dev/null +++ b/Subtrees/hyperleveldb/port/port_example.h @@ -0,0 +1,135 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// This file contains the specification, but not the implementations, +// of the types/operations/etc. that should be defined by a platform +// specific port_.h file. Use this file as a reference for +// how to port this package to a new platform. + +#ifndef STORAGE_HYPERLEVELDB_PORT_PORT_EXAMPLE_H_ +#define STORAGE_HYPERLEVELDB_PORT_PORT_EXAMPLE_H_ + +namespace hyperleveldb { +namespace port { + +// TODO(jorlow): Many of these belong more in the environment class rather than +// here. We should try moving them and see if it affects perf. + +// The following boolean constant must be true on a little-endian machine +// and false otherwise. +static const bool kLittleEndian = true /* or some other expression */; + +// ------------------ Threading ------------------- + +// A Mutex represents an exclusive lock. +class Mutex { + public: + Mutex(); + ~Mutex(); + + // Lock the mutex. Waits until other lockers have exited. + // Will deadlock if the mutex is already locked by this thread. + void Lock(); + + // Unlock the mutex. + // REQUIRES: This mutex was locked by this thread. + void Unlock(); + + // Optionally crash if this thread does not hold this mutex. + // The implementation must be fast, especially if NDEBUG is + // defined. The implementation is allowed to skip all checks. + void AssertHeld(); +}; + +class CondVar { + public: + explicit CondVar(Mutex* mu); + ~CondVar(); + + // Atomically release *mu and block on this condition variable until + // either a call to SignalAll(), or a call to Signal() that picks + // this thread to wakeup. + // REQUIRES: this thread holds *mu + void Wait(); + + // If there are some threads waiting, wake up at least one of them. + void Signal(); + + // Wake up all waiting threads. + void SignallAll(); +}; + +// Thread-safe initialization. +// Used as follows: +// static port::OnceType init_control = LEVELDB_ONCE_INIT; +// static void Initializer() { ... do something ...; } +// ... +// port::InitOnce(&init_control, &Initializer); +typedef intptr_t OnceType; +#define LEVELDB_ONCE_INIT 0 +extern void InitOnce(port::OnceType*, void (*initializer)()); + +// A type that holds a pointer that can be read or written atomically +// (i.e., without word-tearing.) +class AtomicPointer { + private: + intptr_t rep_; + public: + // Initialize to arbitrary value + AtomicPointer(); + + // Initialize to hold v + explicit AtomicPointer(void* v) : rep_(v) { } + + // Read and return the stored pointer with the guarantee that no + // later memory access (read or write) by this thread can be + // reordered ahead of this read. + void* Acquire_Load() const; + + // Set v as the stored pointer with the guarantee that no earlier + // memory access (read or write) by this thread can be reordered + // after this store. + void Release_Store(void* v); + + // Read the stored pointer with no ordering guarantees. + void* NoBarrier_Load() const; + + // Set va as the stored pointer with no ordering guarantees. + void NoBarrier_Store(void* v); +}; + +// ------------------ Compression ------------------- + +// Store the snappy compression of "input[0,input_length-1]" in *output. +// Returns false if snappy is not supported by this port. +extern bool Snappy_Compress(const char* input, size_t input_length, + std::string* output); + +// If input[0,input_length-1] looks like a valid snappy compressed +// buffer, store the size of the uncompressed data in *result and +// return true. Else return false. +extern bool Snappy_GetUncompressedLength(const char* input, size_t length, + size_t* result); + +// Attempt to snappy uncompress input[0,input_length-1] into *output. +// Returns true if successful, false if the input is invalid lightweight +// compressed data. +// +// REQUIRES: at least the first "n" bytes of output[] must be writable +// where "n" is the result of a successful call to +// Snappy_GetUncompressedLength. +extern bool Snappy_Uncompress(const char* input_data, size_t input_length, + char* output); + +// ------------------ Miscellaneous ------------------- + +// If heap profiling is not supported, returns false. +// Else repeatedly calls (*func)(arg, data, n) and then returns true. +// The concatenation of all "data[0,n-1]" fragments is the heap profile. +extern bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg); + +} // namespace port +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_PORT_PORT_EXAMPLE_H_ diff --git a/Subtrees/hyperleveldb/port/port_posix.cc b/Subtrees/hyperleveldb/port/port_posix.cc new file mode 100644 index 0000000000..196df8233f --- /dev/null +++ b/Subtrees/hyperleveldb/port/port_posix.cc @@ -0,0 +1,54 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "port/port_posix.h" + +#include +#include +#include +#include "../util/logging.h" + +namespace hyperleveldb { +namespace port { + +static void PthreadCall(const char* label, int result) { + if (result != 0) { + fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); + abort(); + } +} + +Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } + +Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } + +void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } + +void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } + +CondVar::CondVar(Mutex* mu) + : mu_(mu) { + PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); +} + +CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } + +void CondVar::Wait() { + PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); +} + +void CondVar::Signal() { + PthreadCall("signal", pthread_cond_signal(&cv_)); +} + +void CondVar::SignalAll() { + PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); +} + +void InitOnce(OnceType* once, void (*initializer)()) { + PthreadCall("once", pthread_once(once, initializer)); +} + +} // namespace port +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/port/port_posix.h b/Subtrees/hyperleveldb/port/port_posix.h new file mode 100644 index 0000000000..933b259b36 --- /dev/null +++ b/Subtrees/hyperleveldb/port/port_posix.h @@ -0,0 +1,157 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// See port_example.h for documentation for the following types/functions. + +#ifndef STORAGE_HYPERLEVELDB_PORT_PORT_POSIX_H_ +#define STORAGE_HYPERLEVELDB_PORT_PORT_POSIX_H_ + +#undef PLATFORM_IS_LITTLE_ENDIAN +#if defined(OS_MACOSX) + #include + #if defined(__DARWIN_LITTLE_ENDIAN) && defined(__DARWIN_BYTE_ORDER) + #define PLATFORM_IS_LITTLE_ENDIAN \ + (__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN) + #endif +#elif defined(OS_SOLARIS) + #include + #ifdef _LITTLE_ENDIAN + #define PLATFORM_IS_LITTLE_ENDIAN true + #else + #define PLATFORM_IS_LITTLE_ENDIAN false + #endif +#elif defined(OS_FREEBSD) + #include + #include + #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) +#elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\ + defined(OS_DRAGONFLYBSD) + #include + #include +#elif defined(OS_HPUX) + #define PLATFORM_IS_LITTLE_ENDIAN false +#elif defined(OS_ANDROID) + // Due to a bug in the NDK x86 definition, + // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android. + // See http://code.google.com/p/android/issues/detail?id=39824 + #include + #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) +#else + #include +#endif + +#include +#ifdef SNAPPY +#include +#endif +#include +#include +#include "atomic_pointer.h" + +#ifndef PLATFORM_IS_LITTLE_ENDIAN +#define PLATFORM_IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN) +#endif + +#if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ + defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\ + defined(OS_ANDROID) || defined(OS_HPUX) +// Use fread/fwrite/fflush on platforms without _unlocked variants +#define fread_unlocked fread +#define fwrite_unlocked fwrite +#define fflush_unlocked fflush +#endif + +#if defined(OS_MACOSX) || defined(OS_FREEBSD) ||\ + defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) +// Use fsync() on platforms without fdatasync() +#define fdatasync fsync +#endif + +#if defined(OS_ANDROID) && __ANDROID_API__ < 9 +// fdatasync() was only introduced in API level 9 on Android. Use fsync() +// when targetting older platforms. +#define fdatasync fsync +#endif + +namespace hyperleveldb { +namespace port { + +static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN; +#undef PLATFORM_IS_LITTLE_ENDIAN + +class CondVar; + +class Mutex { + public: + Mutex(); + ~Mutex(); + + void Lock(); + void Unlock(); + void AssertHeld() { } + + private: + friend class CondVar; + pthread_mutex_t mu_; + + // No copying + Mutex(const Mutex&); + void operator=(const Mutex&); +}; + +class CondVar { + public: + explicit CondVar(Mutex* mu); + ~CondVar(); + void Wait(); + void Signal(); + void SignalAll(); + private: + pthread_cond_t cv_; + Mutex* mu_; +}; + +typedef pthread_once_t OnceType; +#define LEVELDB_ONCE_INIT PTHREAD_ONCE_INIT +extern void InitOnce(OnceType* once, void (*initializer)()); + +inline bool Snappy_Compress(const char* input, size_t length, + ::std::string* output) { +#ifdef SNAPPY + output->resize(snappy::MaxCompressedLength(length)); + size_t outlen; + snappy::RawCompress(input, length, &(*output)[0], &outlen); + output->resize(outlen); + return true; +#endif + + return false; +} + +inline bool Snappy_GetUncompressedLength(const char* input, size_t length, + size_t* result) { +#ifdef SNAPPY + return snappy::GetUncompressedLength(input, length, result); +#else + return false; +#endif +} + +inline bool Snappy_Uncompress(const char* input, size_t length, + char* output) { +#ifdef SNAPPY + return snappy::RawUncompress(input, length, output); +#else + return false; +#endif +} + +inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { + return false; +} + +} // namespace port +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_PORT_PORT_POSIX_H_ diff --git a/Subtrees/hyperleveldb/port/thread_annotations.h b/Subtrees/hyperleveldb/port/thread_annotations.h new file mode 100644 index 0000000000..00db106904 --- /dev/null +++ b/Subtrees/hyperleveldb/port/thread_annotations.h @@ -0,0 +1,59 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_PORT_THREAD_ANNOTATIONS_H + +// Some environments provide custom macros to aid in static thread-safety +// analysis. Provide empty definitions of such macros unless they are already +// defined. + +#ifndef EXCLUSIVE_LOCKS_REQUIRED +#define EXCLUSIVE_LOCKS_REQUIRED(...) +#endif + +#ifndef SHARED_LOCKS_REQUIRED +#define SHARED_LOCKS_REQUIRED(...) +#endif + +#ifndef LOCKS_EXCLUDED +#define LOCKS_EXCLUDED(...) +#endif + +#ifndef LOCK_RETURNED +#define LOCK_RETURNED(x) +#endif + +#ifndef LOCKABLE +#define LOCKABLE +#endif + +#ifndef SCOPED_LOCKABLE +#define SCOPED_LOCKABLE +#endif + +#ifndef EXCLUSIVE_LOCK_FUNCTION +#define EXCLUSIVE_LOCK_FUNCTION(...) +#endif + +#ifndef SHARED_LOCK_FUNCTION +#define SHARED_LOCK_FUNCTION(...) +#endif + +#ifndef EXCLUSIVE_TRYLOCK_FUNCTION +#define EXCLUSIVE_TRYLOCK_FUNCTION(...) +#endif + +#ifndef SHARED_TRYLOCK_FUNCTION +#define SHARED_TRYLOCK_FUNCTION(...) +#endif + +#ifndef UNLOCK_FUNCTION +#define UNLOCK_FUNCTION(...) +#endif + +#ifndef NO_THREAD_SAFETY_ANALYSIS +#define NO_THREAD_SAFETY_ANALYSIS +#endif + +#endif // STORAGE_HYPERLEVELDB_PORT_THREAD_ANNOTATIONS_H diff --git a/Subtrees/hyperleveldb/port/win/stdint.h b/Subtrees/hyperleveldb/port/win/stdint.h new file mode 100644 index 0000000000..c5204e7f74 --- /dev/null +++ b/Subtrees/hyperleveldb/port/win/stdint.h @@ -0,0 +1,24 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// MSVC didn't ship with this file until the 2010 version. + +#ifndef STORAGE_HYPERLEVELDB_PORT_WIN_STDINT_H_ +#define STORAGE_HYPERLEVELDB_PORT_WIN_STDINT_H_ + +#if !defined(_MSC_VER) +#error This file should only be included when compiling with MSVC. +#endif + +// Define C99 equivalent types. +typedef signed char int8_t; +typedef signed short int16_t; +typedef signed int int32_t; +typedef signed long long int64_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +#endif // STORAGE_HYPERLEVELDB_PORT_WIN_STDINT_H_ diff --git a/Subtrees/hyperleveldb/table/block.cc b/Subtrees/hyperleveldb/table/block.cc new file mode 100644 index 0000000000..4fd08d0ee6 --- /dev/null +++ b/Subtrees/hyperleveldb/table/block.cc @@ -0,0 +1,268 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Decodes the blocks generated by block_builder.cc. + +#include "block.h" + +#include +#include +#include "../hyperleveldb/comparator.h" +#include "format.h" +#include "../util/coding.h" +#include "../util/logging.h" + +namespace hyperleveldb { + +inline uint32_t Block::NumRestarts() const { + assert(size_ >= sizeof(uint32_t)); + return DecodeFixed32(data_ + size_ - sizeof(uint32_t)); +} + +Block::Block(const BlockContents& contents) + : data_(contents.data.data()), + size_(contents.data.size()), + owned_(contents.heap_allocated) { + if (size_ < sizeof(uint32_t)) { + size_ = 0; // Error marker + } else { + size_t max_restarts_allowed = (size_-sizeof(uint32_t)) / sizeof(uint32_t); + if (NumRestarts() > max_restarts_allowed) { + // The size is too small for NumRestarts() + size_ = 0; + } else { + restart_offset_ = size_ - (1 + NumRestarts()) * sizeof(uint32_t); + } + } +} + +Block::~Block() { + if (owned_) { + delete[] data_; + } +} + +// Helper routine: decode the next block entry starting at "p", +// storing the number of shared key bytes, non_shared key bytes, +// and the length of the value in "*shared", "*non_shared", and +// "*value_length", respectively. Will not derefence past "limit". +// +// If any errors are detected, returns NULL. Otherwise, returns a +// pointer to the key delta (just past the three decoded values). +static inline const char* DecodeEntry(const char* p, const char* limit, + uint32_t* shared, + uint32_t* non_shared, + uint32_t* value_length) { + if (limit - p < 3) return NULL; + *shared = reinterpret_cast(p)[0]; + *non_shared = reinterpret_cast(p)[1]; + *value_length = reinterpret_cast(p)[2]; + if ((*shared | *non_shared | *value_length) < 128) { + // Fast path: all three values are encoded in one byte each + p += 3; + } else { + if ((p = GetVarint32Ptr(p, limit, shared)) == NULL) return NULL; + if ((p = GetVarint32Ptr(p, limit, non_shared)) == NULL) return NULL; + if ((p = GetVarint32Ptr(p, limit, value_length)) == NULL) return NULL; + } + + if (static_cast(limit - p) < (*non_shared + *value_length)) { + return NULL; + } + return p; +} + +class Block::Iter : public Iterator { + private: + const Comparator* const comparator_; + const char* const data_; // underlying block contents + uint32_t const restarts_; // Offset of restart array (list of fixed32) + uint32_t const num_restarts_; // Number of uint32_t entries in restart array + + // current_ is offset in data_ of current entry. >= restarts_ if !Valid + uint32_t current_; + uint32_t restart_index_; // Index of restart block in which current_ falls + std::string key_; + Slice value_; + Status status_; + + inline int Compare(const Slice& a, const Slice& b) const { + return comparator_->Compare(a, b); + } + + // Return the offset in data_ just past the end of the current entry. + inline uint32_t NextEntryOffset() const { + return (value_.data() + value_.size()) - data_; + } + + uint32_t GetRestartPoint(uint32_t index) { + assert(index < num_restarts_); + return DecodeFixed32(data_ + restarts_ + index * sizeof(uint32_t)); + } + + void SeekToRestartPoint(uint32_t index) { + key_.clear(); + restart_index_ = index; + // current_ will be fixed by ParseNextKey(); + + // ParseNextKey() starts at the end of value_, so set value_ accordingly + uint32_t offset = GetRestartPoint(index); + value_ = Slice(data_ + offset, 0); + } + + public: + Iter(const Comparator* comparator, + const char* data, + uint32_t restarts, + uint32_t num_restarts) + : comparator_(comparator), + data_(data), + restarts_(restarts), + num_restarts_(num_restarts), + current_(restarts_), + restart_index_(num_restarts_) { + assert(num_restarts_ > 0); + } + + virtual bool Valid() const { return current_ < restarts_; } + virtual Status status() const { return status_; } + virtual Slice key() const { + assert(Valid()); + return key_; + } + virtual Slice value() const { + assert(Valid()); + return value_; + } + + virtual void Next() { + assert(Valid()); + ParseNextKey(); + } + + virtual void Prev() { + assert(Valid()); + + // Scan backwards to a restart point before current_ + const uint32_t original = current_; + while (GetRestartPoint(restart_index_) >= original) { + if (restart_index_ == 0) { + // No more entries + current_ = restarts_; + restart_index_ = num_restarts_; + return; + } + restart_index_--; + } + + SeekToRestartPoint(restart_index_); + do { + // Loop until end of current entry hits the start of original entry + } while (ParseNextKey() && NextEntryOffset() < original); + } + + virtual void Seek(const Slice& target) { + // Binary search in restart array to find the last restart point + // with a key < target + uint32_t left = 0; + uint32_t right = num_restarts_ - 1; + while (left < right) { + uint32_t mid = (left + right + 1) / 2; + uint32_t region_offset = GetRestartPoint(mid); + uint32_t shared, non_shared, value_length; + const char* key_ptr = DecodeEntry(data_ + region_offset, + data_ + restarts_, + &shared, &non_shared, &value_length); + if (key_ptr == NULL || (shared != 0)) { + CorruptionError(); + return; + } + Slice mid_key(key_ptr, non_shared); + if (Compare(mid_key, target) < 0) { + // Key at "mid" is smaller than "target". Therefore all + // blocks before "mid" are uninteresting. + left = mid; + } else { + // Key at "mid" is >= "target". Therefore all blocks at or + // after "mid" are uninteresting. + right = mid - 1; + } + } + + // Linear search (within restart block) for first key >= target + SeekToRestartPoint(left); + while (true) { + if (!ParseNextKey()) { + return; + } + if (Compare(key_, target) >= 0) { + return; + } + } + } + + virtual void SeekToFirst() { + SeekToRestartPoint(0); + ParseNextKey(); + } + + virtual void SeekToLast() { + SeekToRestartPoint(num_restarts_ - 1); + while (ParseNextKey() && NextEntryOffset() < restarts_) { + // Keep skipping + } + } + + private: + void CorruptionError() { + current_ = restarts_; + restart_index_ = num_restarts_; + status_ = Status::Corruption("bad entry in block"); + key_.clear(); + value_.clear(); + } + + bool ParseNextKey() { + current_ = NextEntryOffset(); + const char* p = data_ + current_; + const char* limit = data_ + restarts_; // Restarts come right after data + if (p >= limit) { + // No more entries to return. Mark as invalid. + current_ = restarts_; + restart_index_ = num_restarts_; + return false; + } + + // Decode next entry + uint32_t shared, non_shared, value_length; + p = DecodeEntry(p, limit, &shared, &non_shared, &value_length); + if (p == NULL || key_.size() < shared) { + CorruptionError(); + return false; + } else { + key_.resize(shared); + key_.append(p, non_shared); + value_ = Slice(p + non_shared, value_length); + while (restart_index_ + 1 < num_restarts_ && + GetRestartPoint(restart_index_ + 1) < current_) { + ++restart_index_; + } + return true; + } + } +}; + +Iterator* Block::NewIterator(const Comparator* cmp) { + if (size_ < sizeof(uint32_t)) { + return NewErrorIterator(Status::Corruption("bad block contents")); + } + const uint32_t num_restarts = NumRestarts(); + if (num_restarts == 0) { + return NewEmptyIterator(); + } else { + return new Iter(cmp, data_, restart_offset_, num_restarts); + } +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/table/block.h b/Subtrees/hyperleveldb/table/block.h new file mode 100644 index 0000000000..26e3f9d1a7 --- /dev/null +++ b/Subtrees/hyperleveldb/table/block.h @@ -0,0 +1,44 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_TABLE_BLOCK_H_ +#define STORAGE_HYPERLEVELDB_TABLE_BLOCK_H_ + +#include +#include +#include "../hyperleveldb/iterator.h" + +namespace hyperleveldb { + +struct BlockContents; +class Comparator; + +class Block { + public: + // Initialize the block with the specified contents. + explicit Block(const BlockContents& contents); + + ~Block(); + + size_t size() const { return size_; } + Iterator* NewIterator(const Comparator* comparator); + + private: + uint32_t NumRestarts() const; + + const char* data_; + size_t size_; + uint32_t restart_offset_; // Offset in data_ of restart array + bool owned_; // Block owns data_[] + + // No copying allowed + Block(const Block&); + void operator=(const Block&); + + class Iter; +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_TABLE_BLOCK_H_ diff --git a/Subtrees/hyperleveldb/table/block_builder.cc b/Subtrees/hyperleveldb/table/block_builder.cc new file mode 100644 index 0000000000..1e76b7c768 --- /dev/null +++ b/Subtrees/hyperleveldb/table/block_builder.cc @@ -0,0 +1,109 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// BlockBuilder generates blocks where keys are prefix-compressed: +// +// When we store a key, we drop the prefix shared with the previous +// string. This helps reduce the space requirement significantly. +// Furthermore, once every K keys, we do not apply the prefix +// compression and store the entire key. We call this a "restart +// point". The tail end of the block stores the offsets of all of the +// restart points, and can be used to do a binary search when looking +// for a particular key. Values are stored as-is (without compression) +// immediately following the corresponding key. +// +// An entry for a particular key-value pair has the form: +// shared_bytes: varint32 +// unshared_bytes: varint32 +// value_length: varint32 +// key_delta: char[unshared_bytes] +// value: char[value_length] +// shared_bytes == 0 for restart points. +// +// The trailer of the block has the form: +// restarts: uint32[num_restarts] +// num_restarts: uint32 +// restarts[i] contains the offset within the block of the ith restart point. + +#include "block_builder.h" + +#include +#include +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/table_builder.h" +#include "../util/coding.h" + +namespace hyperleveldb { + +BlockBuilder::BlockBuilder(const Options* options) + : options_(options), + restarts_(), + counter_(0), + finished_(false) { + assert(options->block_restart_interval >= 1); + restarts_.push_back(0); // First restart point is at offset 0 +} + +void BlockBuilder::Reset() { + buffer_.clear(); + restarts_.clear(); + restarts_.push_back(0); // First restart point is at offset 0 + counter_ = 0; + finished_ = false; + last_key_.clear(); +} + +size_t BlockBuilder::CurrentSizeEstimate() const { + return (buffer_.size() + // Raw data buffer + restarts_.size() * sizeof(uint32_t) + // Restart array + sizeof(uint32_t)); // Restart array length +} + +Slice BlockBuilder::Finish() { + // Append restart array + for (size_t i = 0; i < restarts_.size(); i++) { + PutFixed32(&buffer_, restarts_[i]); + } + PutFixed32(&buffer_, restarts_.size()); + finished_ = true; + return Slice(buffer_); +} + +void BlockBuilder::Add(const Slice& key, const Slice& value) { + Slice last_key_piece(last_key_); + assert(!finished_); + assert(counter_ <= options_->block_restart_interval); + assert(buffer_.empty() // No values yet? + || options_->comparator->Compare(key, last_key_piece) > 0); + size_t shared = 0; + if (counter_ < options_->block_restart_interval) { + // See how much sharing to do with previous string + const size_t min_length = std::min(last_key_piece.size(), key.size()); + while ((shared < min_length) && (last_key_piece[shared] == key[shared])) { + shared++; + } + } else { + // Restart compression + restarts_.push_back(buffer_.size()); + counter_ = 0; + } + const size_t non_shared = key.size() - shared; + + // Add "" to buffer_ + PutVarint32(&buffer_, shared); + PutVarint32(&buffer_, non_shared); + PutVarint32(&buffer_, value.size()); + + // Add string delta to buffer_ followed by value + buffer_.append(key.data() + shared, non_shared); + buffer_.append(value.data(), value.size()); + + // Update state + last_key_.resize(shared); + last_key_.append(key.data() + shared, non_shared); + assert(Slice(last_key_) == key); + counter_++; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/table/block_builder.h b/Subtrees/hyperleveldb/table/block_builder.h new file mode 100644 index 0000000000..c5216d71fd --- /dev/null +++ b/Subtrees/hyperleveldb/table/block_builder.h @@ -0,0 +1,57 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_TABLE_BLOCK_BUILDER_H_ +#define STORAGE_HYPERLEVELDB_TABLE_BLOCK_BUILDER_H_ + +#include + +#include +#include "../hyperleveldb/slice.h" + +namespace hyperleveldb { + +struct Options; + +class BlockBuilder { + public: + explicit BlockBuilder(const Options* options); + + // Reset the contents as if the BlockBuilder was just constructed. + void Reset(); + + // REQUIRES: Finish() has not been callled since the last call to Reset(). + // REQUIRES: key is larger than any previously added key + void Add(const Slice& key, const Slice& value); + + // Finish building the block and return a slice that refers to the + // block contents. The returned slice will remain valid for the + // lifetime of this builder or until Reset() is called. + Slice Finish(); + + // Returns an estimate of the current (uncompressed) size of the block + // we are building. + size_t CurrentSizeEstimate() const; + + // Return true iff no entries have been added since the last Reset() + bool empty() const { + return buffer_.empty(); + } + + private: + const Options* options_; + std::string buffer_; // Destination buffer + std::vector restarts_; // Restart points + int counter_; // Number of entries emitted since restart + bool finished_; // Has Finish() been called? + std::string last_key_; + + // No copying allowed + BlockBuilder(const BlockBuilder&); + void operator=(const BlockBuilder&); +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_TABLE_BLOCK_BUILDER_H_ diff --git a/Subtrees/hyperleveldb/table/filter_block.cc b/Subtrees/hyperleveldb/table/filter_block.cc new file mode 100644 index 0000000000..f74b9545a4 --- /dev/null +++ b/Subtrees/hyperleveldb/table/filter_block.cc @@ -0,0 +1,111 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "filter_block.h" + +#include "../hyperleveldb/filter_policy.h" +#include "../util/coding.h" + +namespace hyperleveldb { + +// See doc/table_format.txt for an explanation of the filter block format. + +// Generate new filter every 2KB of data +static const size_t kFilterBaseLg = 11; +static const size_t kFilterBase = 1 << kFilterBaseLg; + +FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy) + : policy_(policy) { +} + +void FilterBlockBuilder::StartBlock(uint64_t block_offset) { + uint64_t filter_index = (block_offset / kFilterBase); + assert(filter_index >= filter_offsets_.size()); + while (filter_index > filter_offsets_.size()) { + GenerateFilter(); + } +} + +void FilterBlockBuilder::AddKey(const Slice& key) { + Slice k = key; + start_.push_back(keys_.size()); + keys_.append(k.data(), k.size()); +} + +Slice FilterBlockBuilder::Finish() { + if (!start_.empty()) { + GenerateFilter(); + } + + // Append array of per-filter offsets + const uint32_t array_offset = result_.size(); + for (size_t i = 0; i < filter_offsets_.size(); i++) { + PutFixed32(&result_, filter_offsets_[i]); + } + + PutFixed32(&result_, array_offset); + result_.push_back(kFilterBaseLg); // Save encoding parameter in result + return Slice(result_); +} + +void FilterBlockBuilder::GenerateFilter() { + const size_t num_keys = start_.size(); + if (num_keys == 0) { + // Fast path if there are no keys for this filter + filter_offsets_.push_back(result_.size()); + return; + } + + // Make list of keys from flattened key structure + start_.push_back(keys_.size()); // Simplify length computation + tmp_keys_.resize(num_keys); + for (size_t i = 0; i < num_keys; i++) { + const char* base = keys_.data() + start_[i]; + size_t length = start_[i+1] - start_[i]; + tmp_keys_[i] = Slice(base, length); + } + + // Generate filter for current set of keys and append to result_. + filter_offsets_.push_back(result_.size()); + policy_->CreateFilter(&tmp_keys_[0], num_keys, &result_); + + tmp_keys_.clear(); + keys_.clear(); + start_.clear(); +} + +FilterBlockReader::FilterBlockReader(const FilterPolicy* policy, + const Slice& contents) + : policy_(policy), + data_(NULL), + offset_(NULL), + num_(0), + base_lg_(0) { + size_t n = contents.size(); + if (n < 5) return; // 1 byte for base_lg_ and 4 for start of offset array + base_lg_ = contents[n-1]; + uint32_t last_word = DecodeFixed32(contents.data() + n - 5); + if (last_word > n - 5) return; + data_ = contents.data(); + offset_ = data_ + last_word; + num_ = (n - 5 - last_word) / 4; +} + +bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) { + uint64_t index = block_offset >> base_lg_; + if (index < num_) { + uint32_t start = DecodeFixed32(offset_ + index*4); + uint32_t limit = DecodeFixed32(offset_ + index*4 + 4); + if (start <= limit && limit <= (offset_ - data_)) { + Slice filter = Slice(data_ + start, limit - start); + return policy_->KeyMayMatch(key, filter); + } else if (start == limit) { + // Empty filters do not match any keys + return false; + } + } + return true; // Errors are treated as potential matches +} + +} diff --git a/Subtrees/hyperleveldb/table/filter_block.h b/Subtrees/hyperleveldb/table/filter_block.h new file mode 100644 index 0000000000..131cb13cfb --- /dev/null +++ b/Subtrees/hyperleveldb/table/filter_block.h @@ -0,0 +1,68 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A filter block is stored near the end of a Table file. It contains +// filters (e.g., bloom filters) for all data blocks in the table combined +// into a single filter block. + +#ifndef STORAGE_HYPERLEVELDB_TABLE_FILTER_BLOCK_H_ +#define STORAGE_HYPERLEVELDB_TABLE_FILTER_BLOCK_H_ + +#include +#include +#include +#include +#include "../hyperleveldb/slice.h" +#include "../util/hash.h" + +namespace hyperleveldb { + +class FilterPolicy; + +// A FilterBlockBuilder is used to construct all of the filters for a +// particular Table. It generates a single string which is stored as +// a special block in the Table. +// +// The sequence of calls to FilterBlockBuilder must match the regexp: +// (StartBlock AddKey*)* Finish +class FilterBlockBuilder { + public: + explicit FilterBlockBuilder(const FilterPolicy*); + + void StartBlock(uint64_t block_offset); + void AddKey(const Slice& key); + Slice Finish(); + + private: + void GenerateFilter(); + + const FilterPolicy* policy_; + std::string keys_; // Flattened key contents + std::vector start_; // Starting index in keys_ of each key + std::string result_; // Filter data computed so far + std::vector tmp_keys_; // policy_->CreateFilter() argument + std::vector filter_offsets_; + + // No copying allowed + FilterBlockBuilder(const FilterBlockBuilder&); + void operator=(const FilterBlockBuilder&); +}; + +class FilterBlockReader { + public: + // REQUIRES: "contents" and *policy must stay live while *this is live. + FilterBlockReader(const FilterPolicy* policy, const Slice& contents); + bool KeyMayMatch(uint64_t block_offset, const Slice& key); + + private: + const FilterPolicy* policy_; + const char* data_; // Pointer to filter data (at block-start) + const char* offset_; // Pointer to beginning of offset array (at block-end) + size_t num_; // Number of entries in offset array + size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file) +}; + +} + +#endif // STORAGE_HYPERLEVELDB_TABLE_FILTER_BLOCK_H_ diff --git a/Subtrees/hyperleveldb/table/filter_block_test.cc b/Subtrees/hyperleveldb/table/filter_block_test.cc new file mode 100644 index 0000000000..901f7603ed --- /dev/null +++ b/Subtrees/hyperleveldb/table/filter_block_test.cc @@ -0,0 +1,128 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../table/filter_block.h" + +#include "../hyperleveldb/filter_policy.h" +#include "coding.h" +#include "hash.h" +#include "logging.h" +#include "testharness.h" +#include "testutil.h" + +namespace hyperleveldb { + +// For testing: emit an array with one hash value per key +class TestHashFilter : public FilterPolicy { + public: + virtual const char* Name() const { + return "TestHashFilter"; + } + + virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { + for (int i = 0; i < n; i++) { + uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); + PutFixed32(dst, h); + } + } + + virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { + uint32_t h = Hash(key.data(), key.size(), 1); + for (int i = 0; i + 4 <= filter.size(); i += 4) { + if (h == DecodeFixed32(filter.data() + i)) { + return true; + } + } + return false; + } +}; + +class FilterBlockTest { + public: + TestHashFilter policy_; +}; + +TEST(FilterBlockTest, EmptyBuilder) { + FilterBlockBuilder builder(&policy_); + Slice block = builder.Finish(); + ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); + FilterBlockReader reader(&policy_, block); + ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); + ASSERT_TRUE(reader.KeyMayMatch(100000, "foo")); +} + +TEST(FilterBlockTest, SingleChunk) { + FilterBlockBuilder builder(&policy_); + builder.StartBlock(100); + builder.AddKey("foo"); + builder.AddKey("bar"); + builder.AddKey("box"); + builder.StartBlock(200); + builder.AddKey("box"); + builder.StartBlock(300); + builder.AddKey("hello"); + Slice block = builder.Finish(); + FilterBlockReader reader(&policy_, block); + ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); + ASSERT_TRUE(reader.KeyMayMatch(100, "bar")); + ASSERT_TRUE(reader.KeyMayMatch(100, "box")); + ASSERT_TRUE(reader.KeyMayMatch(100, "hello")); + ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); + ASSERT_TRUE(! reader.KeyMayMatch(100, "missing")); + ASSERT_TRUE(! reader.KeyMayMatch(100, "other")); +} + +TEST(FilterBlockTest, MultiChunk) { + FilterBlockBuilder builder(&policy_); + + // First filter + builder.StartBlock(0); + builder.AddKey("foo"); + builder.StartBlock(2000); + builder.AddKey("bar"); + + // Second filter + builder.StartBlock(3100); + builder.AddKey("box"); + + // Third filter is empty + + // Last filter + builder.StartBlock(9000); + builder.AddKey("box"); + builder.AddKey("hello"); + + Slice block = builder.Finish(); + FilterBlockReader reader(&policy_, block); + + // Check first filter + ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); + ASSERT_TRUE(reader.KeyMayMatch(2000, "bar")); + ASSERT_TRUE(! reader.KeyMayMatch(0, "box")); + ASSERT_TRUE(! reader.KeyMayMatch(0, "hello")); + + // Check second filter + ASSERT_TRUE(reader.KeyMayMatch(3100, "box")); + ASSERT_TRUE(! reader.KeyMayMatch(3100, "foo")); + ASSERT_TRUE(! reader.KeyMayMatch(3100, "bar")); + ASSERT_TRUE(! reader.KeyMayMatch(3100, "hello")); + + // Check third filter (empty) + ASSERT_TRUE(! reader.KeyMayMatch(4100, "foo")); + ASSERT_TRUE(! reader.KeyMayMatch(4100, "bar")); + ASSERT_TRUE(! reader.KeyMayMatch(4100, "box")); + ASSERT_TRUE(! reader.KeyMayMatch(4100, "hello")); + + // Check last filter + ASSERT_TRUE(reader.KeyMayMatch(9000, "box")); + ASSERT_TRUE(reader.KeyMayMatch(9000, "hello")); + ASSERT_TRUE(! reader.KeyMayMatch(9000, "foo")); + ASSERT_TRUE(! reader.KeyMayMatch(9000, "bar")); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/table/format.cc b/Subtrees/hyperleveldb/table/format.cc new file mode 100644 index 0000000000..9b365c05f3 --- /dev/null +++ b/Subtrees/hyperleveldb/table/format.cc @@ -0,0 +1,145 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "format.h" + +#include "../hyperleveldb/env.h" +#include "../port/port.h" +#include "block.h" +#include "../util/coding.h" +#include "../util/crc32c.h" + +namespace hyperleveldb { + +void BlockHandle::EncodeTo(std::string* dst) const { + // Sanity check that all fields have been set + assert(offset_ != ~static_cast(0)); + assert(size_ != ~static_cast(0)); + PutVarint64(dst, offset_); + PutVarint64(dst, size_); +} + +Status BlockHandle::DecodeFrom(Slice* input) { + if (GetVarint64(input, &offset_) && + GetVarint64(input, &size_)) { + return Status::OK(); + } else { + return Status::Corruption("bad block handle"); + } +} + +void Footer::EncodeTo(std::string* dst) const { +#ifndef NDEBUG + const size_t original_size = dst->size(); +#endif + metaindex_handle_.EncodeTo(dst); + index_handle_.EncodeTo(dst); + dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding + PutFixed32(dst, static_cast(kTableMagicNumber & 0xffffffffu)); + PutFixed32(dst, static_cast(kTableMagicNumber >> 32)); + assert(dst->size() == original_size + kEncodedLength); +} + +Status Footer::DecodeFrom(Slice* input) { + const char* magic_ptr = input->data() + kEncodedLength - 8; + const uint32_t magic_lo = DecodeFixed32(magic_ptr); + const uint32_t magic_hi = DecodeFixed32(magic_ptr + 4); + const uint64_t magic = ((static_cast(magic_hi) << 32) | + (static_cast(magic_lo))); + if (magic != kTableMagicNumber) { + return Status::InvalidArgument("not an sstable (bad magic number)"); + } + + Status result = metaindex_handle_.DecodeFrom(input); + if (result.ok()) { + result = index_handle_.DecodeFrom(input); + } + if (result.ok()) { + // We skip over any leftover data (just padding for now) in "input" + const char* end = magic_ptr + 8; + *input = Slice(end, input->data() + input->size() - end); + } + return result; +} + +Status ReadBlock(RandomAccessFile* file, + const ReadOptions& options, + const BlockHandle& handle, + BlockContents* result) { + result->data = Slice(); + result->cachable = false; + result->heap_allocated = false; + + // Read the block contents as well as the type/crc footer. + // See table_builder.cc for the code that built this structure. + size_t n = static_cast(handle.size()); + char* buf = new char[n + kBlockTrailerSize]; + Slice contents; + Status s = file->Read(handle.offset(), n + kBlockTrailerSize, &contents, buf); + if (!s.ok()) { + delete[] buf; + return s; + } + if (contents.size() != n + kBlockTrailerSize) { + delete[] buf; + return Status::Corruption("truncated block read"); + } + + // Check the crc of the type and the block contents + const char* data = contents.data(); // Pointer to where Read put the data + if (options.verify_checksums) { + const uint32_t crc = crc32c::Unmask(DecodeFixed32(data + n + 1)); + const uint32_t actual = crc32c::Value(data, n + 1); + if (actual != crc) { + delete[] buf; + s = Status::Corruption("block checksum mismatch"); + return s; + } + } + + switch (data[n]) { + case kNoCompression: + if (data != buf) { + // File implementation gave us pointer to some other data. + // Use it directly under the assumption that it will be live + // while the file is open. + delete[] buf; + result->data = Slice(data, n); + result->heap_allocated = false; + result->cachable = false; // Do not double-cache + } else { + result->data = Slice(buf, n); + result->heap_allocated = true; + result->cachable = true; + } + + // Ok + break; + case kSnappyCompression: { + size_t ulength = 0; + if (!port::Snappy_GetUncompressedLength(data, n, &ulength)) { + delete[] buf; + return Status::Corruption("corrupted compressed block contents"); + } + char* ubuf = new char[ulength]; + if (!port::Snappy_Uncompress(data, n, ubuf)) { + delete[] buf; + delete[] ubuf; + return Status::Corruption("corrupted compressed block contents"); + } + delete[] buf; + result->data = Slice(ubuf, ulength); + result->heap_allocated = true; + result->cachable = true; + break; + } + default: + delete[] buf; + return Status::Corruption("bad block type"); + } + + return Status::OK(); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/table/format.h b/Subtrees/hyperleveldb/table/format.h new file mode 100644 index 0000000000..20a5103a9f --- /dev/null +++ b/Subtrees/hyperleveldb/table/format.h @@ -0,0 +1,108 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_TABLE_FORMAT_H_ +#define STORAGE_HYPERLEVELDB_TABLE_FORMAT_H_ + +#include +#include +#include "../hyperleveldb/slice.h" +#include "../hyperleveldb/status.h" +#include "../hyperleveldb/table_builder.h" + +namespace hyperleveldb { + +class Block; +class RandomAccessFile; +struct ReadOptions; + +// BlockHandle is a pointer to the extent of a file that stores a data +// block or a meta block. +class BlockHandle { + public: + BlockHandle(); + + // The offset of the block in the file. + uint64_t offset() const { return offset_; } + void set_offset(uint64_t offset) { offset_ = offset; } + + // The size of the stored block + uint64_t size() const { return size_; } + void set_size(uint64_t size) { size_ = size; } + + void EncodeTo(std::string* dst) const; + Status DecodeFrom(Slice* input); + + // Maximum encoding length of a BlockHandle + enum { kMaxEncodedLength = 10 + 10 }; + + private: + uint64_t offset_; + uint64_t size_; +}; + +// Footer encapsulates the fixed information stored at the tail +// end of every table file. +class Footer { + public: + Footer() { } + + // The block handle for the metaindex block of the table + const BlockHandle& metaindex_handle() const { return metaindex_handle_; } + void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; } + + // The block handle for the index block of the table + const BlockHandle& index_handle() const { + return index_handle_; + } + void set_index_handle(const BlockHandle& h) { + index_handle_ = h; + } + + void EncodeTo(std::string* dst) const; + Status DecodeFrom(Slice* input); + + // Encoded length of a Footer. Note that the serialization of a + // Footer will always occupy exactly this many bytes. It consists + // of two block handles and a magic number. + enum { + kEncodedLength = 2*BlockHandle::kMaxEncodedLength + 8 + }; + + private: + BlockHandle metaindex_handle_; + BlockHandle index_handle_; +}; + +// kTableMagicNumber was picked by running +// echo http://code.google.com/p/leveldb/ | sha1sum +// and taking the leading 64 bits. +static const uint64_t kTableMagicNumber = 0xdb4775248b80fb57ull; + +// 1-byte type + 32-bit crc +static const size_t kBlockTrailerSize = 5; + +struct BlockContents { + Slice data; // Actual contents of data + bool cachable; // True iff data can be cached + bool heap_allocated; // True iff caller should delete[] data.data() +}; + +// Read the block identified by "handle" from "file". On failure +// return non-OK. On success fill *result and return OK. +extern Status ReadBlock(RandomAccessFile* file, + const ReadOptions& options, + const BlockHandle& handle, + BlockContents* result); + +// Implementation details follow. Clients should ignore, + +inline BlockHandle::BlockHandle() + : offset_(~static_cast(0)), + size_(~static_cast(0)) { +} + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_TABLE_FORMAT_H_ diff --git a/Subtrees/hyperleveldb/table/iterator.cc b/Subtrees/hyperleveldb/table/iterator.cc new file mode 100644 index 0000000000..087e6e4fdb --- /dev/null +++ b/Subtrees/hyperleveldb/table/iterator.cc @@ -0,0 +1,67 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/iterator.h" + +namespace hyperleveldb { + +Iterator::Iterator() { + cleanup_.function = NULL; + cleanup_.next = NULL; +} + +Iterator::~Iterator() { + if (cleanup_.function != NULL) { + (*cleanup_.function)(cleanup_.arg1, cleanup_.arg2); + for (Cleanup* c = cleanup_.next; c != NULL; ) { + (*c->function)(c->arg1, c->arg2); + Cleanup* next = c->next; + delete c; + c = next; + } + } +} + +void Iterator::RegisterCleanup(CleanupFunction func, void* arg1, void* arg2) { + assert(func != NULL); + Cleanup* c; + if (cleanup_.function == NULL) { + c = &cleanup_; + } else { + c = new Cleanup; + c->next = cleanup_.next; + cleanup_.next = c; + } + c->function = func; + c->arg1 = arg1; + c->arg2 = arg2; +} + +namespace { +class EmptyIterator : public Iterator { + public: + EmptyIterator(const Status& s) : status_(s) { } + virtual bool Valid() const { return false; } + virtual void Seek(const Slice& target) { } + virtual void SeekToFirst() { } + virtual void SeekToLast() { } + virtual void Next() { assert(false); } + virtual void Prev() { assert(false); } + Slice key() const { assert(false); return Slice(); } + Slice value() const { assert(false); return Slice(); } + virtual Status status() const { return status_; } + private: + Status status_; +}; +} // namespace + +Iterator* NewEmptyIterator() { + return new EmptyIterator(Status::OK()); +} + +Iterator* NewErrorIterator(const Status& status) { + return new EmptyIterator(status); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/table/iterator_wrapper.h b/Subtrees/hyperleveldb/table/iterator_wrapper.h new file mode 100644 index 0000000000..e2f03c0e07 --- /dev/null +++ b/Subtrees/hyperleveldb/table/iterator_wrapper.h @@ -0,0 +1,63 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_TABLE_ITERATOR_WRAPPER_H_ +#define STORAGE_HYPERLEVELDB_TABLE_ITERATOR_WRAPPER_H_ + +namespace hyperleveldb { + +// A internal wrapper class with an interface similar to Iterator that +// caches the valid() and key() results for an underlying iterator. +// This can help avoid virtual function calls and also gives better +// cache locality. +class IteratorWrapper { + public: + IteratorWrapper(): iter_(NULL), valid_(false) { } + explicit IteratorWrapper(Iterator* iter): iter_(NULL) { + Set(iter); + } + ~IteratorWrapper() { delete iter_; } + Iterator* iter() const { return iter_; } + + // Takes ownership of "iter" and will delete it when destroyed, or + // when Set() is invoked again. + void Set(Iterator* iter) { + delete iter_; + iter_ = iter; + if (iter_ == NULL) { + valid_ = false; + } else { + Update(); + } + } + + + // Iterator interface methods + bool Valid() const { return valid_; } + Slice key() const { assert(Valid()); return key_; } + Slice value() const { assert(Valid()); return iter_->value(); } + // Methods below require iter() != NULL + Status status() const { assert(iter_); return iter_->status(); } + void Next() { assert(iter_); iter_->Next(); Update(); } + void Prev() { assert(iter_); iter_->Prev(); Update(); } + void Seek(const Slice& k) { assert(iter_); iter_->Seek(k); Update(); } + void SeekToFirst() { assert(iter_); iter_->SeekToFirst(); Update(); } + void SeekToLast() { assert(iter_); iter_->SeekToLast(); Update(); } + + private: + void Update() { + valid_ = iter_->Valid(); + if (valid_) { + key_ = iter_->key(); + } + } + + Iterator* iter_; + bool valid_; + Slice key_; +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_TABLE_ITERATOR_WRAPPER_H_ diff --git a/Subtrees/hyperleveldb/table/merger.cc b/Subtrees/hyperleveldb/table/merger.cc new file mode 100644 index 0000000000..31c1577307 --- /dev/null +++ b/Subtrees/hyperleveldb/table/merger.cc @@ -0,0 +1,197 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "merger.h" + +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/iterator.h" +#include "iterator_wrapper.h" + +namespace hyperleveldb { + +namespace { +class MergingIterator : public Iterator { + public: + MergingIterator(const Comparator* comparator, Iterator** children, int n) + : comparator_(comparator), + children_(new IteratorWrapper[n]), + n_(n), + current_(NULL), + direction_(kForward) { + for (int i = 0; i < n; i++) { + children_[i].Set(children[i]); + } + } + + virtual ~MergingIterator() { + delete[] children_; + } + + virtual bool Valid() const { + return (current_ != NULL); + } + + virtual void SeekToFirst() { + for (int i = 0; i < n_; i++) { + children_[i].SeekToFirst(); + } + FindSmallest(); + direction_ = kForward; + } + + virtual void SeekToLast() { + for (int i = 0; i < n_; i++) { + children_[i].SeekToLast(); + } + FindLargest(); + direction_ = kReverse; + } + + virtual void Seek(const Slice& target) { + for (int i = 0; i < n_; i++) { + children_[i].Seek(target); + } + FindSmallest(); + direction_ = kForward; + } + + virtual void Next() { + assert(Valid()); + + // Ensure that all children are positioned after key(). + // If we are moving in the forward direction, it is already + // true for all of the non-current_ children since current_ is + // the smallest child and key() == current_->key(). Otherwise, + // we explicitly position the non-current_ children. + if (direction_ != kForward) { + for (int i = 0; i < n_; i++) { + IteratorWrapper* child = &children_[i]; + if (child != current_) { + child->Seek(key()); + if (child->Valid() && + comparator_->Compare(key(), child->key()) == 0) { + child->Next(); + } + } + } + direction_ = kForward; + } + + current_->Next(); + FindSmallest(); + } + + virtual void Prev() { + assert(Valid()); + + // Ensure that all children are positioned before key(). + // If we are moving in the reverse direction, it is already + // true for all of the non-current_ children since current_ is + // the largest child and key() == current_->key(). Otherwise, + // we explicitly position the non-current_ children. + if (direction_ != kReverse) { + for (int i = 0; i < n_; i++) { + IteratorWrapper* child = &children_[i]; + if (child != current_) { + child->Seek(key()); + if (child->Valid()) { + // Child is at first entry >= key(). Step back one to be < key() + child->Prev(); + } else { + // Child has no entries >= key(). Position at last entry. + child->SeekToLast(); + } + } + } + direction_ = kReverse; + } + + current_->Prev(); + FindLargest(); + } + + virtual Slice key() const { + assert(Valid()); + return current_->key(); + } + + virtual Slice value() const { + assert(Valid()); + return current_->value(); + } + + virtual Status status() const { + Status status; + for (int i = 0; i < n_; i++) { + status = children_[i].status(); + if (!status.ok()) { + break; + } + } + return status; + } + + private: + void FindSmallest(); + void FindLargest(); + + // We might want to use a heap in case there are lots of children. + // For now we use a simple array since we expect a very small number + // of children in leveldb. + const Comparator* comparator_; + IteratorWrapper* children_; + int n_; + IteratorWrapper* current_; + + // Which direction is the iterator moving? + enum Direction { + kForward, + kReverse + }; + Direction direction_; +}; + +void MergingIterator::FindSmallest() { + IteratorWrapper* smallest = NULL; + for (int i = 0; i < n_; i++) { + IteratorWrapper* child = &children_[i]; + if (child->Valid()) { + if (smallest == NULL) { + smallest = child; + } else if (comparator_->Compare(child->key(), smallest->key()) < 0) { + smallest = child; + } + } + } + current_ = smallest; +} + +void MergingIterator::FindLargest() { + IteratorWrapper* largest = NULL; + for (int i = n_-1; i >= 0; i--) { + IteratorWrapper* child = &children_[i]; + if (child->Valid()) { + if (largest == NULL) { + largest = child; + } else if (comparator_->Compare(child->key(), largest->key()) > 0) { + largest = child; + } + } + } + current_ = largest; +} +} // namespace + +Iterator* NewMergingIterator(const Comparator* cmp, Iterator** list, int n) { + assert(n >= 0); + if (n == 0) { + return NewEmptyIterator(); + } else if (n == 1) { + return list[0]; + } else { + return new MergingIterator(cmp, list, n); + } +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/table/merger.h b/Subtrees/hyperleveldb/table/merger.h new file mode 100644 index 0000000000..222bd45c8f --- /dev/null +++ b/Subtrees/hyperleveldb/table/merger.h @@ -0,0 +1,26 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_TABLE_MERGER_H_ +#define STORAGE_HYPERLEVELDB_TABLE_MERGER_H_ + +namespace hyperleveldb { + +class Comparator; +class Iterator; + +// Return an iterator that provided the union of the data in +// children[0,n-1]. Takes ownership of the child iterators and +// will delete them when the result iterator is deleted. +// +// The result does no duplicate suppression. I.e., if a particular +// key is present in K child iterators, it will be yielded K times. +// +// REQUIRES: n >= 0 +extern Iterator* NewMergingIterator( + const Comparator* comparator, Iterator** children, int n); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_TABLE_MERGER_H_ diff --git a/Subtrees/hyperleveldb/table/table.cc b/Subtrees/hyperleveldb/table/table.cc new file mode 100644 index 0000000000..bd56a09673 --- /dev/null +++ b/Subtrees/hyperleveldb/table/table.cc @@ -0,0 +1,275 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/table.h" + +#include "../hyperleveldb/cache.h" +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/filter_policy.h" +#include "../hyperleveldb/options.h" +#include "block.h" +#include "filter_block.h" +#include "format.h" +#include "two_level_iterator.h" +#include "../util/coding.h" + +namespace hyperleveldb { + +struct Table::Rep { + ~Rep() { + delete filter; + delete [] filter_data; + delete index_block; + } + + Options options; + Status status; + RandomAccessFile* file; + uint64_t cache_id; + FilterBlockReader* filter; + const char* filter_data; + + BlockHandle metaindex_handle; // Handle to metaindex_block: saved from footer + Block* index_block; +}; + +Status Table::Open(const Options& options, + RandomAccessFile* file, + uint64_t size, + Table** table) { + *table = NULL; + if (size < Footer::kEncodedLength) { + return Status::InvalidArgument("file is too short to be an sstable"); + } + + char footer_space[Footer::kEncodedLength]; + Slice footer_input; + Status s = file->Read(size - Footer::kEncodedLength, Footer::kEncodedLength, + &footer_input, footer_space); + if (!s.ok()) return s; + + Footer footer; + s = footer.DecodeFrom(&footer_input); + if (!s.ok()) return s; + + // Read the index block + BlockContents contents; + Block* index_block = NULL; + if (s.ok()) { + s = ReadBlock(file, ReadOptions(), footer.index_handle(), &contents); + if (s.ok()) { + index_block = new Block(contents); + } + } + + if (s.ok()) { + // We've successfully read the footer and the index block: we're + // ready to serve requests. + Rep* rep = new Table::Rep; + rep->options = options; + rep->file = file; + rep->metaindex_handle = footer.metaindex_handle(); + rep->index_block = index_block; + rep->cache_id = (options.block_cache ? options.block_cache->NewId() : 0); + rep->filter_data = NULL; + rep->filter = NULL; + *table = new Table(rep); + (*table)->ReadMeta(footer); + } else { + if (index_block) delete index_block; + } + + return s; +} + +void Table::ReadMeta(const Footer& footer) { + if (rep_->options.filter_policy == NULL) { + return; // Do not need any metadata + } + + // TODO(sanjay): Skip this if footer.metaindex_handle() size indicates + // it is an empty block. + ReadOptions opt; + BlockContents contents; + if (!ReadBlock(rep_->file, opt, footer.metaindex_handle(), &contents).ok()) { + // Do not propagate errors since meta info is not needed for operation + return; + } + Block* meta = new Block(contents); + + Iterator* iter = meta->NewIterator(BytewiseComparator()); + std::string key = "filter."; + key.append(rep_->options.filter_policy->Name()); + iter->Seek(key); + if (iter->Valid() && iter->key() == Slice(key)) { + ReadFilter(iter->value()); + } + delete iter; + delete meta; +} + +void Table::ReadFilter(const Slice& filter_handle_value) { + Slice v = filter_handle_value; + BlockHandle filter_handle; + if (!filter_handle.DecodeFrom(&v).ok()) { + return; + } + + // We might want to unify with ReadBlock() if we start + // requiring checksum verification in Table::Open. + ReadOptions opt; + BlockContents block; + if (!ReadBlock(rep_->file, opt, filter_handle, &block).ok()) { + return; + } + if (block.heap_allocated) { + rep_->filter_data = block.data.data(); // Will need to delete later + } + rep_->filter = new FilterBlockReader(rep_->options.filter_policy, block.data); +} + +Table::~Table() { + delete rep_; +} + +static void DeleteBlock(void* arg, void* ignored) { + delete reinterpret_cast(arg); +} + +static void DeleteCachedBlock(const Slice& key, void* value) { + Block* block = reinterpret_cast(value); + delete block; +} + +static void ReleaseBlock(void* arg, void* h) { + Cache* cache = reinterpret_cast(arg); + Cache::Handle* handle = reinterpret_cast(h); + cache->Release(handle); +} + +// Convert an index iterator value (i.e., an encoded BlockHandle) +// into an iterator over the contents of the corresponding block. +Iterator* Table::BlockReader(void* arg, + const ReadOptions& options, + const Slice& index_value) { + Table* table = reinterpret_cast(arg); + Cache* block_cache = table->rep_->options.block_cache; + Block* block = NULL; + Cache::Handle* cache_handle = NULL; + + BlockHandle handle; + Slice input = index_value; + Status s = handle.DecodeFrom(&input); + // We intentionally allow extra stuff in index_value so that we + // can add more features in the future. + + if (s.ok()) { + BlockContents contents; + if (block_cache != NULL) { + char cache_key_buffer[16]; + EncodeFixed64(cache_key_buffer, table->rep_->cache_id); + EncodeFixed64(cache_key_buffer+8, handle.offset()); + Slice key(cache_key_buffer, sizeof(cache_key_buffer)); + cache_handle = block_cache->Lookup(key); + if (cache_handle != NULL) { + block = reinterpret_cast(block_cache->Value(cache_handle)); + } else { + s = ReadBlock(table->rep_->file, options, handle, &contents); + if (s.ok()) { + block = new Block(contents); + if (contents.cachable && options.fill_cache) { + cache_handle = block_cache->Insert( + key, block, block->size(), &DeleteCachedBlock); + } + } + } + } else { + s = ReadBlock(table->rep_->file, options, handle, &contents); + if (s.ok()) { + block = new Block(contents); + } + } + } + + Iterator* iter; + if (block != NULL) { + iter = block->NewIterator(table->rep_->options.comparator); + if (cache_handle == NULL) { + iter->RegisterCleanup(&DeleteBlock, block, NULL); + } else { + iter->RegisterCleanup(&ReleaseBlock, block_cache, cache_handle); + } + } else { + iter = NewErrorIterator(s); + } + return iter; +} + +Iterator* Table::NewIterator(const ReadOptions& options) const { + return NewTwoLevelIterator( + rep_->index_block->NewIterator(rep_->options.comparator), + &Table::BlockReader, const_cast(this), options); +} + +Status Table::InternalGet(const ReadOptions& options, const Slice& k, + void* arg, + void (*saver)(void*, const Slice&, const Slice&)) { + Status s; + Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator); + iiter->Seek(k); + if (iiter->Valid()) { + Slice handle_value = iiter->value(); + FilterBlockReader* filter = rep_->filter; + BlockHandle handle; + if (filter != NULL && + handle.DecodeFrom(&handle_value).ok() && + !filter->KeyMayMatch(handle.offset(), k)) { + // Not found + } else { + Iterator* block_iter = BlockReader(this, options, iiter->value()); + block_iter->Seek(k); + if (block_iter->Valid()) { + (*saver)(arg, block_iter->key(), block_iter->value()); + } + s = block_iter->status(); + delete block_iter; + } + } + if (s.ok()) { + s = iiter->status(); + } + delete iiter; + return s; +} + + +uint64_t Table::ApproximateOffsetOf(const Slice& key) const { + Iterator* index_iter = + rep_->index_block->NewIterator(rep_->options.comparator); + index_iter->Seek(key); + uint64_t result; + if (index_iter->Valid()) { + BlockHandle handle; + Slice input = index_iter->value(); + Status s = handle.DecodeFrom(&input); + if (s.ok()) { + result = handle.offset(); + } else { + // Strange: we can't decode the block handle in the index block. + // We'll just return the offset of the metaindex block, which is + // close to the whole file size for this case. + result = rep_->metaindex_handle.offset(); + } + } else { + // key is past the last key in the file. Approximate the offset + // by returning the offset of the metaindex block (which is + // right near the end of the file). + result = rep_->metaindex_handle.offset(); + } + delete index_iter; + return result; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/table/table_builder.cc b/Subtrees/hyperleveldb/table/table_builder.cc new file mode 100644 index 0000000000..28a6677039 --- /dev/null +++ b/Subtrees/hyperleveldb/table/table_builder.cc @@ -0,0 +1,269 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/table_builder.h" + +#include +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/filter_policy.h" +#include "../hyperleveldb/options.h" +#include "block_builder.h" +#include "filter_block.h" +#include "format.h" +#include "../util/coding.h" +#include "../util/crc32c.h" + +namespace hyperleveldb { + +struct TableBuilder::Rep { + Options options; + Options index_block_options; + WritableFile* file; + uint64_t offset; + Status status; + BlockBuilder data_block; + BlockBuilder index_block; + std::string last_key; + int64_t num_entries; + bool closed; // Either Finish() or Abandon() has been called. + FilterBlockBuilder* filter_block; + + // We do not emit the index entry for a block until we have seen the + // first key for the next data block. This allows us to use shorter + // keys in the index block. For example, consider a block boundary + // between the keys "the quick brown fox" and "the who". We can use + // "the r" as the key for the index block entry since it is >= all + // entries in the first block and < all entries in subsequent + // blocks. + // + // Invariant: r->pending_index_entry is true only if data_block is empty. + bool pending_index_entry; + BlockHandle pending_handle; // Handle to add to index block + + std::string compressed_output; + + Rep(const Options& opt, WritableFile* f) + : options(opt), + index_block_options(opt), + file(f), + offset(0), + data_block(&options), + index_block(&index_block_options), + num_entries(0), + closed(false), + filter_block(opt.filter_policy == NULL ? NULL + : new FilterBlockBuilder(opt.filter_policy)), + pending_index_entry(false) { + index_block_options.block_restart_interval = 1; + } +}; + +TableBuilder::TableBuilder(const Options& options, WritableFile* file) + : rep_(new Rep(options, file)) { + if (rep_->filter_block != NULL) { + rep_->filter_block->StartBlock(0); + } +} + +TableBuilder::~TableBuilder() { + assert(rep_->closed); // Catch errors where caller forgot to call Finish() + delete rep_->filter_block; + delete rep_; +} + +Status TableBuilder::ChangeOptions(const Options& options) { + // Note: if more fields are added to Options, update + // this function to catch changes that should not be allowed to + // change in the middle of building a Table. + if (options.comparator != rep_->options.comparator) { + return Status::InvalidArgument("changing comparator while building table"); + } + + // Note that any live BlockBuilders point to rep_->options and therefore + // will automatically pick up the updated options. + rep_->options = options; + rep_->index_block_options = options; + rep_->index_block_options.block_restart_interval = 1; + return Status::OK(); +} + +void TableBuilder::Add(const Slice& key, const Slice& value) { + Rep* r = rep_; + assert(!r->closed); + if (!ok()) return; + if (r->num_entries > 0) { + assert(r->options.comparator->Compare(key, Slice(r->last_key)) > 0); + } + + if (r->pending_index_entry) { + assert(r->data_block.empty()); + r->options.comparator->FindShortestSeparator(&r->last_key, key); + std::string handle_encoding; + r->pending_handle.EncodeTo(&handle_encoding); + r->index_block.Add(r->last_key, Slice(handle_encoding)); + r->pending_index_entry = false; + } + + if (r->filter_block != NULL) { + r->filter_block->AddKey(key); + } + + r->last_key.assign(key.data(), key.size()); + r->num_entries++; + r->data_block.Add(key, value); + + const size_t estimated_block_size = r->data_block.CurrentSizeEstimate(); + if (estimated_block_size >= r->options.block_size) { + Flush(); + } +} + +void TableBuilder::Flush() { + Rep* r = rep_; + assert(!r->closed); + if (!ok()) return; + if (r->data_block.empty()) return; + assert(!r->pending_index_entry); + WriteBlock(&r->data_block, &r->pending_handle); + if (ok()) { + r->pending_index_entry = true; + } + if (r->filter_block != NULL) { + r->filter_block->StartBlock(r->offset); + } +} + +void TableBuilder::WriteBlock(BlockBuilder* block, BlockHandle* handle) { + // File format contains a sequence of blocks where each block has: + // block_data: uint8[n] + // type: uint8 + // crc: uint32 + assert(ok()); + Rep* r = rep_; + Slice raw = block->Finish(); + + Slice block_contents; + CompressionType type = r->options.compression; + // TODO(postrelease): Support more compression options: zlib? + switch (type) { + case kNoCompression: + block_contents = raw; + break; + + case kSnappyCompression: { + std::string* compressed = &r->compressed_output; + if (port::Snappy_Compress(raw.data(), raw.size(), compressed) && + compressed->size() < raw.size() - (raw.size() / 8u)) { + block_contents = *compressed; + } else { + // Snappy not supported, or compressed less than 12.5%, so just + // store uncompressed form + block_contents = raw; + type = kNoCompression; + } + break; + } + } + WriteRawBlock(block_contents, type, handle); + r->compressed_output.clear(); + block->Reset(); +} + +void TableBuilder::WriteRawBlock(const Slice& block_contents, + CompressionType type, + BlockHandle* handle) { + Rep* r = rep_; + handle->set_offset(r->offset); + handle->set_size(block_contents.size()); + r->status = r->file->Append(block_contents); + if (r->status.ok()) { + char trailer[kBlockTrailerSize]; + trailer[0] = type; + uint32_t crc = crc32c::Value(block_contents.data(), block_contents.size()); + crc = crc32c::Extend(crc, trailer, 1); // Extend crc to cover block type + EncodeFixed32(trailer+1, crc32c::Mask(crc)); + r->status = r->file->Append(Slice(trailer, kBlockTrailerSize)); + if (r->status.ok()) { + r->offset += block_contents.size() + kBlockTrailerSize; + } + } +} + +Status TableBuilder::status() const { + return rep_->status; +} + +Status TableBuilder::Finish() { + Rep* r = rep_; + Flush(); + assert(!r->closed); + r->closed = true; + + BlockHandle filter_block_handle, metaindex_block_handle, index_block_handle; + + // Write filter block + if (ok() && r->filter_block != NULL) { + WriteRawBlock(r->filter_block->Finish(), kNoCompression, + &filter_block_handle); + } + + // Write metaindex block + if (ok()) { + BlockBuilder meta_index_block(&r->options); + if (r->filter_block != NULL) { + // Add mapping from "filter.Name" to location of filter data + std::string key = "filter."; + key.append(r->options.filter_policy->Name()); + std::string handle_encoding; + filter_block_handle.EncodeTo(&handle_encoding); + meta_index_block.Add(key, handle_encoding); + } + + // TODO(postrelease): Add stats and other meta blocks + WriteBlock(&meta_index_block, &metaindex_block_handle); + } + + // Write index block + if (ok()) { + if (r->pending_index_entry) { + r->options.comparator->FindShortSuccessor(&r->last_key); + std::string handle_encoding; + r->pending_handle.EncodeTo(&handle_encoding); + r->index_block.Add(r->last_key, Slice(handle_encoding)); + r->pending_index_entry = false; + } + WriteBlock(&r->index_block, &index_block_handle); + } + + // Write footer + if (ok()) { + Footer footer; + footer.set_metaindex_handle(metaindex_block_handle); + footer.set_index_handle(index_block_handle); + std::string footer_encoding; + footer.EncodeTo(&footer_encoding); + r->status = r->file->Append(footer_encoding); + if (r->status.ok()) { + r->offset += footer_encoding.size(); + } + } + return r->status; +} + +void TableBuilder::Abandon() { + Rep* r = rep_; + assert(!r->closed); + r->closed = true; +} + +uint64_t TableBuilder::NumEntries() const { + return rep_->num_entries; +} + +uint64_t TableBuilder::FileSize() const { + return rep_->offset; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/table/table_test.cc b/Subtrees/hyperleveldb/table/table_test.cc new file mode 100644 index 0000000000..e47269b6ad --- /dev/null +++ b/Subtrees/hyperleveldb/table/table_test.cc @@ -0,0 +1,876 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "hyperleveldb/table.h" + +#include +#include +#include "../db/dbformat.h" +#include "../db/memtable.h" +#include "../db/write_batch_internal.h" +#include "../hyperleveldb/db.h" +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/iterator.h" +#include "../hyperleveldb/table_builder.h" +#include "block.h" +#include "block_builder.h" +#include "format.h" +#include "../util/random.h" +#include "../util/testharness.h" +#include "../util/testutil.h" + +namespace hyperleveldb { + +// Return reverse of "key". +// Used to test non-lexicographic comparators. +static std::string Reverse(const Slice& key) { + std::string str(key.ToString()); + std::string rev(""); + for (std::string::reverse_iterator rit = str.rbegin(); + rit != str.rend(); ++rit) { + rev.push_back(*rit); + } + return rev; +} + +namespace { +class ReverseKeyComparator : public Comparator { + public: + virtual const char* Name() const { + return "leveldb.ReverseBytewiseComparator"; + } + + virtual int Compare(const Slice& a, const Slice& b) const { + return BytewiseComparator()->Compare(Reverse(a), Reverse(b)); + } + + virtual void FindShortestSeparator( + std::string* start, + const Slice& limit) const { + std::string s = Reverse(*start); + std::string l = Reverse(limit); + BytewiseComparator()->FindShortestSeparator(&s, l); + *start = Reverse(s); + } + + virtual void FindShortSuccessor(std::string* key) const { + std::string s = Reverse(*key); + BytewiseComparator()->FindShortSuccessor(&s); + *key = Reverse(s); + } +}; +} // namespace +static ReverseKeyComparator reverse_key_comparator; + +static void Increment(const Comparator* cmp, std::string* key) { + if (cmp == BytewiseComparator()) { + key->push_back('\0'); + } else { + assert(cmp == &reverse_key_comparator); + std::string rev = Reverse(*key); + rev.push_back('\0'); + *key = Reverse(rev); + } +} + +// An STL comparator that uses a Comparator +namespace { +struct STLLessThan { + const Comparator* cmp; + + STLLessThan() : cmp(BytewiseComparator()) { } + STLLessThan(const Comparator* c) : cmp(c) { } + bool operator()(const std::string& a, const std::string& b) const { + return cmp->Compare(Slice(a), Slice(b)) < 0; + } +}; +} // namespace + +class StringSink: public WritableFile { + public: + ~StringSink() { } + + const std::string& contents() const { return contents_; } + + virtual Status Close() { return Status::OK(); } + virtual Status Sync() { return Status::OK(); } + + virtual Status WriteAt(uint64_t offset, const Slice& slice) { + std::string tmp = contents_.substr(0, offset); + tmp.append(slice.data(), slice.size()); + if (contents_.size() > offset + slice.size()) { + tmp += contents_.substr(offset + slice.size()); + } + contents_ = tmp; + return Status::OK(); + } + virtual Status Append(const Slice& data) { + contents_.append(data.data(), data.size()); + return Status::OK(); + } + + private: + std::string contents_; +}; + + +class StringSource: public RandomAccessFile { + public: + StringSource(const Slice& contents) + : contents_(contents.data(), contents.size()) { + } + + virtual ~StringSource() { } + + uint64_t Size() const { return contents_.size(); } + + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const { + if (offset > contents_.size()) { + return Status::InvalidArgument("invalid Read offset"); + } + if (offset + n > contents_.size()) { + n = contents_.size() - offset; + } + memcpy(scratch, &contents_[offset], n); + *result = Slice(scratch, n); + return Status::OK(); + } + + private: + std::string contents_; +}; + +typedef std::map KVMap; + +// Helper class for tests to unify the interface between +// BlockBuilder/TableBuilder and Block/Table. +class Constructor { + public: + explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) { } + virtual ~Constructor() { } + + void Add(const std::string& key, const Slice& value) { + data_[key] = value.ToString(); + } + + // Finish constructing the data structure with all the keys that have + // been added so far. Returns the keys in sorted order in "*keys" + // and stores the key/value pairs in "*kvmap" + void Finish(const Options& options, + std::vector* keys, + KVMap* kvmap) { + *kvmap = data_; + keys->clear(); + for (KVMap::const_iterator it = data_.begin(); + it != data_.end(); + ++it) { + keys->push_back(it->first); + } + data_.clear(); + Status s = FinishImpl(options, *kvmap); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + + // Construct the data structure from the data in "data" + virtual Status FinishImpl(const Options& options, const KVMap& data) = 0; + + virtual Iterator* NewIterator() const = 0; + + virtual const KVMap& data() { return data_; } + + virtual DB* db() const { return NULL; } // Overridden in DBConstructor + + private: + KVMap data_; +}; + +class BlockConstructor: public Constructor { + public: + explicit BlockConstructor(const Comparator* cmp) + : Constructor(cmp), + comparator_(cmp), + block_(NULL) { } + ~BlockConstructor() { + delete block_; + } + virtual Status FinishImpl(const Options& options, const KVMap& data) { + delete block_; + block_ = NULL; + BlockBuilder builder(&options); + + for (KVMap::const_iterator it = data.begin(); + it != data.end(); + ++it) { + builder.Add(it->first, it->second); + } + // Open the block + data_ = builder.Finish().ToString(); + BlockContents contents; + contents.data = data_; + contents.cachable = false; + contents.heap_allocated = false; + block_ = new Block(contents); + return Status::OK(); + } + virtual Iterator* NewIterator() const { + return block_->NewIterator(comparator_); + } + + private: + const Comparator* comparator_; + std::string data_; + Block* block_; + + BlockConstructor(); +}; + +class TableConstructor: public Constructor { + public: + TableConstructor(const Comparator* cmp) + : Constructor(cmp), + source_(NULL), table_(NULL) { + } + ~TableConstructor() { + Reset(); + } + virtual Status FinishImpl(const Options& options, const KVMap& data) { + Reset(); + StringSink sink; + TableBuilder builder(options, &sink); + + for (KVMap::const_iterator it = data.begin(); + it != data.end(); + ++it) { + builder.Add(it->first, it->second); + ASSERT_TRUE(builder.status().ok()); + } + Status s = builder.Finish(); + ASSERT_TRUE(s.ok()) << s.ToString(); + + ASSERT_EQ(sink.contents().size(), builder.FileSize()); + + // Open the table + source_ = new StringSource(sink.contents()); + Options table_options; + table_options.comparator = options.comparator; + return Table::Open(table_options, source_, sink.contents().size(), &table_); + } + + virtual Iterator* NewIterator() const { + return table_->NewIterator(ReadOptions()); + } + + uint64_t ApproximateOffsetOf(const Slice& key) const { + return table_->ApproximateOffsetOf(key); + } + + private: + void Reset() { + delete table_; + delete source_; + table_ = NULL; + source_ = NULL; + } + + StringSource* source_; + Table* table_; + + TableConstructor(); +}; + +// A helper class that converts internal format keys into user keys +class KeyConvertingIterator: public Iterator { + public: + explicit KeyConvertingIterator(Iterator* iter) : iter_(iter) { } + virtual ~KeyConvertingIterator() { delete iter_; } + virtual bool Valid() const { return iter_->Valid(); } + virtual void Seek(const Slice& target) { + ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue); + std::string encoded; + AppendInternalKey(&encoded, ikey); + iter_->Seek(encoded); + } + virtual void SeekToFirst() { iter_->SeekToFirst(); } + virtual void SeekToLast() { iter_->SeekToLast(); } + virtual void Next() { iter_->Next(); } + virtual void Prev() { iter_->Prev(); } + + virtual Slice key() const { + assert(Valid()); + ParsedInternalKey key; + if (!ParseInternalKey(iter_->key(), &key)) { + status_ = Status::Corruption("malformed internal key"); + return Slice("corrupted key"); + } + return key.user_key; + } + + virtual Slice value() const { return iter_->value(); } + virtual Status status() const { + return status_.ok() ? iter_->status() : status_; + } + + private: + mutable Status status_; + Iterator* iter_; + + // No copying allowed + KeyConvertingIterator(const KeyConvertingIterator&); + void operator=(const KeyConvertingIterator&); +}; + +class MemTableConstructor: public Constructor { + public: + explicit MemTableConstructor(const Comparator* cmp) + : Constructor(cmp), + internal_comparator_(cmp) { + memtable_ = new MemTable(internal_comparator_); + memtable_->Ref(); + } + ~MemTableConstructor() { + memtable_->Unref(); + } + virtual Status FinishImpl(const Options& options, const KVMap& data) { + memtable_->Unref(); + memtable_ = new MemTable(internal_comparator_); + memtable_->Ref(); + int seq = 1; + for (KVMap::const_iterator it = data.begin(); + it != data.end(); + ++it) { + memtable_->Add(seq, kTypeValue, it->first, it->second); + seq++; + } + return Status::OK(); + } + virtual Iterator* NewIterator() const { + return new KeyConvertingIterator(memtable_->NewIterator()); + } + + private: + InternalKeyComparator internal_comparator_; + MemTable* memtable_; +}; + +class DBConstructor: public Constructor { + public: + explicit DBConstructor(const Comparator* cmp) + : Constructor(cmp), + comparator_(cmp) { + db_ = NULL; + NewDB(); + } + ~DBConstructor() { + delete db_; + } + virtual Status FinishImpl(const Options& options, const KVMap& data) { + delete db_; + db_ = NULL; + NewDB(); + for (KVMap::const_iterator it = data.begin(); + it != data.end(); + ++it) { + WriteBatch batch; + batch.Put(it->first, it->second); + ASSERT_TRUE(db_->Write(WriteOptions(), &batch).ok()); + } + return Status::OK(); + } + virtual Iterator* NewIterator() const { + return db_->NewIterator(ReadOptions()); + } + + virtual DB* db() const { return db_; } + + private: + void NewDB() { + std::string name = test::TmpDir() + "/table_testdb"; + + Options options; + options.comparator = comparator_; + Status status = DestroyDB(name, options); + ASSERT_TRUE(status.ok()) << status.ToString(); + + options.create_if_missing = true; + options.error_if_exists = true; + options.write_buffer_size = 10000; // Something small to force merging + status = DB::Open(options, name, &db_); + ASSERT_TRUE(status.ok()) << status.ToString(); + } + + const Comparator* comparator_; + DB* db_; +}; + +enum TestType { + TABLE_TEST, + BLOCK_TEST, + MEMTABLE_TEST, + DB_TEST +}; + +struct TestArgs { + TestType type; + bool reverse_compare; + int restart_interval; +}; + +static const TestArgs kTestArgList[] = { + { TABLE_TEST, false, 16 }, + { TABLE_TEST, false, 1 }, + { TABLE_TEST, false, 1024 }, + { TABLE_TEST, true, 16 }, + { TABLE_TEST, true, 1 }, + { TABLE_TEST, true, 1024 }, + + { BLOCK_TEST, false, 16 }, + { BLOCK_TEST, false, 1 }, + { BLOCK_TEST, false, 1024 }, + { BLOCK_TEST, true, 16 }, + { BLOCK_TEST, true, 1 }, + { BLOCK_TEST, true, 1024 }, + + // Restart interval does not matter for memtables + { MEMTABLE_TEST, false, 16 }, + { MEMTABLE_TEST, true, 16 }, + + // Do not bother with restart interval variations for DB + { DB_TEST, false, 16 }, + { DB_TEST, true, 16 }, +}; +static const int kNumTestArgs = sizeof(kTestArgList) / sizeof(kTestArgList[0]); + +class Harness { + public: + Harness() : constructor_(NULL) { } + + void Init(const TestArgs& args) { + delete constructor_; + constructor_ = NULL; + options_ = Options(); + + options_.block_restart_interval = args.restart_interval; + // Use shorter block size for tests to exercise block boundary + // conditions more. + options_.block_size = 256; + if (args.reverse_compare) { + options_.comparator = &reverse_key_comparator; + } + switch (args.type) { + case TABLE_TEST: + constructor_ = new TableConstructor(options_.comparator); + break; + case BLOCK_TEST: + constructor_ = new BlockConstructor(options_.comparator); + break; + case MEMTABLE_TEST: + constructor_ = new MemTableConstructor(options_.comparator); + break; + case DB_TEST: + constructor_ = new DBConstructor(options_.comparator); + break; + } + } + + ~Harness() { + delete constructor_; + } + + void Add(const std::string& key, const std::string& value) { + constructor_->Add(key, value); + } + + void Test(Random* rnd) { + std::vector keys; + KVMap data; + constructor_->Finish(options_, &keys, &data); + + TestForwardScan(keys, data); + TestBackwardScan(keys, data); + TestRandomAccess(rnd, keys, data); + } + + void TestForwardScan(const std::vector& keys, + const KVMap& data) { + Iterator* iter = constructor_->NewIterator(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToFirst(); + for (KVMap::const_iterator model_iter = data.begin(); + model_iter != data.end(); + ++model_iter) { + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + iter->Next(); + } + ASSERT_TRUE(!iter->Valid()); + delete iter; + } + + void TestBackwardScan(const std::vector& keys, + const KVMap& data) { + Iterator* iter = constructor_->NewIterator(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToLast(); + for (KVMap::const_reverse_iterator model_iter = data.rbegin(); + model_iter != data.rend(); + ++model_iter) { + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + iter->Prev(); + } + ASSERT_TRUE(!iter->Valid()); + delete iter; + } + + void TestRandomAccess(Random* rnd, + const std::vector& keys, + const KVMap& data) { + static const bool kVerbose = false; + Iterator* iter = constructor_->NewIterator(); + ASSERT_TRUE(!iter->Valid()); + KVMap::const_iterator model_iter = data.begin(); + if (kVerbose) fprintf(stderr, "---\n"); + for (int i = 0; i < 200; i++) { + const int toss = rnd->Uniform(5); + switch (toss) { + case 0: { + if (iter->Valid()) { + if (kVerbose) fprintf(stderr, "Next\n"); + iter->Next(); + ++model_iter; + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + } + break; + } + + case 1: { + if (kVerbose) fprintf(stderr, "SeekToFirst\n"); + iter->SeekToFirst(); + model_iter = data.begin(); + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + break; + } + + case 2: { + std::string key = PickRandomKey(rnd, keys); + model_iter = data.lower_bound(key); + if (kVerbose) fprintf(stderr, "Seek '%s'\n", + EscapeString(key).c_str()); + iter->Seek(Slice(key)); + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + break; + } + + case 3: { + if (iter->Valid()) { + if (kVerbose) fprintf(stderr, "Prev\n"); + iter->Prev(); + if (model_iter == data.begin()) { + model_iter = data.end(); // Wrap around to invalid value + } else { + --model_iter; + } + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + } + break; + } + + case 4: { + if (kVerbose) fprintf(stderr, "SeekToLast\n"); + iter->SeekToLast(); + if (keys.empty()) { + model_iter = data.end(); + } else { + std::string last = data.rbegin()->first; + model_iter = data.lower_bound(last); + } + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + break; + } + } + } + delete iter; + } + + std::string ToString(const KVMap& data, const KVMap::const_iterator& it) { + if (it == data.end()) { + return "END"; + } else { + return "'" + it->first + "->" + it->second + "'"; + } + } + + std::string ToString(const KVMap& data, + const KVMap::const_reverse_iterator& it) { + if (it == data.rend()) { + return "END"; + } else { + return "'" + it->first + "->" + it->second + "'"; + } + } + + std::string ToString(const Iterator* it) { + if (!it->Valid()) { + return "END"; + } else { + return "'" + it->key().ToString() + "->" + it->value().ToString() + "'"; + } + } + + std::string PickRandomKey(Random* rnd, const std::vector& keys) { + if (keys.empty()) { + return "foo"; + } else { + const int index = rnd->Uniform(keys.size()); + std::string result = keys[index]; + switch (rnd->Uniform(3)) { + case 0: + // Return an existing key + break; + case 1: { + // Attempt to return something smaller than an existing key + if (result.size() > 0 && result[result.size()-1] > '\0') { + result[result.size()-1]--; + } + break; + } + case 2: { + // Return something larger than an existing key + Increment(options_.comparator, &result); + break; + } + } + return result; + } + } + + // Returns NULL if not running against a DB + DB* db() const { return constructor_->db(); } + + private: + Options options_; + Constructor* constructor_; +}; + +// Test empty table/block. +TEST(Harness, Empty) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 1); + Test(&rnd); + } +} + +// Special test for a block with no restart entries. The C++ leveldb +// code never generates such blocks, but the Java version of leveldb +// seems to. +TEST(Harness, ZeroRestartPointsInBlock) { + char data[sizeof(uint32_t)]; + memset(data, 0, sizeof(data)); + BlockContents contents; + contents.data = Slice(data, sizeof(data)); + contents.cachable = false; + contents.heap_allocated = false; + Block block(contents); + Iterator* iter = block.NewIterator(BytewiseComparator()); + iter->SeekToFirst(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToLast(); + ASSERT_TRUE(!iter->Valid()); + iter->Seek("foo"); + ASSERT_TRUE(!iter->Valid()); + delete iter; +} + +// Test the empty key +TEST(Harness, SimpleEmptyKey) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 1); + Add("", "v"); + Test(&rnd); + } +} + +TEST(Harness, SimpleSingle) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 2); + Add("abc", "v"); + Test(&rnd); + } +} + +TEST(Harness, SimpleMulti) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 3); + Add("abc", "v"); + Add("abcd", "v"); + Add("ac", "v2"); + Test(&rnd); + } +} + +TEST(Harness, SimpleSpecialKey) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 4); + Add("\xff\xff", "v3"); + Test(&rnd); + } +} + +TEST(Harness, Randomized) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 5); + for (int num_entries = 0; num_entries < 2000; + num_entries += (num_entries < 50 ? 1 : 200)) { + if ((num_entries % 10) == 0) { + fprintf(stderr, "case %d of %d: num_entries = %d\n", + (i + 1), int(kNumTestArgs), num_entries); + } + for (int e = 0; e < num_entries; e++) { + std::string v; + Add(test::RandomKey(&rnd, rnd.Skewed(4)), + test::RandomString(&rnd, rnd.Skewed(5), &v).ToString()); + } + Test(&rnd); + } + } +} + +TEST(Harness, RandomizedLongDB) { + Random rnd(test::RandomSeed()); + TestArgs args = { DB_TEST, false, 16 }; + Init(args); + int num_entries = 100000; + for (int e = 0; e < num_entries; e++) { + std::string v; + Add(test::RandomKey(&rnd, rnd.Skewed(4)), + test::RandomString(&rnd, rnd.Skewed(5), &v).ToString()); + } + Test(&rnd); + + // We must have created enough data to force merging + int files = 0; + for (int level = 0; level < config::kNumLevels; level++) { + std::string value; + char name[100]; + snprintf(name, sizeof(name), "leveldb.num-files-at-level%d", level); + ASSERT_TRUE(db()->GetProperty(name, &value)); + files += atoi(value.c_str()); + } + ASSERT_GT(files, 0); +} + +class MemTableTest { }; + +TEST(MemTableTest, Simple) { + InternalKeyComparator cmp(BytewiseComparator()); + MemTable* memtable = new MemTable(cmp); + memtable->Ref(); + WriteBatch batch; + WriteBatchInternal::SetSequence(&batch, 100); + batch.Put(std::string("k1"), std::string("v1")); + batch.Put(std::string("k2"), std::string("v2")); + batch.Put(std::string("k3"), std::string("v3")); + batch.Put(std::string("largekey"), std::string("vlarge")); + ASSERT_TRUE(WriteBatchInternal::InsertInto(&batch, memtable).ok()); + + Iterator* iter = memtable->NewIterator(); + iter->SeekToFirst(); + while (iter->Valid()) { + fprintf(stderr, "key: '%s' -> '%s'\n", + iter->key().ToString().c_str(), + iter->value().ToString().c_str()); + iter->Next(); + } + + delete iter; + memtable->Unref(); +} + +static bool Between(uint64_t val, uint64_t low, uint64_t high) { + bool result = (val >= low) && (val <= high); + if (!result) { + fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", + (unsigned long long)(val), + (unsigned long long)(low), + (unsigned long long)(high)); + } + return result; +} + +class TableTest { }; + +TEST(TableTest, ApproximateOffsetOfPlain) { + TableConstructor c(BytewiseComparator()); + c.Add("k01", "hello"); + c.Add("k02", "hello2"); + c.Add("k03", std::string(10000, 'x')); + c.Add("k04", std::string(200000, 'x')); + c.Add("k05", std::string(300000, 'x')); + c.Add("k06", "hello3"); + c.Add("k07", std::string(100000, 'x')); + std::vector keys; + KVMap kvmap; + Options options; + options.block_size = 1024; + options.compression = kNoCompression; + c.Finish(options, &keys, &kvmap); + + ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01a"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 10000, 11000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04a"), 210000, 211000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k05"), 210000, 211000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 612000)); + +} + +static bool SnappyCompressionSupported() { + std::string out; + Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + return port::Snappy_Compress(in.data(), in.size(), &out); +} + +TEST(TableTest, ApproximateOffsetOfCompressed) { + if (!SnappyCompressionSupported()) { + fprintf(stderr, "skipping compression tests\n"); + return; + } + + Random rnd(301); + TableConstructor c(BytewiseComparator()); + std::string tmp; + c.Add("k01", "hello"); + c.Add("k02", test::CompressibleString(&rnd, 0.25, 10000, &tmp)); + c.Add("k03", "hello3"); + c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp)); + std::vector keys; + KVMap kvmap; + Options options; + options.block_size = 1024; + options.compression = kSnappyCompression; + c.Finish(options, &keys, &kvmap); + + ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 2000, 3000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 2000, 3000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 4000, 6000)); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/table/two_level_iterator.cc b/Subtrees/hyperleveldb/table/two_level_iterator.cc new file mode 100644 index 0000000000..770e12eadc --- /dev/null +++ b/Subtrees/hyperleveldb/table/two_level_iterator.cc @@ -0,0 +1,182 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "two_level_iterator.h" + +#include "../hyperleveldb/table.h" +#include "block.h" +#include "format.h" +#include "iterator_wrapper.h" + +namespace hyperleveldb { + +namespace { + +typedef Iterator* (*BlockFunction)(void*, const ReadOptions&, const Slice&); + +class TwoLevelIterator: public Iterator { + public: + TwoLevelIterator( + Iterator* index_iter, + BlockFunction block_function, + void* arg, + const ReadOptions& options); + + virtual ~TwoLevelIterator(); + + virtual void Seek(const Slice& target); + virtual void SeekToFirst(); + virtual void SeekToLast(); + virtual void Next(); + virtual void Prev(); + + virtual bool Valid() const { + return data_iter_.Valid(); + } + virtual Slice key() const { + assert(Valid()); + return data_iter_.key(); + } + virtual Slice value() const { + assert(Valid()); + return data_iter_.value(); + } + virtual Status status() const { + // It'd be nice if status() returned a const Status& instead of a Status + if (!index_iter_.status().ok()) { + return index_iter_.status(); + } else if (data_iter_.iter() != NULL && !data_iter_.status().ok()) { + return data_iter_.status(); + } else { + return status_; + } + } + + private: + void SaveError(const Status& s) { + if (status_.ok() && !s.ok()) status_ = s; + } + void SkipEmptyDataBlocksForward(); + void SkipEmptyDataBlocksBackward(); + void SetDataIterator(Iterator* data_iter); + void InitDataBlock(); + + BlockFunction block_function_; + void* arg_; + const ReadOptions options_; + Status status_; + IteratorWrapper index_iter_; + IteratorWrapper data_iter_; // May be NULL + // If data_iter_ is non-NULL, then "data_block_handle_" holds the + // "index_value" passed to block_function_ to create the data_iter_. + std::string data_block_handle_; +}; + +TwoLevelIterator::TwoLevelIterator( + Iterator* index_iter, + BlockFunction block_function, + void* arg, + const ReadOptions& options) + : block_function_(block_function), + arg_(arg), + options_(options), + index_iter_(index_iter), + data_iter_(NULL) { +} + +TwoLevelIterator::~TwoLevelIterator() { +} + +void TwoLevelIterator::Seek(const Slice& target) { + index_iter_.Seek(target); + InitDataBlock(); + if (data_iter_.iter() != NULL) data_iter_.Seek(target); + SkipEmptyDataBlocksForward(); +} + +void TwoLevelIterator::SeekToFirst() { + index_iter_.SeekToFirst(); + InitDataBlock(); + if (data_iter_.iter() != NULL) data_iter_.SeekToFirst(); + SkipEmptyDataBlocksForward(); +} + +void TwoLevelIterator::SeekToLast() { + index_iter_.SeekToLast(); + InitDataBlock(); + if (data_iter_.iter() != NULL) data_iter_.SeekToLast(); + SkipEmptyDataBlocksBackward(); +} + +void TwoLevelIterator::Next() { + assert(Valid()); + data_iter_.Next(); + SkipEmptyDataBlocksForward(); +} + +void TwoLevelIterator::Prev() { + assert(Valid()); + data_iter_.Prev(); + SkipEmptyDataBlocksBackward(); +} + + +void TwoLevelIterator::SkipEmptyDataBlocksForward() { + while (data_iter_.iter() == NULL || !data_iter_.Valid()) { + // Move to next block + if (!index_iter_.Valid()) { + SetDataIterator(NULL); + return; + } + index_iter_.Next(); + InitDataBlock(); + if (data_iter_.iter() != NULL) data_iter_.SeekToFirst(); + } +} + +void TwoLevelIterator::SkipEmptyDataBlocksBackward() { + while (data_iter_.iter() == NULL || !data_iter_.Valid()) { + // Move to next block + if (!index_iter_.Valid()) { + SetDataIterator(NULL); + return; + } + index_iter_.Prev(); + InitDataBlock(); + if (data_iter_.iter() != NULL) data_iter_.SeekToLast(); + } +} + +void TwoLevelIterator::SetDataIterator(Iterator* data_iter) { + if (data_iter_.iter() != NULL) SaveError(data_iter_.status()); + data_iter_.Set(data_iter); +} + +void TwoLevelIterator::InitDataBlock() { + if (!index_iter_.Valid()) { + SetDataIterator(NULL); + } else { + Slice handle = index_iter_.value(); + if (data_iter_.iter() != NULL && handle.compare(data_block_handle_) == 0) { + // data_iter_ is already constructed with this iterator, so + // no need to change anything + } else { + Iterator* iter = (*block_function_)(arg_, options_, handle); + data_block_handle_.assign(handle.data(), handle.size()); + SetDataIterator(iter); + } + } +} + +} // namespace + +Iterator* NewTwoLevelIterator( + Iterator* index_iter, + BlockFunction block_function, + void* arg, + const ReadOptions& options) { + return new TwoLevelIterator(index_iter, block_function, arg, options); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/table/two_level_iterator.h b/Subtrees/hyperleveldb/table/two_level_iterator.h new file mode 100644 index 0000000000..99c71be064 --- /dev/null +++ b/Subtrees/hyperleveldb/table/two_level_iterator.h @@ -0,0 +1,34 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ +#define STORAGE_HYPERLEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ + +#include "../hyperleveldb/iterator.h" + +namespace hyperleveldb { + +struct ReadOptions; + +// Return a new two level iterator. A two-level iterator contains an +// index iterator whose values point to a sequence of blocks where +// each block is itself a sequence of key,value pairs. The returned +// two-level iterator yields the concatenation of all key/value pairs +// in the sequence of blocks. Takes ownership of "index_iter" and +// will delete it when no longer needed. +// +// Uses a supplied function to convert an index_iter value into +// an iterator over the contents of the corresponding block. +extern Iterator* NewTwoLevelIterator( + Iterator* index_iter, + Iterator* (*block_function)( + void* arg, + const ReadOptions& options, + const Slice& index_value), + void* arg, + const ReadOptions& options); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ diff --git a/Subtrees/hyperleveldb/util/arena.cc b/Subtrees/hyperleveldb/util/arena.cc new file mode 100644 index 0000000000..b83d14e1c8 --- /dev/null +++ b/Subtrees/hyperleveldb/util/arena.cc @@ -0,0 +1,68 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "arena.h" +#include + +namespace hyperleveldb { + +static const int kBlockSize = 4096; + +Arena::Arena() { + blocks_memory_ = 0; + alloc_ptr_ = NULL; // First allocation will allocate a block + alloc_bytes_remaining_ = 0; +} + +Arena::~Arena() { + for (size_t i = 0; i < blocks_.size(); i++) { + delete[] blocks_[i]; + } +} + +char* Arena::AllocateFallback(size_t bytes) { + if (bytes > kBlockSize / 4) { + // Object is more than a quarter of our block size. Allocate it separately + // to avoid wasting too much space in leftover bytes. + char* result = AllocateNewBlock(bytes); + return result; + } + + // We waste the remaining space in the current block. + alloc_ptr_ = AllocateNewBlock(kBlockSize); + alloc_bytes_remaining_ = kBlockSize; + + char* result = alloc_ptr_; + alloc_ptr_ += bytes; + alloc_bytes_remaining_ -= bytes; + return result; +} + +char* Arena::AllocateAligned(size_t bytes) { + const int align = sizeof(void*); // We'll align to pointer size + assert((align & (align-1)) == 0); // Pointer size should be a power of 2 + size_t current_mod = reinterpret_cast(alloc_ptr_) & (align-1); + size_t slop = (current_mod == 0 ? 0 : align - current_mod); + size_t needed = bytes + slop; + char* result; + if (needed <= alloc_bytes_remaining_) { + result = alloc_ptr_ + slop; + alloc_ptr_ += needed; + alloc_bytes_remaining_ -= needed; + } else { + // AllocateFallback always returned aligned memory + result = AllocateFallback(bytes); + } + assert((reinterpret_cast(result) & (align-1)) == 0); + return result; +} + +char* Arena::AllocateNewBlock(size_t block_bytes) { + char* result = new char[block_bytes]; + blocks_memory_ += block_bytes; + blocks_.push_back(result); + return result; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/arena.h b/Subtrees/hyperleveldb/util/arena.h new file mode 100644 index 0000000000..04ad883589 --- /dev/null +++ b/Subtrees/hyperleveldb/util/arena.h @@ -0,0 +1,68 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_UTIL_ARENA_H_ +#define STORAGE_HYPERLEVELDB_UTIL_ARENA_H_ + +#include +#include +#include +#include + +namespace hyperleveldb { + +class Arena { + public: + Arena(); + ~Arena(); + + // Return a pointer to a newly allocated memory block of "bytes" bytes. + char* Allocate(size_t bytes); + + // Allocate memory with the normal alignment guarantees provided by malloc + char* AllocateAligned(size_t bytes); + + // Returns an estimate of the total memory usage of data allocated + // by the arena (including space allocated but not yet used for user + // allocations). + size_t MemoryUsage() const { + return blocks_memory_ + blocks_.capacity() * sizeof(char*); + } + + private: + char* AllocateFallback(size_t bytes); + char* AllocateNewBlock(size_t block_bytes); + + // Allocation state + char* alloc_ptr_; + size_t alloc_bytes_remaining_; + + // Array of new[] allocated memory blocks + std::vector blocks_; + + // Bytes of memory in blocks allocated so far + size_t blocks_memory_; + + // No copying allowed + Arena(const Arena&); + void operator=(const Arena&); +}; + +inline char* Arena::Allocate(size_t bytes) { + // The semantics of what to return are a bit messy if we allow + // 0-byte allocations, so we disallow them here (we don't need + // them for our internal use). + assert(bytes > 0); + if (bytes <= alloc_bytes_remaining_) { + char* result = alloc_ptr_; + alloc_ptr_ += bytes; + alloc_bytes_remaining_ -= bytes; + return result; + } + return AllocateFallback(bytes); +} + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_ARENA_H_ diff --git a/Subtrees/hyperleveldb/util/arena_test.cc b/Subtrees/hyperleveldb/util/arena_test.cc new file mode 100644 index 0000000000..05137bd663 --- /dev/null +++ b/Subtrees/hyperleveldb/util/arena_test.cc @@ -0,0 +1,68 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "arena.h" + +#include "random.h" +#include "testharness.h" + +namespace hyperleveldb { + +class ArenaTest { }; + +TEST(ArenaTest, Empty) { + Arena arena; +} + +TEST(ArenaTest, Simple) { + std::vector > allocated; + Arena arena; + const int N = 100000; + size_t bytes = 0; + Random rnd(301); + for (int i = 0; i < N; i++) { + size_t s; + if (i % (N / 10) == 0) { + s = i; + } else { + s = rnd.OneIn(4000) ? rnd.Uniform(6000) : + (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); + } + if (s == 0) { + // Our arena disallows size 0 allocations. + s = 1; + } + char* r; + if (rnd.OneIn(10)) { + r = arena.AllocateAligned(s); + } else { + r = arena.Allocate(s); + } + + for (int b = 0; b < s; b++) { + // Fill the "i"th allocation with a known bit pattern + r[b] = i % 256; + } + bytes += s; + allocated.push_back(std::make_pair(s, r)); + ASSERT_GE(arena.MemoryUsage(), bytes); + if (i > N/10) { + ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); + } + } + for (int i = 0; i < allocated.size(); i++) { + size_t num_bytes = allocated[i].first; + const char* p = allocated[i].second; + for (int b = 0; b < num_bytes; b++) { + // Check the "i"th allocation for the known bit pattern + ASSERT_EQ(int(p[b]) & 0xff, i % 256); + } + } +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/util/bloom.cc b/Subtrees/hyperleveldb/util/bloom.cc new file mode 100644 index 0000000000..5bc91037ea --- /dev/null +++ b/Subtrees/hyperleveldb/util/bloom.cc @@ -0,0 +1,95 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/filter_policy.h" + +#include "../hyperleveldb/slice.h" +#include "hash.h" + +namespace hyperleveldb { + +namespace { +static uint32_t BloomHash(const Slice& key) { + return Hash(key.data(), key.size(), 0xbc9f1d34); +} + +class BloomFilterPolicy : public FilterPolicy { + private: + size_t bits_per_key_; + size_t k_; + + public: + explicit BloomFilterPolicy(int bits_per_key) + : bits_per_key_(bits_per_key) { + // We intentionally round down to reduce probing cost a little bit + k_ = static_cast(bits_per_key * 0.69); // 0.69 =~ ln(2) + if (k_ < 1) k_ = 1; + if (k_ > 30) k_ = 30; + } + + virtual const char* Name() const { + return "leveldb.BuiltinBloomFilter"; + } + + virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { + // Compute bloom filter size (in both bits and bytes) + size_t bits = n * bits_per_key_; + + // For small n, we can see a very high false positive rate. Fix it + // by enforcing a minimum bloom filter length. + if (bits < 64) bits = 64; + + size_t bytes = (bits + 7) / 8; + bits = bytes * 8; + + const size_t init_size = dst->size(); + dst->resize(init_size + bytes, 0); + dst->push_back(static_cast(k_)); // Remember # of probes in filter + char* array = &(*dst)[init_size]; + for (size_t i = 0; i < n; i++) { + // Use double-hashing to generate a sequence of hash values. + // See analysis in [Kirsch,Mitzenmacher 2006]. + uint32_t h = BloomHash(keys[i]); + const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits + for (size_t j = 0; j < k_; j++) { + const uint32_t bitpos = h % bits; + array[bitpos/8] |= (1 << (bitpos % 8)); + h += delta; + } + } + } + + virtual bool KeyMayMatch(const Slice& key, const Slice& bloom_filter) const { + const size_t len = bloom_filter.size(); + if (len < 2) return false; + + const char* array = bloom_filter.data(); + const size_t bits = (len - 1) * 8; + + // Use the encoded k so that we can read filters generated by + // bloom filters created using different parameters. + const size_t k = array[len-1]; + if (k > 30) { + // Reserved for potentially new encodings for short bloom filters. + // Consider it a match. + return true; + } + + uint32_t h = BloomHash(key); + const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits + for (size_t j = 0; j < k; j++) { + const uint32_t bitpos = h % bits; + if ((array[bitpos/8] & (1 << (bitpos % 8))) == 0) return false; + h += delta; + } + return true; + } +}; +} + +const FilterPolicy* NewBloomFilterPolicy(int bits_per_key) { + return new BloomFilterPolicy(bits_per_key); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/bloom_test.cc b/Subtrees/hyperleveldb/util/bloom_test.cc new file mode 100644 index 0000000000..19c3a4389b --- /dev/null +++ b/Subtrees/hyperleveldb/util/bloom_test.cc @@ -0,0 +1,160 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/filter_policy.h" + +#include "coding.h" +#include "logging.h" +#include "testharness.h" +#include "testutil.h" + +namespace hyperleveldb { + +static const int kVerbose = 1; + +static Slice Key(int i, char* buffer) { + EncodeFixed32(buffer, i); + return Slice(buffer, sizeof(uint32_t)); +} + +class BloomTest { + private: + const FilterPolicy* policy_; + std::string filter_; + std::vector keys_; + + public: + BloomTest() : policy_(NewBloomFilterPolicy(10)) { } + + ~BloomTest() { + delete policy_; + } + + void Reset() { + keys_.clear(); + filter_.clear(); + } + + void Add(const Slice& s) { + keys_.push_back(s.ToString()); + } + + void Build() { + std::vector key_slices; + for (size_t i = 0; i < keys_.size(); i++) { + key_slices.push_back(Slice(keys_[i])); + } + filter_.clear(); + policy_->CreateFilter(&key_slices[0], key_slices.size(), &filter_); + keys_.clear(); + if (kVerbose >= 2) DumpFilter(); + } + + size_t FilterSize() const { + return filter_.size(); + } + + void DumpFilter() { + fprintf(stderr, "F("); + for (size_t i = 0; i+1 < filter_.size(); i++) { + const unsigned int c = static_cast(filter_[i]); + for (int j = 0; j < 8; j++) { + fprintf(stderr, "%c", (c & (1 <KeyMayMatch(s, filter_); + } + + double FalsePositiveRate() { + char buffer[sizeof(int)]; + int result = 0; + for (int i = 0; i < 10000; i++) { + if (Matches(Key(i + 1000000000, buffer))) { + result++; + } + } + return result / 10000.0; + } +}; + +TEST(BloomTest, EmptyFilter) { + ASSERT_TRUE(! Matches("hello")); + ASSERT_TRUE(! Matches("world")); +} + +TEST(BloomTest, Small) { + Add("hello"); + Add("world"); + ASSERT_TRUE(Matches("hello")); + ASSERT_TRUE(Matches("world")); + ASSERT_TRUE(! Matches("x")); + ASSERT_TRUE(! Matches("foo")); +} + +static int NextLength(int length) { + if (length < 10) { + length += 1; + } else if (length < 100) { + length += 10; + } else if (length < 1000) { + length += 100; + } else { + length += 1000; + } + return length; +} + +TEST(BloomTest, VaryingLengths) { + char buffer[sizeof(int)]; + + // Count number of filters that significantly exceed the false positive rate + int mediocre_filters = 0; + int good_filters = 0; + + for (int length = 1; length <= 10000; length = NextLength(length)) { + Reset(); + for (int i = 0; i < length; i++) { + Add(Key(i, buffer)); + } + Build(); + + ASSERT_LE(FilterSize(), (length * 10 / 8) + 40) << length; + + // All added keys must match + for (int i = 0; i < length; i++) { + ASSERT_TRUE(Matches(Key(i, buffer))) + << "Length " << length << "; key " << i; + } + + // Check false positive rate + double rate = FalsePositiveRate(); + if (kVerbose >= 1) { + fprintf(stderr, "False positives: %5.2f%% @ length = %6d ; bytes = %6d\n", + rate*100.0, length, static_cast(FilterSize())); + } + ASSERT_LE(rate, 0.02); // Must not be over 2% + if (rate > 0.0125) mediocre_filters++; // Allowed, but not too often + else good_filters++; + } + if (kVerbose >= 1) { + fprintf(stderr, "Filters: %d good, %d mediocre\n", + good_filters, mediocre_filters); + } + ASSERT_LE(mediocre_filters, good_filters/5); +} + +// Different bits-per-byte + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/util/cache.cc b/Subtrees/hyperleveldb/util/cache.cc new file mode 100644 index 0000000000..18427f7730 --- /dev/null +++ b/Subtrees/hyperleveldb/util/cache.cc @@ -0,0 +1,325 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include + +#include "../hyperleveldb/cache.h" +#include "../port/port.h" +#include "hash.h" +#include "mutexlock.h" + +namespace hyperleveldb { + +Cache::~Cache() { +} + +namespace { + +// LRU cache implementation + +// An entry is a variable length heap-allocated structure. Entries +// are kept in a circular doubly linked list ordered by access time. +struct LRUHandle { + void* value; + void (*deleter)(const Slice&, void* value); + LRUHandle* next_hash; + LRUHandle* next; + LRUHandle* prev; + size_t charge; // TODO(opt): Only allow uint32_t? + size_t key_length; + uint32_t refs; + uint32_t hash; // Hash of key(); used for fast sharding and comparisons + char key_data[1]; // Beginning of key + + Slice key() const { + // For cheaper lookups, we allow a temporary Handle object + // to store a pointer to a key in "value". + if (next == this) { + return *(reinterpret_cast(value)); + } else { + return Slice(key_data, key_length); + } + } +}; + +// We provide our own simple hash table since it removes a whole bunch +// of porting hacks and is also faster than some of the built-in hash +// table implementations in some of the compiler/runtime combinations +// we have tested. E.g., readrandom speeds up by ~5% over the g++ +// 4.4.3's builtin hashtable. +class HandleTable { + public: + HandleTable() : length_(0), elems_(0), list_(NULL) { Resize(); } + ~HandleTable() { delete[] list_; } + + LRUHandle* Lookup(const Slice& key, uint32_t hash) { + return *FindPointer(key, hash); + } + + LRUHandle* Insert(LRUHandle* h) { + LRUHandle** ptr = FindPointer(h->key(), h->hash); + LRUHandle* old = *ptr; + h->next_hash = (old == NULL ? NULL : old->next_hash); + *ptr = h; + if (old == NULL) { + ++elems_; + if (elems_ > length_) { + // Since each cache entry is fairly large, we aim for a small + // average linked list length (<= 1). + Resize(); + } + } + return old; + } + + LRUHandle* Remove(const Slice& key, uint32_t hash) { + LRUHandle** ptr = FindPointer(key, hash); + LRUHandle* result = *ptr; + if (result != NULL) { + *ptr = result->next_hash; + --elems_; + } + return result; + } + + private: + // The table consists of an array of buckets where each bucket is + // a linked list of cache entries that hash into the bucket. + uint32_t length_; + uint32_t elems_; + LRUHandle** list_; + + // Return a pointer to slot that points to a cache entry that + // matches key/hash. If there is no such cache entry, return a + // pointer to the trailing slot in the corresponding linked list. + LRUHandle** FindPointer(const Slice& key, uint32_t hash) { + LRUHandle** ptr = &list_[hash & (length_ - 1)]; + while (*ptr != NULL && + ((*ptr)->hash != hash || key != (*ptr)->key())) { + ptr = &(*ptr)->next_hash; + } + return ptr; + } + + void Resize() { + uint32_t new_length = 4; + while (new_length < elems_) { + new_length *= 2; + } + LRUHandle** new_list = new LRUHandle*[new_length]; + memset(new_list, 0, sizeof(new_list[0]) * new_length); + uint32_t count = 0; + for (uint32_t i = 0; i < length_; i++) { + LRUHandle* h = list_[i]; + while (h != NULL) { + LRUHandle* next = h->next_hash; + uint32_t hash = h->hash; + LRUHandle** ptr = &new_list[hash & (new_length - 1)]; + h->next_hash = *ptr; + *ptr = h; + h = next; + count++; + } + } + assert(elems_ == count); + delete[] list_; + list_ = new_list; + length_ = new_length; + } +}; + +// A single shard of sharded cache. +class LRUCache { + public: + LRUCache(); + ~LRUCache(); + + // Separate from constructor so caller can easily make an array of LRUCache + void SetCapacity(size_t capacity) { capacity_ = capacity; } + + // Like Cache methods, but with an extra "hash" parameter. + Cache::Handle* Insert(const Slice& key, uint32_t hash, + void* value, size_t charge, + void (*deleter)(const Slice& key, void* value)); + Cache::Handle* Lookup(const Slice& key, uint32_t hash); + void Release(Cache::Handle* handle); + void Erase(const Slice& key, uint32_t hash); + + private: + void LRU_Remove(LRUHandle* e); + void LRU_Append(LRUHandle* e); + void Unref(LRUHandle* e); + + // Initialized before use. + size_t capacity_; + + // mutex_ protects the following state. + port::Mutex mutex_; + size_t usage_; + + // Dummy head of LRU list. + // lru.prev is newest entry, lru.next is oldest entry. + LRUHandle lru_; + + HandleTable table_; +}; + +LRUCache::LRUCache() + : usage_(0) { + // Make empty circular linked list + lru_.next = &lru_; + lru_.prev = &lru_; +} + +LRUCache::~LRUCache() { + for (LRUHandle* e = lru_.next; e != &lru_; ) { + LRUHandle* next = e->next; + assert(e->refs == 1); // Error if caller has an unreleased handle + Unref(e); + e = next; + } +} + +void LRUCache::Unref(LRUHandle* e) { + assert(e->refs > 0); + e->refs--; + if (e->refs <= 0) { + usage_ -= e->charge; + (*e->deleter)(e->key(), e->value); + free(e); + } +} + +void LRUCache::LRU_Remove(LRUHandle* e) { + e->next->prev = e->prev; + e->prev->next = e->next; +} + +void LRUCache::LRU_Append(LRUHandle* e) { + // Make "e" newest entry by inserting just before lru_ + e->next = &lru_; + e->prev = lru_.prev; + e->prev->next = e; + e->next->prev = e; +} + +Cache::Handle* LRUCache::Lookup(const Slice& key, uint32_t hash) { + MutexLock l(&mutex_); + LRUHandle* e = table_.Lookup(key, hash); + if (e != NULL) { + e->refs++; + LRU_Remove(e); + LRU_Append(e); + } + return reinterpret_cast(e); +} + +void LRUCache::Release(Cache::Handle* handle) { + MutexLock l(&mutex_); + Unref(reinterpret_cast(handle)); +} + +Cache::Handle* LRUCache::Insert( + const Slice& key, uint32_t hash, void* value, size_t charge, + void (*deleter)(const Slice& key, void* value)) { + MutexLock l(&mutex_); + + LRUHandle* e = reinterpret_cast( + malloc(sizeof(LRUHandle)-1 + key.size())); + e->value = value; + e->deleter = deleter; + e->charge = charge; + e->key_length = key.size(); + e->hash = hash; + e->refs = 2; // One from LRUCache, one for the returned handle + memcpy(e->key_data, key.data(), key.size()); + LRU_Append(e); + usage_ += charge; + + LRUHandle* old = table_.Insert(e); + if (old != NULL) { + LRU_Remove(old); + Unref(old); + } + + while (usage_ > capacity_ && lru_.next != &lru_) { + LRUHandle* old = lru_.next; + LRU_Remove(old); + table_.Remove(old->key(), old->hash); + Unref(old); + } + + return reinterpret_cast(e); +} + +void LRUCache::Erase(const Slice& key, uint32_t hash) { + MutexLock l(&mutex_); + LRUHandle* e = table_.Remove(key, hash); + if (e != NULL) { + LRU_Remove(e); + Unref(e); + } +} + +static const int kNumShardBits = 4; +static const int kNumShards = 1 << kNumShardBits; + +class ShardedLRUCache : public Cache { + private: + LRUCache shard_[kNumShards]; + port::Mutex id_mutex_; + uint64_t last_id_; + + static inline uint32_t HashSlice(const Slice& s) { + return Hash(s.data(), s.size(), 0); + } + + static uint32_t Shard(uint32_t hash) { + return hash >> (32 - kNumShardBits); + } + + public: + explicit ShardedLRUCache(size_t capacity) + : last_id_(0) { + const size_t per_shard = (capacity + (kNumShards - 1)) / kNumShards; + for (int s = 0; s < kNumShards; s++) { + shard_[s].SetCapacity(per_shard); + } + } + virtual ~ShardedLRUCache() { } + virtual Handle* Insert(const Slice& key, void* value, size_t charge, + void (*deleter)(const Slice& key, void* value)) { + const uint32_t hash = HashSlice(key); + return shard_[Shard(hash)].Insert(key, hash, value, charge, deleter); + } + virtual Handle* Lookup(const Slice& key) { + const uint32_t hash = HashSlice(key); + return shard_[Shard(hash)].Lookup(key, hash); + } + virtual void Release(Handle* handle) { + LRUHandle* h = reinterpret_cast(handle); + shard_[Shard(h->hash)].Release(handle); + } + virtual void Erase(const Slice& key) { + const uint32_t hash = HashSlice(key); + shard_[Shard(hash)].Erase(key, hash); + } + virtual void* Value(Handle* handle) { + return reinterpret_cast(handle)->value; + } + virtual uint64_t NewId() { + MutexLock l(&id_mutex_); + return ++(last_id_); + } +}; + +} // end anonymous namespace + +Cache* NewLRUCache(size_t capacity) { + return new ShardedLRUCache(capacity); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/cache_test.cc b/Subtrees/hyperleveldb/util/cache_test.cc new file mode 100644 index 0000000000..40be3913e2 --- /dev/null +++ b/Subtrees/hyperleveldb/util/cache_test.cc @@ -0,0 +1,186 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/cache.h" + +#include +#include "coding.h" +#include "testharness.h" + +namespace hyperleveldb { + +// Conversions between numeric keys/values and the types expected by Cache. +static std::string EncodeKey(int k) { + std::string result; + PutFixed32(&result, k); + return result; +} +static int DecodeKey(const Slice& k) { + assert(k.size() == 4); + return DecodeFixed32(k.data()); +} +static void* EncodeValue(uintptr_t v) { return reinterpret_cast(v); } +static int DecodeValue(void* v) { return reinterpret_cast(v); } + +class CacheTest { + public: + static CacheTest* current_; + + static void Deleter(const Slice& key, void* v) { + current_->deleted_keys_.push_back(DecodeKey(key)); + current_->deleted_values_.push_back(DecodeValue(v)); + } + + static const int kCacheSize = 1000; + std::vector deleted_keys_; + std::vector deleted_values_; + Cache* cache_; + + CacheTest() : cache_(NewLRUCache(kCacheSize)) { + current_ = this; + } + + ~CacheTest() { + delete cache_; + } + + int Lookup(int key) { + Cache::Handle* handle = cache_->Lookup(EncodeKey(key)); + const int r = (handle == NULL) ? -1 : DecodeValue(cache_->Value(handle)); + if (handle != NULL) { + cache_->Release(handle); + } + return r; + } + + void Insert(int key, int value, int charge = 1) { + cache_->Release(cache_->Insert(EncodeKey(key), EncodeValue(value), charge, + &CacheTest::Deleter)); + } + + void Erase(int key) { + cache_->Erase(EncodeKey(key)); + } +}; +CacheTest* CacheTest::current_; + +TEST(CacheTest, HitAndMiss) { + ASSERT_EQ(-1, Lookup(100)); + + Insert(100, 101); + ASSERT_EQ(101, Lookup(100)); + ASSERT_EQ(-1, Lookup(200)); + ASSERT_EQ(-1, Lookup(300)); + + Insert(200, 201); + ASSERT_EQ(101, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(-1, Lookup(300)); + + Insert(100, 102); + ASSERT_EQ(102, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(-1, Lookup(300)); + + ASSERT_EQ(1, deleted_keys_.size()); + ASSERT_EQ(100, deleted_keys_[0]); + ASSERT_EQ(101, deleted_values_[0]); +} + +TEST(CacheTest, Erase) { + Erase(200); + ASSERT_EQ(0, deleted_keys_.size()); + + Insert(100, 101); + Insert(200, 201); + Erase(100); + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(1, deleted_keys_.size()); + ASSERT_EQ(100, deleted_keys_[0]); + ASSERT_EQ(101, deleted_values_[0]); + + Erase(100); + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(1, deleted_keys_.size()); +} + +TEST(CacheTest, EntriesArePinned) { + Insert(100, 101); + Cache::Handle* h1 = cache_->Lookup(EncodeKey(100)); + ASSERT_EQ(101, DecodeValue(cache_->Value(h1))); + + Insert(100, 102); + Cache::Handle* h2 = cache_->Lookup(EncodeKey(100)); + ASSERT_EQ(102, DecodeValue(cache_->Value(h2))); + ASSERT_EQ(0, deleted_keys_.size()); + + cache_->Release(h1); + ASSERT_EQ(1, deleted_keys_.size()); + ASSERT_EQ(100, deleted_keys_[0]); + ASSERT_EQ(101, deleted_values_[0]); + + Erase(100); + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(1, deleted_keys_.size()); + + cache_->Release(h2); + ASSERT_EQ(2, deleted_keys_.size()); + ASSERT_EQ(100, deleted_keys_[1]); + ASSERT_EQ(102, deleted_values_[1]); +} + +TEST(CacheTest, EvictionPolicy) { + Insert(100, 101); + Insert(200, 201); + + // Frequently used entry must be kept around + for (int i = 0; i < kCacheSize + 100; i++) { + Insert(1000+i, 2000+i); + ASSERT_EQ(2000+i, Lookup(1000+i)); + ASSERT_EQ(101, Lookup(100)); + } + ASSERT_EQ(101, Lookup(100)); + ASSERT_EQ(-1, Lookup(200)); +} + +TEST(CacheTest, HeavyEntries) { + // Add a bunch of light and heavy entries and then count the combined + // size of items still in the cache, which must be approximately the + // same as the total capacity. + const int kLight = 1; + const int kHeavy = 10; + int added = 0; + int index = 0; + while (added < 2*kCacheSize) { + const int weight = (index & 1) ? kLight : kHeavy; + Insert(index, 1000+index, weight); + added += weight; + index++; + } + + int cached_weight = 0; + for (int i = 0; i < index; i++) { + const int weight = (i & 1 ? kLight : kHeavy); + int r = Lookup(i); + if (r >= 0) { + cached_weight += weight; + ASSERT_EQ(1000+i, r); + } + } + ASSERT_LE(cached_weight, kCacheSize + kCacheSize/10); +} + +TEST(CacheTest, NewId) { + uint64_t a = cache_->NewId(); + uint64_t b = cache_->NewId(); + ASSERT_NE(a, b); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/util/coding.cc b/Subtrees/hyperleveldb/util/coding.cc new file mode 100644 index 0000000000..587cdcce5e --- /dev/null +++ b/Subtrees/hyperleveldb/util/coding.cc @@ -0,0 +1,194 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "coding.h" + +namespace hyperleveldb { + +void EncodeFixed32(char* buf, uint32_t value) { + if (port::kLittleEndian) { + memcpy(buf, &value, sizeof(value)); + } else { + buf[0] = value & 0xff; + buf[1] = (value >> 8) & 0xff; + buf[2] = (value >> 16) & 0xff; + buf[3] = (value >> 24) & 0xff; + } +} + +void EncodeFixed64(char* buf, uint64_t value) { + if (port::kLittleEndian) { + memcpy(buf, &value, sizeof(value)); + } else { + buf[0] = value & 0xff; + buf[1] = (value >> 8) & 0xff; + buf[2] = (value >> 16) & 0xff; + buf[3] = (value >> 24) & 0xff; + buf[4] = (value >> 32) & 0xff; + buf[5] = (value >> 40) & 0xff; + buf[6] = (value >> 48) & 0xff; + buf[7] = (value >> 56) & 0xff; + } +} + +void PutFixed32(std::string* dst, uint32_t value) { + char buf[sizeof(value)]; + EncodeFixed32(buf, value); + dst->append(buf, sizeof(buf)); +} + +void PutFixed64(std::string* dst, uint64_t value) { + char buf[sizeof(value)]; + EncodeFixed64(buf, value); + dst->append(buf, sizeof(buf)); +} + +char* EncodeVarint32(char* dst, uint32_t v) { + // Operate on characters as unsigneds + unsigned char* ptr = reinterpret_cast(dst); + static const int B = 128; + if (v < (1<<7)) { + *(ptr++) = v; + } else if (v < (1<<14)) { + *(ptr++) = v | B; + *(ptr++) = v>>7; + } else if (v < (1<<21)) { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = v>>14; + } else if (v < (1<<28)) { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = (v>>14) | B; + *(ptr++) = v>>21; + } else { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = (v>>14) | B; + *(ptr++) = (v>>21) | B; + *(ptr++) = v>>28; + } + return reinterpret_cast(ptr); +} + +void PutVarint32(std::string* dst, uint32_t v) { + char buf[5]; + char* ptr = EncodeVarint32(buf, v); + dst->append(buf, ptr - buf); +} + +char* EncodeVarint64(char* dst, uint64_t v) { + static const int B = 128; + unsigned char* ptr = reinterpret_cast(dst); + while (v >= B) { + *(ptr++) = (v & (B-1)) | B; + v >>= 7; + } + *(ptr++) = static_cast(v); + return reinterpret_cast(ptr); +} + +void PutVarint64(std::string* dst, uint64_t v) { + char buf[10]; + char* ptr = EncodeVarint64(buf, v); + dst->append(buf, ptr - buf); +} + +void PutLengthPrefixedSlice(std::string* dst, const Slice& value) { + PutVarint32(dst, value.size()); + dst->append(value.data(), value.size()); +} + +int VarintLength(uint64_t v) { + int len = 1; + while (v >= 128) { + v >>= 7; + len++; + } + return len; +} + +const char* GetVarint32PtrFallback(const char* p, + const char* limit, + uint32_t* value) { + uint32_t result = 0; + for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) { + uint32_t byte = *(reinterpret_cast(p)); + p++; + if (byte & 128) { + // More bytes are present + result |= ((byte & 127) << shift); + } else { + result |= (byte << shift); + *value = result; + return reinterpret_cast(p); + } + } + return NULL; +} + +bool GetVarint32(Slice* input, uint32_t* value) { + const char* p = input->data(); + const char* limit = p + input->size(); + const char* q = GetVarint32Ptr(p, limit, value); + if (q == NULL) { + return false; + } else { + *input = Slice(q, limit - q); + return true; + } +} + +const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) { + uint64_t result = 0; + for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) { + uint64_t byte = *(reinterpret_cast(p)); + p++; + if (byte & 128) { + // More bytes are present + result |= ((byte & 127) << shift); + } else { + result |= (byte << shift); + *value = result; + return reinterpret_cast(p); + } + } + return NULL; +} + +bool GetVarint64(Slice* input, uint64_t* value) { + const char* p = input->data(); + const char* limit = p + input->size(); + const char* q = GetVarint64Ptr(p, limit, value); + if (q == NULL) { + return false; + } else { + *input = Slice(q, limit - q); + return true; + } +} + +const char* GetLengthPrefixedSlice(const char* p, const char* limit, + Slice* result) { + uint32_t len; + p = GetVarint32Ptr(p, limit, &len); + if (p == NULL) return NULL; + if (p + len > limit) return NULL; + *result = Slice(p, len); + return p + len; +} + +bool GetLengthPrefixedSlice(Slice* input, Slice* result) { + uint32_t len; + if (GetVarint32(input, &len) && + input->size() >= len) { + *result = Slice(input->data(), len); + input->remove_prefix(len); + return true; + } else { + return false; + } +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/coding.h b/Subtrees/hyperleveldb/util/coding.h new file mode 100644 index 0000000000..970f337061 --- /dev/null +++ b/Subtrees/hyperleveldb/util/coding.h @@ -0,0 +1,104 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Endian-neutral encoding: +// * Fixed-length numbers are encoded with least-significant byte first +// * In addition we support variable length "varint" encoding +// * Strings are encoded prefixed by their length in varint format + +#ifndef STORAGE_HYPERLEVELDB_UTIL_CODING_H_ +#define STORAGE_HYPERLEVELDB_UTIL_CODING_H_ + +#include +#include +#include +#include "../hyperleveldb/slice.h" +#include "../port/port.h" + +namespace hyperleveldb { + +// Standard Put... routines append to a string +extern void PutFixed32(std::string* dst, uint32_t value); +extern void PutFixed64(std::string* dst, uint64_t value); +extern void PutVarint32(std::string* dst, uint32_t value); +extern void PutVarint64(std::string* dst, uint64_t value); +extern void PutLengthPrefixedSlice(std::string* dst, const Slice& value); + +// Standard Get... routines parse a value from the beginning of a Slice +// and advance the slice past the parsed value. +extern bool GetVarint32(Slice* input, uint32_t* value); +extern bool GetVarint64(Slice* input, uint64_t* value); +extern bool GetLengthPrefixedSlice(Slice* input, Slice* result); + +// Pointer-based variants of GetVarint... These either store a value +// in *v and return a pointer just past the parsed value, or return +// NULL on error. These routines only look at bytes in the range +// [p..limit-1] +extern const char* GetVarint32Ptr(const char* p,const char* limit, uint32_t* v); +extern const char* GetVarint64Ptr(const char* p,const char* limit, uint64_t* v); + +// Returns the length of the varint32 or varint64 encoding of "v" +extern int VarintLength(uint64_t v); + +// Lower-level versions of Put... that write directly into a character buffer +// REQUIRES: dst has enough space for the value being written +extern void EncodeFixed32(char* dst, uint32_t value); +extern void EncodeFixed64(char* dst, uint64_t value); + +// Lower-level versions of Put... that write directly into a character buffer +// and return a pointer just past the last byte written. +// REQUIRES: dst has enough space for the value being written +extern char* EncodeVarint32(char* dst, uint32_t value); +extern char* EncodeVarint64(char* dst, uint64_t value); + +// Lower-level versions of Get... that read directly from a character buffer +// without any bounds checking. + +inline uint32_t DecodeFixed32(const char* ptr) { + if (port::kLittleEndian) { + // Load the raw bytes + uint32_t result; + memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load + return result; + } else { + return ((static_cast(static_cast(ptr[0]))) + | (static_cast(static_cast(ptr[1])) << 8) + | (static_cast(static_cast(ptr[2])) << 16) + | (static_cast(static_cast(ptr[3])) << 24)); + } +} + +inline uint64_t DecodeFixed64(const char* ptr) { + if (port::kLittleEndian) { + // Load the raw bytes + uint64_t result; + memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load + return result; + } else { + uint64_t lo = DecodeFixed32(ptr); + uint64_t hi = DecodeFixed32(ptr + 4); + return (hi << 32) | lo; + } +} + +// Internal routine for use by fallback path of GetVarint32Ptr +extern const char* GetVarint32PtrFallback(const char* p, + const char* limit, + uint32_t* value); +inline const char* GetVarint32Ptr(const char* p, + const char* limit, + uint32_t* value) { + if (p < limit) { + uint32_t result = *(reinterpret_cast(p)); + if ((result & 128) == 0) { + *value = result; + return p + 1; + } + } + return GetVarint32PtrFallback(p, limit, value); +} + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_CODING_H_ diff --git a/Subtrees/hyperleveldb/util/coding_test.cc b/Subtrees/hyperleveldb/util/coding_test.cc new file mode 100644 index 0000000000..58322a62b9 --- /dev/null +++ b/Subtrees/hyperleveldb/util/coding_test.cc @@ -0,0 +1,196 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "coding.h" + +#include "testharness.h" + +namespace hyperleveldb { + +class Coding { }; + +TEST(Coding, Fixed32) { + std::string s; + for (uint32_t v = 0; v < 100000; v++) { + PutFixed32(&s, v); + } + + const char* p = s.data(); + for (uint32_t v = 0; v < 100000; v++) { + uint32_t actual = DecodeFixed32(p); + ASSERT_EQ(v, actual); + p += sizeof(uint32_t); + } +} + +TEST(Coding, Fixed64) { + std::string s; + for (int power = 0; power <= 63; power++) { + uint64_t v = static_cast(1) << power; + PutFixed64(&s, v - 1); + PutFixed64(&s, v + 0); + PutFixed64(&s, v + 1); + } + + const char* p = s.data(); + for (int power = 0; power <= 63; power++) { + uint64_t v = static_cast(1) << power; + uint64_t actual; + actual = DecodeFixed64(p); + ASSERT_EQ(v-1, actual); + p += sizeof(uint64_t); + + actual = DecodeFixed64(p); + ASSERT_EQ(v+0, actual); + p += sizeof(uint64_t); + + actual = DecodeFixed64(p); + ASSERT_EQ(v+1, actual); + p += sizeof(uint64_t); + } +} + +// Test that encoding routines generate little-endian encodings +TEST(Coding, EncodingOutput) { + std::string dst; + PutFixed32(&dst, 0x04030201); + ASSERT_EQ(4, dst.size()); + ASSERT_EQ(0x01, static_cast(dst[0])); + ASSERT_EQ(0x02, static_cast(dst[1])); + ASSERT_EQ(0x03, static_cast(dst[2])); + ASSERT_EQ(0x04, static_cast(dst[3])); + + dst.clear(); + PutFixed64(&dst, 0x0807060504030201ull); + ASSERT_EQ(8, dst.size()); + ASSERT_EQ(0x01, static_cast(dst[0])); + ASSERT_EQ(0x02, static_cast(dst[1])); + ASSERT_EQ(0x03, static_cast(dst[2])); + ASSERT_EQ(0x04, static_cast(dst[3])); + ASSERT_EQ(0x05, static_cast(dst[4])); + ASSERT_EQ(0x06, static_cast(dst[5])); + ASSERT_EQ(0x07, static_cast(dst[6])); + ASSERT_EQ(0x08, static_cast(dst[7])); +} + +TEST(Coding, Varint32) { + std::string s; + for (uint32_t i = 0; i < (32 * 32); i++) { + uint32_t v = (i / 32) << (i % 32); + PutVarint32(&s, v); + } + + const char* p = s.data(); + const char* limit = p + s.size(); + for (uint32_t i = 0; i < (32 * 32); i++) { + uint32_t expected = (i / 32) << (i % 32); + uint32_t actual; + const char* start = p; + p = GetVarint32Ptr(p, limit, &actual); + ASSERT_TRUE(p != NULL); + ASSERT_EQ(expected, actual); + ASSERT_EQ(VarintLength(actual), p - start); + } + ASSERT_EQ(p, s.data() + s.size()); +} + +TEST(Coding, Varint64) { + // Construct the list of values to check + std::vector values; + // Some special values + values.push_back(0); + values.push_back(100); + values.push_back(~static_cast(0)); + values.push_back(~static_cast(0) - 1); + for (uint32_t k = 0; k < 64; k++) { + // Test values near powers of two + const uint64_t power = 1ull << k; + values.push_back(power); + values.push_back(power-1); + values.push_back(power+1); + } + + std::string s; + for (int i = 0; i < values.size(); i++) { + PutVarint64(&s, values[i]); + } + + const char* p = s.data(); + const char* limit = p + s.size(); + for (int i = 0; i < values.size(); i++) { + ASSERT_TRUE(p < limit); + uint64_t actual; + const char* start = p; + p = GetVarint64Ptr(p, limit, &actual); + ASSERT_TRUE(p != NULL); + ASSERT_EQ(values[i], actual); + ASSERT_EQ(VarintLength(actual), p - start); + } + ASSERT_EQ(p, limit); + +} + +TEST(Coding, Varint32Overflow) { + uint32_t result; + std::string input("\x81\x82\x83\x84\x85\x11"); + ASSERT_TRUE(GetVarint32Ptr(input.data(), input.data() + input.size(), &result) + == NULL); +} + +TEST(Coding, Varint32Truncation) { + uint32_t large_value = (1u << 31) + 100; + std::string s; + PutVarint32(&s, large_value); + uint32_t result; + for (int len = 0; len < s.size() - 1; len++) { + ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + len, &result) == NULL); + } + ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + s.size(), &result) != NULL); + ASSERT_EQ(large_value, result); +} + +TEST(Coding, Varint64Overflow) { + uint64_t result; + std::string input("\x81\x82\x83\x84\x85\x81\x82\x83\x84\x85\x11"); + ASSERT_TRUE(GetVarint64Ptr(input.data(), input.data() + input.size(), &result) + == NULL); +} + +TEST(Coding, Varint64Truncation) { + uint64_t large_value = (1ull << 63) + 100ull; + std::string s; + PutVarint64(&s, large_value); + uint64_t result; + for (int len = 0; len < s.size() - 1; len++) { + ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + len, &result) == NULL); + } + ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + s.size(), &result) != NULL); + ASSERT_EQ(large_value, result); +} + +TEST(Coding, Strings) { + std::string s; + PutLengthPrefixedSlice(&s, Slice("")); + PutLengthPrefixedSlice(&s, Slice("foo")); + PutLengthPrefixedSlice(&s, Slice("bar")); + PutLengthPrefixedSlice(&s, Slice(std::string(200, 'x'))); + + Slice input(s); + Slice v; + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ("", v.ToString()); + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ("foo", v.ToString()); + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ("bar", v.ToString()); + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ(std::string(200, 'x'), v.ToString()); + ASSERT_EQ("", input.ToString()); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/util/comparator.cc b/Subtrees/hyperleveldb/util/comparator.cc new file mode 100644 index 0000000000..c6f6072a6d --- /dev/null +++ b/Subtrees/hyperleveldb/util/comparator.cc @@ -0,0 +1,81 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/slice.h" +#include "../port/port.h" +#include "logging.h" + +namespace hyperleveldb { + +Comparator::~Comparator() { } + +namespace { +class BytewiseComparatorImpl : public Comparator { + public: + BytewiseComparatorImpl() { } + + virtual const char* Name() const { + return "leveldb.BytewiseComparator"; + } + + virtual int Compare(const Slice& a, const Slice& b) const { + return a.compare(b); + } + + virtual void FindShortestSeparator( + std::string* start, + const Slice& limit) const { + // Find length of common prefix + size_t min_length = std::min(start->size(), limit.size()); + size_t diff_index = 0; + while ((diff_index < min_length) && + ((*start)[diff_index] == limit[diff_index])) { + diff_index++; + } + + if (diff_index >= min_length) { + // Do not shorten if one string is a prefix of the other + } else { + uint8_t diff_byte = static_cast((*start)[diff_index]); + if (diff_byte < static_cast(0xff) && + diff_byte + 1 < static_cast(limit[diff_index])) { + (*start)[diff_index]++; + start->resize(diff_index + 1); + assert(Compare(*start, limit) < 0); + } + } + } + + virtual void FindShortSuccessor(std::string* key) const { + // Find first character that can be incremented + size_t n = key->size(); + for (size_t i = 0; i < n; i++) { + const uint8_t byte = (*key)[i]; + if (byte != static_cast(0xff)) { + (*key)[i] = byte + 1; + key->resize(i+1); + return; + } + } + // *key is a run of 0xffs. Leave it alone. + } +}; +} // namespace + +static port::OnceType onceComparator = LEVELDB_ONCE_INIT; +static const Comparator* bytewise; + +static void InitModule() { + bytewise = new BytewiseComparatorImpl; +} + +const Comparator* BytewiseComparator() { + port::InitOnce(&onceComparator, InitModule); + return bytewise; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/crc32c.cc b/Subtrees/hyperleveldb/util/crc32c.cc new file mode 100644 index 0000000000..d3f80bbff7 --- /dev/null +++ b/Subtrees/hyperleveldb/util/crc32c.cc @@ -0,0 +1,332 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A portable implementation of crc32c, optimized to handle +// four bytes at a time. + +#include "crc32c.h" + +#include +#include "coding.h" + +namespace hyperleveldb { +namespace crc32c { + +static const uint32_t table0_[256] = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 +}; +static const uint32_t table1_[256] = { + 0x00000000, 0x13a29877, 0x274530ee, 0x34e7a899, + 0x4e8a61dc, 0x5d28f9ab, 0x69cf5132, 0x7a6dc945, + 0x9d14c3b8, 0x8eb65bcf, 0xba51f356, 0xa9f36b21, + 0xd39ea264, 0xc03c3a13, 0xf4db928a, 0xe7790afd, + 0x3fc5f181, 0x2c6769f6, 0x1880c16f, 0x0b225918, + 0x714f905d, 0x62ed082a, 0x560aa0b3, 0x45a838c4, + 0xa2d13239, 0xb173aa4e, 0x859402d7, 0x96369aa0, + 0xec5b53e5, 0xfff9cb92, 0xcb1e630b, 0xd8bcfb7c, + 0x7f8be302, 0x6c297b75, 0x58ced3ec, 0x4b6c4b9b, + 0x310182de, 0x22a31aa9, 0x1644b230, 0x05e62a47, + 0xe29f20ba, 0xf13db8cd, 0xc5da1054, 0xd6788823, + 0xac154166, 0xbfb7d911, 0x8b507188, 0x98f2e9ff, + 0x404e1283, 0x53ec8af4, 0x670b226d, 0x74a9ba1a, + 0x0ec4735f, 0x1d66eb28, 0x298143b1, 0x3a23dbc6, + 0xdd5ad13b, 0xcef8494c, 0xfa1fe1d5, 0xe9bd79a2, + 0x93d0b0e7, 0x80722890, 0xb4958009, 0xa737187e, + 0xff17c604, 0xecb55e73, 0xd852f6ea, 0xcbf06e9d, + 0xb19da7d8, 0xa23f3faf, 0x96d89736, 0x857a0f41, + 0x620305bc, 0x71a19dcb, 0x45463552, 0x56e4ad25, + 0x2c896460, 0x3f2bfc17, 0x0bcc548e, 0x186eccf9, + 0xc0d23785, 0xd370aff2, 0xe797076b, 0xf4359f1c, + 0x8e585659, 0x9dface2e, 0xa91d66b7, 0xbabffec0, + 0x5dc6f43d, 0x4e646c4a, 0x7a83c4d3, 0x69215ca4, + 0x134c95e1, 0x00ee0d96, 0x3409a50f, 0x27ab3d78, + 0x809c2506, 0x933ebd71, 0xa7d915e8, 0xb47b8d9f, + 0xce1644da, 0xddb4dcad, 0xe9537434, 0xfaf1ec43, + 0x1d88e6be, 0x0e2a7ec9, 0x3acdd650, 0x296f4e27, + 0x53028762, 0x40a01f15, 0x7447b78c, 0x67e52ffb, + 0xbf59d487, 0xacfb4cf0, 0x981ce469, 0x8bbe7c1e, + 0xf1d3b55b, 0xe2712d2c, 0xd69685b5, 0xc5341dc2, + 0x224d173f, 0x31ef8f48, 0x050827d1, 0x16aabfa6, + 0x6cc776e3, 0x7f65ee94, 0x4b82460d, 0x5820de7a, + 0xfbc3faf9, 0xe861628e, 0xdc86ca17, 0xcf245260, + 0xb5499b25, 0xa6eb0352, 0x920cabcb, 0x81ae33bc, + 0x66d73941, 0x7575a136, 0x419209af, 0x523091d8, + 0x285d589d, 0x3bffc0ea, 0x0f186873, 0x1cbaf004, + 0xc4060b78, 0xd7a4930f, 0xe3433b96, 0xf0e1a3e1, + 0x8a8c6aa4, 0x992ef2d3, 0xadc95a4a, 0xbe6bc23d, + 0x5912c8c0, 0x4ab050b7, 0x7e57f82e, 0x6df56059, + 0x1798a91c, 0x043a316b, 0x30dd99f2, 0x237f0185, + 0x844819fb, 0x97ea818c, 0xa30d2915, 0xb0afb162, + 0xcac27827, 0xd960e050, 0xed8748c9, 0xfe25d0be, + 0x195cda43, 0x0afe4234, 0x3e19eaad, 0x2dbb72da, + 0x57d6bb9f, 0x447423e8, 0x70938b71, 0x63311306, + 0xbb8de87a, 0xa82f700d, 0x9cc8d894, 0x8f6a40e3, + 0xf50789a6, 0xe6a511d1, 0xd242b948, 0xc1e0213f, + 0x26992bc2, 0x353bb3b5, 0x01dc1b2c, 0x127e835b, + 0x68134a1e, 0x7bb1d269, 0x4f567af0, 0x5cf4e287, + 0x04d43cfd, 0x1776a48a, 0x23910c13, 0x30339464, + 0x4a5e5d21, 0x59fcc556, 0x6d1b6dcf, 0x7eb9f5b8, + 0x99c0ff45, 0x8a626732, 0xbe85cfab, 0xad2757dc, + 0xd74a9e99, 0xc4e806ee, 0xf00fae77, 0xe3ad3600, + 0x3b11cd7c, 0x28b3550b, 0x1c54fd92, 0x0ff665e5, + 0x759baca0, 0x663934d7, 0x52de9c4e, 0x417c0439, + 0xa6050ec4, 0xb5a796b3, 0x81403e2a, 0x92e2a65d, + 0xe88f6f18, 0xfb2df76f, 0xcfca5ff6, 0xdc68c781, + 0x7b5fdfff, 0x68fd4788, 0x5c1aef11, 0x4fb87766, + 0x35d5be23, 0x26772654, 0x12908ecd, 0x013216ba, + 0xe64b1c47, 0xf5e98430, 0xc10e2ca9, 0xd2acb4de, + 0xa8c17d9b, 0xbb63e5ec, 0x8f844d75, 0x9c26d502, + 0x449a2e7e, 0x5738b609, 0x63df1e90, 0x707d86e7, + 0x0a104fa2, 0x19b2d7d5, 0x2d557f4c, 0x3ef7e73b, + 0xd98eedc6, 0xca2c75b1, 0xfecbdd28, 0xed69455f, + 0x97048c1a, 0x84a6146d, 0xb041bcf4, 0xa3e32483 +}; +static const uint32_t table2_[256] = { + 0x00000000, 0xa541927e, 0x4f6f520d, 0xea2ec073, + 0x9edea41a, 0x3b9f3664, 0xd1b1f617, 0x74f06469, + 0x38513ec5, 0x9d10acbb, 0x773e6cc8, 0xd27ffeb6, + 0xa68f9adf, 0x03ce08a1, 0xe9e0c8d2, 0x4ca15aac, + 0x70a27d8a, 0xd5e3eff4, 0x3fcd2f87, 0x9a8cbdf9, + 0xee7cd990, 0x4b3d4bee, 0xa1138b9d, 0x045219e3, + 0x48f3434f, 0xedb2d131, 0x079c1142, 0xa2dd833c, + 0xd62de755, 0x736c752b, 0x9942b558, 0x3c032726, + 0xe144fb14, 0x4405696a, 0xae2ba919, 0x0b6a3b67, + 0x7f9a5f0e, 0xdadbcd70, 0x30f50d03, 0x95b49f7d, + 0xd915c5d1, 0x7c5457af, 0x967a97dc, 0x333b05a2, + 0x47cb61cb, 0xe28af3b5, 0x08a433c6, 0xade5a1b8, + 0x91e6869e, 0x34a714e0, 0xde89d493, 0x7bc846ed, + 0x0f382284, 0xaa79b0fa, 0x40577089, 0xe516e2f7, + 0xa9b7b85b, 0x0cf62a25, 0xe6d8ea56, 0x43997828, + 0x37691c41, 0x92288e3f, 0x78064e4c, 0xdd47dc32, + 0xc76580d9, 0x622412a7, 0x880ad2d4, 0x2d4b40aa, + 0x59bb24c3, 0xfcfab6bd, 0x16d476ce, 0xb395e4b0, + 0xff34be1c, 0x5a752c62, 0xb05bec11, 0x151a7e6f, + 0x61ea1a06, 0xc4ab8878, 0x2e85480b, 0x8bc4da75, + 0xb7c7fd53, 0x12866f2d, 0xf8a8af5e, 0x5de93d20, + 0x29195949, 0x8c58cb37, 0x66760b44, 0xc337993a, + 0x8f96c396, 0x2ad751e8, 0xc0f9919b, 0x65b803e5, + 0x1148678c, 0xb409f5f2, 0x5e273581, 0xfb66a7ff, + 0x26217bcd, 0x8360e9b3, 0x694e29c0, 0xcc0fbbbe, + 0xb8ffdfd7, 0x1dbe4da9, 0xf7908dda, 0x52d11fa4, + 0x1e704508, 0xbb31d776, 0x511f1705, 0xf45e857b, + 0x80aee112, 0x25ef736c, 0xcfc1b31f, 0x6a802161, + 0x56830647, 0xf3c29439, 0x19ec544a, 0xbcadc634, + 0xc85da25d, 0x6d1c3023, 0x8732f050, 0x2273622e, + 0x6ed23882, 0xcb93aafc, 0x21bd6a8f, 0x84fcf8f1, + 0xf00c9c98, 0x554d0ee6, 0xbf63ce95, 0x1a225ceb, + 0x8b277743, 0x2e66e53d, 0xc448254e, 0x6109b730, + 0x15f9d359, 0xb0b84127, 0x5a968154, 0xffd7132a, + 0xb3764986, 0x1637dbf8, 0xfc191b8b, 0x595889f5, + 0x2da8ed9c, 0x88e97fe2, 0x62c7bf91, 0xc7862def, + 0xfb850ac9, 0x5ec498b7, 0xb4ea58c4, 0x11abcaba, + 0x655baed3, 0xc01a3cad, 0x2a34fcde, 0x8f756ea0, + 0xc3d4340c, 0x6695a672, 0x8cbb6601, 0x29faf47f, + 0x5d0a9016, 0xf84b0268, 0x1265c21b, 0xb7245065, + 0x6a638c57, 0xcf221e29, 0x250cde5a, 0x804d4c24, + 0xf4bd284d, 0x51fcba33, 0xbbd27a40, 0x1e93e83e, + 0x5232b292, 0xf77320ec, 0x1d5de09f, 0xb81c72e1, + 0xccec1688, 0x69ad84f6, 0x83834485, 0x26c2d6fb, + 0x1ac1f1dd, 0xbf8063a3, 0x55aea3d0, 0xf0ef31ae, + 0x841f55c7, 0x215ec7b9, 0xcb7007ca, 0x6e3195b4, + 0x2290cf18, 0x87d15d66, 0x6dff9d15, 0xc8be0f6b, + 0xbc4e6b02, 0x190ff97c, 0xf321390f, 0x5660ab71, + 0x4c42f79a, 0xe90365e4, 0x032da597, 0xa66c37e9, + 0xd29c5380, 0x77ddc1fe, 0x9df3018d, 0x38b293f3, + 0x7413c95f, 0xd1525b21, 0x3b7c9b52, 0x9e3d092c, + 0xeacd6d45, 0x4f8cff3b, 0xa5a23f48, 0x00e3ad36, + 0x3ce08a10, 0x99a1186e, 0x738fd81d, 0xd6ce4a63, + 0xa23e2e0a, 0x077fbc74, 0xed517c07, 0x4810ee79, + 0x04b1b4d5, 0xa1f026ab, 0x4bdee6d8, 0xee9f74a6, + 0x9a6f10cf, 0x3f2e82b1, 0xd50042c2, 0x7041d0bc, + 0xad060c8e, 0x08479ef0, 0xe2695e83, 0x4728ccfd, + 0x33d8a894, 0x96993aea, 0x7cb7fa99, 0xd9f668e7, + 0x9557324b, 0x3016a035, 0xda386046, 0x7f79f238, + 0x0b899651, 0xaec8042f, 0x44e6c45c, 0xe1a75622, + 0xdda47104, 0x78e5e37a, 0x92cb2309, 0x378ab177, + 0x437ad51e, 0xe63b4760, 0x0c158713, 0xa954156d, + 0xe5f54fc1, 0x40b4ddbf, 0xaa9a1dcc, 0x0fdb8fb2, + 0x7b2bebdb, 0xde6a79a5, 0x3444b9d6, 0x91052ba8 +}; +static const uint32_t table3_[256] = { + 0x00000000, 0xdd45aab8, 0xbf672381, 0x62228939, + 0x7b2231f3, 0xa6679b4b, 0xc4451272, 0x1900b8ca, + 0xf64463e6, 0x2b01c95e, 0x49234067, 0x9466eadf, + 0x8d665215, 0x5023f8ad, 0x32017194, 0xef44db2c, + 0xe964b13d, 0x34211b85, 0x560392bc, 0x8b463804, + 0x924680ce, 0x4f032a76, 0x2d21a34f, 0xf06409f7, + 0x1f20d2db, 0xc2657863, 0xa047f15a, 0x7d025be2, + 0x6402e328, 0xb9474990, 0xdb65c0a9, 0x06206a11, + 0xd725148b, 0x0a60be33, 0x6842370a, 0xb5079db2, + 0xac072578, 0x71428fc0, 0x136006f9, 0xce25ac41, + 0x2161776d, 0xfc24ddd5, 0x9e0654ec, 0x4343fe54, + 0x5a43469e, 0x8706ec26, 0xe524651f, 0x3861cfa7, + 0x3e41a5b6, 0xe3040f0e, 0x81268637, 0x5c632c8f, + 0x45639445, 0x98263efd, 0xfa04b7c4, 0x27411d7c, + 0xc805c650, 0x15406ce8, 0x7762e5d1, 0xaa274f69, + 0xb327f7a3, 0x6e625d1b, 0x0c40d422, 0xd1057e9a, + 0xaba65fe7, 0x76e3f55f, 0x14c17c66, 0xc984d6de, + 0xd0846e14, 0x0dc1c4ac, 0x6fe34d95, 0xb2a6e72d, + 0x5de23c01, 0x80a796b9, 0xe2851f80, 0x3fc0b538, + 0x26c00df2, 0xfb85a74a, 0x99a72e73, 0x44e284cb, + 0x42c2eeda, 0x9f874462, 0xfda5cd5b, 0x20e067e3, + 0x39e0df29, 0xe4a57591, 0x8687fca8, 0x5bc25610, + 0xb4868d3c, 0x69c32784, 0x0be1aebd, 0xd6a40405, + 0xcfa4bccf, 0x12e11677, 0x70c39f4e, 0xad8635f6, + 0x7c834b6c, 0xa1c6e1d4, 0xc3e468ed, 0x1ea1c255, + 0x07a17a9f, 0xdae4d027, 0xb8c6591e, 0x6583f3a6, + 0x8ac7288a, 0x57828232, 0x35a00b0b, 0xe8e5a1b3, + 0xf1e51979, 0x2ca0b3c1, 0x4e823af8, 0x93c79040, + 0x95e7fa51, 0x48a250e9, 0x2a80d9d0, 0xf7c57368, + 0xeec5cba2, 0x3380611a, 0x51a2e823, 0x8ce7429b, + 0x63a399b7, 0xbee6330f, 0xdcc4ba36, 0x0181108e, + 0x1881a844, 0xc5c402fc, 0xa7e68bc5, 0x7aa3217d, + 0x52a0c93f, 0x8fe56387, 0xedc7eabe, 0x30824006, + 0x2982f8cc, 0xf4c75274, 0x96e5db4d, 0x4ba071f5, + 0xa4e4aad9, 0x79a10061, 0x1b838958, 0xc6c623e0, + 0xdfc69b2a, 0x02833192, 0x60a1b8ab, 0xbde41213, + 0xbbc47802, 0x6681d2ba, 0x04a35b83, 0xd9e6f13b, + 0xc0e649f1, 0x1da3e349, 0x7f816a70, 0xa2c4c0c8, + 0x4d801be4, 0x90c5b15c, 0xf2e73865, 0x2fa292dd, + 0x36a22a17, 0xebe780af, 0x89c50996, 0x5480a32e, + 0x8585ddb4, 0x58c0770c, 0x3ae2fe35, 0xe7a7548d, + 0xfea7ec47, 0x23e246ff, 0x41c0cfc6, 0x9c85657e, + 0x73c1be52, 0xae8414ea, 0xcca69dd3, 0x11e3376b, + 0x08e38fa1, 0xd5a62519, 0xb784ac20, 0x6ac10698, + 0x6ce16c89, 0xb1a4c631, 0xd3864f08, 0x0ec3e5b0, + 0x17c35d7a, 0xca86f7c2, 0xa8a47efb, 0x75e1d443, + 0x9aa50f6f, 0x47e0a5d7, 0x25c22cee, 0xf8878656, + 0xe1873e9c, 0x3cc29424, 0x5ee01d1d, 0x83a5b7a5, + 0xf90696d8, 0x24433c60, 0x4661b559, 0x9b241fe1, + 0x8224a72b, 0x5f610d93, 0x3d4384aa, 0xe0062e12, + 0x0f42f53e, 0xd2075f86, 0xb025d6bf, 0x6d607c07, + 0x7460c4cd, 0xa9256e75, 0xcb07e74c, 0x16424df4, + 0x106227e5, 0xcd278d5d, 0xaf050464, 0x7240aedc, + 0x6b401616, 0xb605bcae, 0xd4273597, 0x09629f2f, + 0xe6264403, 0x3b63eebb, 0x59416782, 0x8404cd3a, + 0x9d0475f0, 0x4041df48, 0x22635671, 0xff26fcc9, + 0x2e238253, 0xf36628eb, 0x9144a1d2, 0x4c010b6a, + 0x5501b3a0, 0x88441918, 0xea669021, 0x37233a99, + 0xd867e1b5, 0x05224b0d, 0x6700c234, 0xba45688c, + 0xa345d046, 0x7e007afe, 0x1c22f3c7, 0xc167597f, + 0xc747336e, 0x1a0299d6, 0x782010ef, 0xa565ba57, + 0xbc65029d, 0x6120a825, 0x0302211c, 0xde478ba4, + 0x31035088, 0xec46fa30, 0x8e647309, 0x5321d9b1, + 0x4a21617b, 0x9764cbc3, 0xf54642fa, 0x2803e842 +}; + +// Used to fetch a naturally-aligned 32-bit word in little endian byte-order +static inline uint32_t LE_LOAD32(const uint8_t *p) { + return DecodeFixed32(reinterpret_cast(p)); +} + +uint32_t Extend(uint32_t crc, const char* buf, size_t size) { + const uint8_t *p = reinterpret_cast(buf); + const uint8_t *e = p + size; + uint32_t l = crc ^ 0xffffffffu; + +#define STEP1 do { \ + int c = (l & 0xff) ^ *p++; \ + l = table0_[c] ^ (l >> 8); \ +} while (0) +#define STEP4 do { \ + uint32_t c = l ^ LE_LOAD32(p); \ + p += 4; \ + l = table3_[c & 0xff] ^ \ + table2_[(c >> 8) & 0xff] ^ \ + table1_[(c >> 16) & 0xff] ^ \ + table0_[c >> 24]; \ +} while (0) + + // Point x at first 4-byte aligned byte in string. This might be + // just past the end of the string. + const uintptr_t pval = reinterpret_cast(p); + const uint8_t* x = reinterpret_cast(((pval + 3) >> 2) << 2); + if (x <= e) { + // Process bytes until finished or p is 4-byte aligned + while (p != x) { + STEP1; + } + } + // Process bytes 16 at a time + while ((e-p) >= 16) { + STEP4; STEP4; STEP4; STEP4; + } + // Process bytes 4 at a time + while ((e-p) >= 4) { + STEP4; + } + // Process the last few bytes + while (p != e) { + STEP1; + } +#undef STEP4 +#undef STEP1 + return l ^ 0xffffffffu; +} + +} // namespace crc32c +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/crc32c.h b/Subtrees/hyperleveldb/util/crc32c.h new file mode 100644 index 0000000000..1774409f4c --- /dev/null +++ b/Subtrees/hyperleveldb/util/crc32c.h @@ -0,0 +1,45 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_UTIL_CRC32C_H_ +#define STORAGE_HYPERLEVELDB_UTIL_CRC32C_H_ + +#include +#include + +namespace hyperleveldb { +namespace crc32c { + +// Return the crc32c of concat(A, data[0,n-1]) where init_crc is the +// crc32c of some string A. Extend() is often used to maintain the +// crc32c of a stream of data. +extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n); + +// Return the crc32c of data[0,n-1] +inline uint32_t Value(const char* data, size_t n) { + return Extend(0, data, n); +} + +static const uint32_t kMaskDelta = 0xa282ead8ul; + +// Return a masked representation of crc. +// +// Motivation: it is problematic to compute the CRC of a string that +// contains embedded CRCs. Therefore we recommend that CRCs stored +// somewhere (e.g., in files) should be masked before being stored. +inline uint32_t Mask(uint32_t crc) { + // Rotate right by 15 bits and add a constant. + return ((crc >> 15) | (crc << 17)) + kMaskDelta; +} + +// Return the crc whose masked representation is masked_crc. +inline uint32_t Unmask(uint32_t masked_crc) { + uint32_t rot = masked_crc - kMaskDelta; + return ((rot >> 17) | (rot << 15)); +} + +} // namespace crc32c +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_CRC32C_H_ diff --git a/Subtrees/hyperleveldb/util/crc32c_test.cc b/Subtrees/hyperleveldb/util/crc32c_test.cc new file mode 100644 index 0000000000..efa7fb6d94 --- /dev/null +++ b/Subtrees/hyperleveldb/util/crc32c_test.cc @@ -0,0 +1,72 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "crc32c.h" +#include "testharness.h" + +namespace hyperleveldb { +namespace crc32c { + +class CRC { }; + +TEST(CRC, StandardResults) { + // From rfc3720 section B.4. + char buf[32]; + + memset(buf, 0, sizeof(buf)); + ASSERT_EQ(0x8a9136aa, Value(buf, sizeof(buf))); + + memset(buf, 0xff, sizeof(buf)); + ASSERT_EQ(0x62a8ab43, Value(buf, sizeof(buf))); + + for (int i = 0; i < 32; i++) { + buf[i] = i; + } + ASSERT_EQ(0x46dd794e, Value(buf, sizeof(buf))); + + for (int i = 0; i < 32; i++) { + buf[i] = 31 - i; + } + ASSERT_EQ(0x113fdb5c, Value(buf, sizeof(buf))); + + unsigned char data[48] = { + 0x01, 0xc0, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x14, + 0x00, 0x00, 0x00, 0x18, + 0x28, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }; + ASSERT_EQ(0xd9963a56, Value(reinterpret_cast(data), sizeof(data))); +} + +TEST(CRC, Values) { + ASSERT_NE(Value("a", 1), Value("foo", 3)); +} + +TEST(CRC, Extend) { + ASSERT_EQ(Value("hello world", 11), + Extend(Value("hello ", 6), "world", 5)); +} + +TEST(CRC, Mask) { + uint32_t crc = Value("foo", 3); + ASSERT_NE(crc, Mask(crc)); + ASSERT_NE(crc, Mask(Mask(crc))); + ASSERT_EQ(crc, Unmask(Mask(crc))); + ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc))))); +} + +} // namespace crc32c +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/util/env.cc b/Subtrees/hyperleveldb/util/env.cc new file mode 100644 index 0000000000..8afb125b48 --- /dev/null +++ b/Subtrees/hyperleveldb/util/env.cc @@ -0,0 +1,96 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/env.h" + +namespace hyperleveldb { + +Env::~Env() { +} + +SequentialFile::~SequentialFile() { +} + +RandomAccessFile::~RandomAccessFile() { +} + +WritableFile::~WritableFile() { +} + +Logger::~Logger() { +} + +FileLock::~FileLock() { +} + +void Log(Logger* info_log, const char* format, ...) { + if (info_log != NULL) { + va_list ap; + va_start(ap, format); + info_log->Logv(format, ap); + va_end(ap); + } +} + +static Status DoWriteStringToFile(Env* env, const Slice& data, + const std::string& fname, + bool should_sync) { + WritableFile* file; + Status s = env->NewWritableFile(fname, &file); + if (!s.ok()) { + return s; + } + s = file->Append(data); + if (s.ok() && should_sync) { + s = file->Sync(); + } + if (s.ok()) { + s = file->Close(); + } + delete file; // Will auto-close if we did not close above + if (!s.ok()) { + env->DeleteFile(fname); + } + return s; +} + +Status WriteStringToFile(Env* env, const Slice& data, + const std::string& fname) { + return DoWriteStringToFile(env, data, fname, false); +} + +Status WriteStringToFileSync(Env* env, const Slice& data, + const std::string& fname) { + return DoWriteStringToFile(env, data, fname, true); +} + +Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { + data->clear(); + SequentialFile* file; + Status s = env->NewSequentialFile(fname, &file); + if (!s.ok()) { + return s; + } + static const int kBufferSize = 8192; + char* space = new char[kBufferSize]; + while (true) { + Slice fragment; + s = file->Read(kBufferSize, &fragment, space); + if (!s.ok()) { + break; + } + data->append(fragment.data(), fragment.size()); + if (fragment.empty()) { + break; + } + } + delete[] space; + delete file; + return s; +} + +EnvWrapper::~EnvWrapper() { +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/env_posix.cc b/Subtrees/hyperleveldb/util/env_posix.cc new file mode 100644 index 0000000000..c0346325dd --- /dev/null +++ b/Subtrees/hyperleveldb/util/env_posix.cc @@ -0,0 +1,698 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(LEVELDB_PLATFORM_ANDROID) +#include +#endif +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/slice.h" +#include "../port/port.h" +#include "logging.h" +#include "mutexlock.h" +#include "posix_logger.h" + +namespace hyperleveldb { + +namespace { + +static Status IOError(const std::string& context, int err_number) { + return Status::IOError(context, strerror(err_number)); +} + +class PosixSequentialFile: public SequentialFile { + private: + std::string filename_; + FILE* file_; + + public: + PosixSequentialFile(const std::string& fname, FILE* f) + : filename_(fname), file_(f) { } + virtual ~PosixSequentialFile() { fclose(file_); } + + virtual Status Read(size_t n, Slice* result, char* scratch) { + Status s; + size_t r = fread_unlocked(scratch, 1, n, file_); + *result = Slice(scratch, r); + if (r < n) { + if (feof(file_)) { + // We leave status as ok if we hit the end of the file + } else { + // A partial read with an error: return a non-ok status + s = IOError(filename_, errno); + } + } + return s; + } + + virtual Status Skip(uint64_t n) { + if (fseek(file_, n, SEEK_CUR)) { + return IOError(filename_, errno); + } + return Status::OK(); + } +}; + +// pread() based random-access +class PosixRandomAccessFile: public RandomAccessFile { + private: + std::string filename_; + int fd_; + + public: + PosixRandomAccessFile(const std::string& fname, int fd) + : filename_(fname), fd_(fd) { } + virtual ~PosixRandomAccessFile() { close(fd_); } + + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const { + Status s; + ssize_t r = pread(fd_, scratch, n, static_cast(offset)); + *result = Slice(scratch, (r < 0) ? 0 : r); + if (r < 0) { + // An error: return a non-ok status + s = IOError(filename_, errno); + } + return s; + } +}; + +// Helper class to limit mmap file usage so that we do not end up +// running out virtual memory or running into kernel performance +// problems for very large databases. +class MmapLimiter { + public: + // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes. + MmapLimiter() { + SetAllowed(sizeof(void*) >= 8 ? 1000 : 0); + } + + // If another mmap slot is available, acquire it and return true. + // Else return false. + bool Acquire() { + if (GetAllowed() <= 0) { + return false; + } + MutexLock l(&mu_); + intptr_t x = GetAllowed(); + if (x <= 0) { + return false; + } else { + SetAllowed(x - 1); + return true; + } + } + + // Release a slot acquired by a previous call to Acquire() that returned true. + void Release() { + MutexLock l(&mu_); + SetAllowed(GetAllowed() + 1); + } + + private: + port::Mutex mu_; + port::AtomicPointer allowed_; + + intptr_t GetAllowed() const { + return reinterpret_cast(allowed_.Acquire_Load()); + } + + // REQUIRES: mu_ must be held + void SetAllowed(intptr_t v) { + allowed_.Release_Store(reinterpret_cast(v)); + } + + MmapLimiter(const MmapLimiter&); + void operator=(const MmapLimiter&); +}; + +// mmap() based random-access +class PosixMmapReadableFile: public RandomAccessFile { + private: + std::string filename_; + void* mmapped_region_; + size_t length_; + MmapLimiter* limiter_; + + public: + // base[0,length-1] contains the mmapped contents of the file. + PosixMmapReadableFile(const std::string& fname, void* base, size_t length, + MmapLimiter* limiter) + : filename_(fname), mmapped_region_(base), length_(length), + limiter_(limiter) { + } + + virtual ~PosixMmapReadableFile() { + munmap(mmapped_region_, length_); + limiter_->Release(); + } + + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const { + Status s; + if (offset + n > length_) { + *result = Slice(); + s = IOError(filename_, EINVAL); + } else { + *result = Slice(reinterpret_cast(mmapped_region_) + offset, n); + } + return s; + } +}; + +// We preallocate up to an extra megabyte and use memcpy to append new +// data to the file. This is safe since we either properly close the +// file before reading from it, or for log files, the reading code +// knows enough to skip zero suffixes. +// TODO: I use GCC intrinsics here. I don't feel bad about this, but it +// hinders portability. +class PosixMmapFile : public WritableFile { + private: + struct MmapSegment { + MmapSegment* next_; // the next-lowest Map segment in the file + uint64_t file_offset_; // Offset of base_ in file + uint64_t written_; // The amount of data written to this segment + uint64_t size_; // The size of the mapped region + char* base_; // The mapped region + }; + + std::string filename_; // Path to the file + int fd_; // The open file + size_t page_size_; // System page size + uint64_t sync_offset_; // Offset of the last sync call + uint64_t end_offset_; // Where does the file end? + MmapSegment* segments_; // mmap'ed regions of memory + port::Mutex mtx_; // Synchronize and shit + + // Roundup x to a multiple of y + static size_t Roundup(size_t x, size_t y) { + return ((x + y - 1) / y) * y; + } + + MmapSegment* GetSegment(uint64_t offset) { + MutexLock l(&mtx_); + while (true) { + MmapSegment* seg = segments_; + while (seg && seg->file_offset_ > offset) { + seg = seg->next_; + } + if (!seg || seg->file_offset_ + seg->size_ <= offset) { + assert(seg == segments_); + MmapSegment* new_seg = new MmapSegment(); + new_seg->next_ = seg; + new_seg->file_offset_ = seg ? seg->file_offset_ + seg-> size_ : 0; + new_seg->written_ = 0; + new_seg->size_ = seg ? seg->size_ : Roundup(1 << 20, page_size_); + if (ftruncate(fd_, new_seg->file_offset_ + new_seg->size_) < 0) { + delete new_seg; + return NULL; + } + void* ptr = mmap(NULL, new_seg->size_, PROT_READ | PROT_WRITE, MAP_SHARED, + fd_, new_seg->file_offset_); + if (ptr == MAP_FAILED) { + delete new_seg; + return NULL; + } + new_seg->base_ = reinterpret_cast(ptr); + segments_ = new_seg; + continue; + } + assert(seg && + seg->file_offset_ <= offset && + seg->file_offset_ + seg->size_ > offset); + return seg; + } + } + + bool ReleaseSegment(MmapSegment* seg, bool full) { + return true; + } + + public: + PosixMmapFile(const std::string& fname, int fd, size_t page_size) + : filename_(fname), + fd_(fd), + page_size_(page_size), + sync_offset_(0), + end_offset_(0), + segments_(NULL), + mtx_() { + assert((page_size & (page_size - 1)) == 0); + } + + ~PosixMmapFile() { + if (fd_ >= 0) { + PosixMmapFile::Close(); + } + } + + virtual Status WriteAt(uint64_t offset, const Slice& data) { + uint64_t end = offset + data.size(); + const char* src = data.data(); + uint64_t left = data.size(); + while (left > 0) { + MmapSegment* seg = GetSegment(offset); + if (!seg) { + return IOError(filename_, errno); + } + + assert(offset >= seg->file_offset_); + assert(offset < seg->file_offset_ + seg->size_); + uint64_t local_offset = offset - seg->file_offset_; + uint64_t avail = seg->size_ - local_offset; + uint64_t n = (left <= avail) ? left : avail; + memcpy(seg->base_ + local_offset, src, n); + src += n; + left -= n; + offset += n; + uint64_t written = __sync_add_and_fetch(&seg->written_, n); + + if (!ReleaseSegment(seg, written == seg->size_)) { + return IOError(filename_, errno); + } + } + uint64_t old_end = end; + do { + old_end = __sync_val_compare_and_swap(&end_offset_, old_end, end); + } while (old_end < end); + return Status::OK(); + } + + virtual Status Append(const Slice& data) { + uint64_t offset = __sync_val_compare_and_swap(&end_offset_, 0, 0); + return WriteAt(offset, data); + } + + virtual Status Close() { + Status s; + while (segments_) { + MmapSegment* seg = segments_; + segments_ = seg->next_; + if (munmap(seg->base_, seg->size_) < 0) { + s = IOError(filename_, errno); + } + seg->base_ = NULL; + delete seg; + } + + if (ftruncate(fd_, end_offset_) < 0) { + s = IOError(filename_, errno); + } + + if (close(fd_) < 0) { + if (s.ok()) { + s = IOError(filename_, errno); + } + } + + fd_ = -1; + return s; + } + + virtual Status Sync() { + Status s; + bool need_sync = false; + + { + MutexLock l(&mtx_); + need_sync = sync_offset_ != end_offset_; + sync_offset_ = end_offset_; + } + + if (need_sync) { + // Some unmapped data was not synced + if (fdatasync(fd_) < 0) { + s = IOError(filename_, errno); + } + } + + return s; + } +}; + +static int LockOrUnlock(int fd, bool lock) { + errno = 0; + struct flock f; + memset(&f, 0, sizeof(f)); + f.l_type = (lock ? F_WRLCK : F_UNLCK); + f.l_whence = SEEK_SET; + f.l_start = 0; + f.l_len = 0; // Lock/unlock entire file + return fcntl(fd, F_SETLK, &f); +} + +class PosixFileLock : public FileLock { + public: + int fd_; + std::string name_; +}; + +// Set of locked files. We keep a separate set instead of just +// relying on fcntrl(F_SETLK) since fcntl(F_SETLK) does not provide +// any protection against multiple uses from the same process. +class PosixLockTable { + private: + port::Mutex mu_; + std::set locked_files_; + public: + bool Insert(const std::string& fname) { + MutexLock l(&mu_); + return locked_files_.insert(fname).second; + } + void Remove(const std::string& fname) { + MutexLock l(&mu_); + locked_files_.erase(fname); + } +}; + +class PosixEnv : public Env { + public: + PosixEnv(); + virtual ~PosixEnv() { + fprintf(stderr, "Destroying Env::Default()\n"); + abort(); + } + + virtual Status NewSequentialFile(const std::string& fname, + SequentialFile** result) { + FILE* f = fopen(fname.c_str(), "r"); + if (f == NULL) { + *result = NULL; + return IOError(fname, errno); + } else { + *result = new PosixSequentialFile(fname, f); + return Status::OK(); + } + } + + virtual Status NewRandomAccessFile(const std::string& fname, + RandomAccessFile** result) { + *result = NULL; + Status s; + int fd = open(fname.c_str(), O_RDONLY); + if (fd < 0) { + s = IOError(fname, errno); + } else if (mmap_limit_.Acquire()) { + uint64_t size; + s = GetFileSize(fname, &size); + if (s.ok()) { + void* base = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0); + if (base != MAP_FAILED) { + *result = new PosixMmapReadableFile(fname, base, size, &mmap_limit_); + } else { + s = IOError(fname, errno); + } + } + close(fd); + if (!s.ok()) { + mmap_limit_.Release(); + } + } else { + *result = new PosixRandomAccessFile(fname, fd); + } + return s; + } + + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result) { + Status s; + const int fd = open(fname.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); + if (fd < 0) { + *result = NULL; + s = IOError(fname, errno); + } else { + *result = new PosixMmapFile(fname, fd, page_size_); + } + return s; + } + + virtual bool FileExists(const std::string& fname) { + return access(fname.c_str(), F_OK) == 0; + } + + virtual Status GetChildren(const std::string& dir, + std::vector* result) { + result->clear(); + DIR* d = opendir(dir.c_str()); + if (d == NULL) { + return IOError(dir, errno); + } + struct dirent* entry; + while ((entry = readdir(d)) != NULL) { + result->push_back(entry->d_name); + } + closedir(d); + return Status::OK(); + } + + virtual Status DeleteFile(const std::string& fname) { + Status result; + if (unlink(fname.c_str()) != 0) { + result = IOError(fname, errno); + } + return result; + } + + virtual Status CreateDir(const std::string& name) { + Status result; + if (mkdir(name.c_str(), 0755) != 0) { + result = IOError(name, errno); + } + return result; + } + + virtual Status DeleteDir(const std::string& name) { + Status result; + if (rmdir(name.c_str()) != 0) { + result = IOError(name, errno); + } + return result; + } + + virtual Status GetFileSize(const std::string& fname, uint64_t* size) { + Status s; + struct stat sbuf; + if (stat(fname.c_str(), &sbuf) != 0) { + *size = 0; + s = IOError(fname, errno); + } else { + *size = sbuf.st_size; + } + return s; + } + + virtual Status RenameFile(const std::string& src, const std::string& target) { + Status result; + if (rename(src.c_str(), target.c_str()) != 0) { + result = IOError(src, errno); + } + return result; + } + + virtual Status LockFile(const std::string& fname, FileLock** lock) { + *lock = NULL; + Status result; + int fd = open(fname.c_str(), O_RDWR | O_CREAT, 0644); + if (fd < 0) { + result = IOError(fname, errno); + } else if (!locks_.Insert(fname)) { + close(fd); + result = Status::IOError("lock " + fname, "already held by process"); + } else if (LockOrUnlock(fd, true) == -1) { + result = IOError("lock " + fname, errno); + close(fd); + locks_.Remove(fname); + } else { + PosixFileLock* my_lock = new PosixFileLock; + my_lock->fd_ = fd; + my_lock->name_ = fname; + *lock = my_lock; + } + return result; + } + + virtual Status UnlockFile(FileLock* lock) { + PosixFileLock* my_lock = reinterpret_cast(lock); + Status result; + if (LockOrUnlock(my_lock->fd_, false) == -1) { + result = IOError("unlock", errno); + } + locks_.Remove(my_lock->name_); + close(my_lock->fd_); + delete my_lock; + return result; + } + + virtual void Schedule(void (*function)(void*), void* arg); + + virtual void StartThread(void (*function)(void* arg), void* arg); + + virtual Status GetTestDirectory(std::string* result) { + const char* env = getenv("TEST_TMPDIR"); + if (env && env[0] != '\0') { + *result = env; + } else { + char buf[100]; + snprintf(buf, sizeof(buf), "/tmp/leveldbtest-%d", int(geteuid())); + *result = buf; + } + // Directory may already exist + CreateDir(*result); + return Status::OK(); + } + + static uint64_t gettid() { + pthread_t tid = pthread_self(); + uint64_t thread_id = 0; + memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid))); + return thread_id; + } + + virtual Status NewLogger(const std::string& fname, Logger** result) { + FILE* f = fopen(fname.c_str(), "w"); + if (f == NULL) { + *result = NULL; + return IOError(fname, errno); + } else { + *result = new PosixLogger(f, &PosixEnv::gettid); + return Status::OK(); + } + } + + virtual uint64_t NowMicros() { + struct timeval tv; + gettimeofday(&tv, NULL); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; + } + + virtual void SleepForMicroseconds(int micros) { + usleep(micros); + } + + private: + void PthreadCall(const char* label, int result) { + if (result != 0) { + fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); + abort(); + } + } + + // BGThread() is the body of the background thread + void BGThread(); + static void* BGThreadWrapper(void* arg) { + reinterpret_cast(arg)->BGThread(); + return NULL; + } + + size_t page_size_; + pthread_mutex_t mu_; + pthread_cond_t bgsignal_; + pthread_t bgthread_; + bool started_bgthread_; + + // Entry per Schedule() call + struct BGItem { void* arg; void (*function)(void*); }; + typedef std::deque BGQueue; + BGQueue queue_; + + PosixLockTable locks_; + MmapLimiter mmap_limit_; +}; + +PosixEnv::PosixEnv() : page_size_(getpagesize()), + started_bgthread_(false) { + PthreadCall("mutex_init", pthread_mutex_init(&mu_, NULL)); + PthreadCall("cvar_init", pthread_cond_init(&bgsignal_, NULL)); +} + +void PosixEnv::Schedule(void (*function)(void*), void* arg) { + PthreadCall("lock", pthread_mutex_lock(&mu_)); + + // Start background thread if necessary + if (!started_bgthread_) { + started_bgthread_ = true; + PthreadCall( + "create thread", + pthread_create(&bgthread_, NULL, &PosixEnv::BGThreadWrapper, this)); + } + + // If the queue is currently empty, the background thread may currently be + // waiting. + if (queue_.empty()) { + PthreadCall("signal", pthread_cond_signal(&bgsignal_)); + } + + // Add to priority queue + queue_.push_back(BGItem()); + queue_.back().function = function; + queue_.back().arg = arg; + + PthreadCall("unlock", pthread_mutex_unlock(&mu_)); +} + +void PosixEnv::BGThread() { + while (true) { + // Wait until there is an item that is ready to run + PthreadCall("lock", pthread_mutex_lock(&mu_)); + while (queue_.empty()) { + PthreadCall("wait", pthread_cond_wait(&bgsignal_, &mu_)); + } + + void (*function)(void*) = queue_.front().function; + void* arg = queue_.front().arg; + queue_.pop_front(); + + PthreadCall("unlock", pthread_mutex_unlock(&mu_)); + (*function)(arg); + } +} + +namespace { +struct StartThreadState { + void (*user_function)(void*); + void* arg; +}; +} +static void* StartThreadWrapper(void* arg) { + StartThreadState* state = reinterpret_cast(arg); + state->user_function(state->arg); + delete state; + return NULL; +} + +void PosixEnv::StartThread(void (*function)(void* arg), void* arg) { + pthread_t t; + StartThreadState* state = new StartThreadState; + state->user_function = function; + state->arg = arg; + PthreadCall("start thread", + pthread_create(&t, NULL, &StartThreadWrapper, state)); +} + +} // namespace + +static pthread_once_t oncePosix = PTHREAD_ONCE_INIT; +static Env* default_env; +static void InitDefaultEnv() { default_env = new PosixEnv; } + +Env* Env::Default() { + pthread_once(&oncePosix, InitDefaultEnv); + return default_env; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/env_test.cc b/Subtrees/hyperleveldb/util/env_test.cc new file mode 100644 index 0000000000..6a608be05d --- /dev/null +++ b/Subtrees/hyperleveldb/util/env_test.cc @@ -0,0 +1,104 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/env.h" + +#include "../port/port.h" +#include "testharness.h" + +namespace hyperleveldb { + +static const int kDelayMicros = 100000; + +class EnvPosixTest { + private: + port::Mutex mu_; + std::string events_; + + public: + Env* env_; + EnvPosixTest() : env_(Env::Default()) { } +}; + +static void SetBool(void* ptr) { + reinterpret_cast(ptr)->NoBarrier_Store(ptr); +} + +TEST(EnvPosixTest, RunImmediately) { + port::AtomicPointer called (NULL); + env_->Schedule(&SetBool, &called); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_TRUE(called.NoBarrier_Load() != NULL); +} + +TEST(EnvPosixTest, RunMany) { + port::AtomicPointer last_id (NULL); + + struct CB { + port::AtomicPointer* last_id_ptr; // Pointer to shared slot + uintptr_t id; // Order# for the execution of this callback + + CB(port::AtomicPointer* p, int i) : last_id_ptr(p), id(i) { } + + static void Run(void* v) { + CB* cb = reinterpret_cast(v); + void* cur = cb->last_id_ptr->NoBarrier_Load(); + ASSERT_EQ(cb->id-1, reinterpret_cast(cur)); + cb->last_id_ptr->Release_Store(reinterpret_cast(cb->id)); + } + }; + + // Schedule in different order than start time + CB cb1(&last_id, 1); + CB cb2(&last_id, 2); + CB cb3(&last_id, 3); + CB cb4(&last_id, 4); + env_->Schedule(&CB::Run, &cb1); + env_->Schedule(&CB::Run, &cb2); + env_->Schedule(&CB::Run, &cb3); + env_->Schedule(&CB::Run, &cb4); + + Env::Default()->SleepForMicroseconds(kDelayMicros); + void* cur = last_id.Acquire_Load(); + ASSERT_EQ(4, reinterpret_cast(cur)); +} + +struct State { + port::Mutex mu; + int val; + int num_running; +}; + +static void ThreadBody(void* arg) { + State* s = reinterpret_cast(arg); + s->mu.Lock(); + s->val += 1; + s->num_running -= 1; + s->mu.Unlock(); +} + +TEST(EnvPosixTest, StartThread) { + State state; + state.val = 0; + state.num_running = 3; + for (int i = 0; i < 3; i++) { + env_->StartThread(&ThreadBody, &state); + } + while (true) { + state.mu.Lock(); + int num = state.num_running; + state.mu.Unlock(); + if (num == 0) { + break; + } + Env::Default()->SleepForMicroseconds(kDelayMicros); + } + ASSERT_EQ(state.val, 3); +} + +} // namespace hyperleveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/Subtrees/hyperleveldb/util/filter_policy.cc b/Subtrees/hyperleveldb/util/filter_policy.cc new file mode 100644 index 0000000000..916bbc9c02 --- /dev/null +++ b/Subtrees/hyperleveldb/util/filter_policy.cc @@ -0,0 +1,11 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/filter_policy.h" + +namespace hyperleveldb { + +FilterPolicy::~FilterPolicy() { } + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/hash.cc b/Subtrees/hyperleveldb/util/hash.cc new file mode 100644 index 0000000000..956facc107 --- /dev/null +++ b/Subtrees/hyperleveldb/util/hash.cc @@ -0,0 +1,52 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include "coding.h" +#include "hash.h" + +// The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through +// between switch labels. The real definition should be provided externally. +// This one is a fallback version for unsupported compilers. +#ifndef FALLTHROUGH_INTENDED +#define FALLTHROUGH_INTENDED do { } while (0) +#endif + +namespace hyperleveldb { + +uint32_t Hash(const char* data, size_t n, uint32_t seed) { + // Similar to murmur hash + const uint32_t m = 0xc6a4a793; + const uint32_t r = 24; + const char* limit = data + n; + uint32_t h = seed ^ (n * m); + + // Pick up four bytes at a time + while (data + 4 <= limit) { + uint32_t w = DecodeFixed32(data); + data += 4; + h += w; + h *= m; + h ^= (h >> 16); + } + + // Pick up remaining bytes + switch (limit - data) { + case 3: + h += data[2] << 16; + FALLTHROUGH_INTENDED; + case 2: + h += data[1] << 8; + FALLTHROUGH_INTENDED; + case 1: + h += data[0]; + h *= m; + h ^= (h >> r); + break; + } + return h; +} + + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/hash.h b/Subtrees/hyperleveldb/util/hash.h new file mode 100644 index 0000000000..8c74ce90b6 --- /dev/null +++ b/Subtrees/hyperleveldb/util/hash.h @@ -0,0 +1,19 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Simple hash function used for internal data structures + +#ifndef STORAGE_HYPERLEVELDB_UTIL_HASH_H_ +#define STORAGE_HYPERLEVELDB_UTIL_HASH_H_ + +#include +#include + +namespace hyperleveldb { + +extern uint32_t Hash(const char* data, size_t n, uint32_t seed); + +} + +#endif // STORAGE_HYPERLEVELDB_UTIL_HASH_H_ diff --git a/Subtrees/hyperleveldb/util/histogram.cc b/Subtrees/hyperleveldb/util/histogram.cc new file mode 100644 index 0000000000..b589db0b42 --- /dev/null +++ b/Subtrees/hyperleveldb/util/histogram.cc @@ -0,0 +1,139 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include "../port/port.h" +#include "histogram.h" + +namespace hyperleveldb { + +const double Histogram::kBucketLimit[kNumBuckets] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 25, 30, 35, 40, 45, + 50, 60, 70, 80, 90, 100, 120, 140, 160, 180, 200, 250, 300, 350, 400, 450, + 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000, 2500, 3000, + 3500, 4000, 4500, 5000, 6000, 7000, 8000, 9000, 10000, 12000, 14000, + 16000, 18000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, 60000, + 70000, 80000, 90000, 100000, 120000, 140000, 160000, 180000, 200000, + 250000, 300000, 350000, 400000, 450000, 500000, 600000, 700000, 800000, + 900000, 1000000, 1200000, 1400000, 1600000, 1800000, 2000000, 2500000, + 3000000, 3500000, 4000000, 4500000, 5000000, 6000000, 7000000, 8000000, + 9000000, 10000000, 12000000, 14000000, 16000000, 18000000, 20000000, + 25000000, 30000000, 35000000, 40000000, 45000000, 50000000, 60000000, + 70000000, 80000000, 90000000, 100000000, 120000000, 140000000, 160000000, + 180000000, 200000000, 250000000, 300000000, 350000000, 400000000, + 450000000, 500000000, 600000000, 700000000, 800000000, 900000000, + 1000000000, 1200000000, 1400000000, 1600000000, 1800000000, 2000000000, + 2500000000.0, 3000000000.0, 3500000000.0, 4000000000.0, 4500000000.0, + 5000000000.0, 6000000000.0, 7000000000.0, 8000000000.0, 9000000000.0, + 1e200, +}; + +void Histogram::Clear() { + min_ = kBucketLimit[kNumBuckets-1]; + max_ = 0; + num_ = 0; + sum_ = 0; + sum_squares_ = 0; + for (int i = 0; i < kNumBuckets; i++) { + buckets_[i] = 0; + } +} + +void Histogram::Add(double value) { + // Linear search is fast enough for our usage in db_bench + int b = 0; + while (b < kNumBuckets - 1 && kBucketLimit[b] <= value) { + b++; + } + buckets_[b] += 1.0; + if (min_ > value) min_ = value; + if (max_ < value) max_ = value; + num_++; + sum_ += value; + sum_squares_ += (value * value); +} + +void Histogram::Merge(const Histogram& other) { + if (other.min_ < min_) min_ = other.min_; + if (other.max_ > max_) max_ = other.max_; + num_ += other.num_; + sum_ += other.sum_; + sum_squares_ += other.sum_squares_; + for (int b = 0; b < kNumBuckets; b++) { + buckets_[b] += other.buckets_[b]; + } +} + +double Histogram::Median() const { + return Percentile(50.0); +} + +double Histogram::Percentile(double p) const { + double threshold = num_ * (p / 100.0); + double sum = 0; + for (int b = 0; b < kNumBuckets; b++) { + sum += buckets_[b]; + if (sum >= threshold) { + // Scale linearly within this bucket + double left_point = (b == 0) ? 0 : kBucketLimit[b-1]; + double right_point = kBucketLimit[b]; + double left_sum = sum - buckets_[b]; + double right_sum = sum; + double pos = (threshold - left_sum) / (right_sum - left_sum); + double r = left_point + (right_point - left_point) * pos; + if (r < min_) r = min_; + if (r > max_) r = max_; + return r; + } + } + return max_; +} + +double Histogram::Average() const { + if (num_ == 0.0) return 0; + return sum_ / num_; +} + +double Histogram::StandardDeviation() const { + if (num_ == 0.0) return 0; + double variance = (sum_squares_ * num_ - sum_ * sum_) / (num_ * num_); + return sqrt(variance); +} + +std::string Histogram::ToString() const { + std::string r; + char buf[200]; + snprintf(buf, sizeof(buf), + "Count: %.0f Average: %.4f StdDev: %.2f\n", + num_, Average(), StandardDeviation()); + r.append(buf); + snprintf(buf, sizeof(buf), + "Min: %.4f Median: %.4f Max: %.4f\n", + (num_ == 0.0 ? 0.0 : min_), Median(), max_); + r.append(buf); + r.append("------------------------------------------------------\n"); + const double mult = 100.0 / num_; + double sum = 0; + for (int b = 0; b < kNumBuckets; b++) { + if (buckets_[b] <= 0.0) continue; + sum += buckets_[b]; + snprintf(buf, sizeof(buf), + "[ %7.0f, %7.0f ) %7.0f %7.3f%% %7.3f%% ", + ((b == 0) ? 0.0 : kBucketLimit[b-1]), // left + kBucketLimit[b], // right + buckets_[b], // count + mult * buckets_[b], // percentage + mult * sum); // cumulative percentage + r.append(buf); + + // Add hash marks based on percentage; 20 marks for 100%. + int marks = static_cast(20*(buckets_[b] / num_) + 0.5); + r.append(marks, '#'); + r.push_back('\n'); + } + return r; +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/histogram.h b/Subtrees/hyperleveldb/util/histogram.h new file mode 100644 index 0000000000..ecf02ac8ad --- /dev/null +++ b/Subtrees/hyperleveldb/util/histogram.h @@ -0,0 +1,42 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_UTIL_HISTOGRAM_H_ +#define STORAGE_HYPERLEVELDB_UTIL_HISTOGRAM_H_ + +#include + +namespace hyperleveldb { + +class Histogram { + public: + Histogram() { } + ~Histogram() { } + + void Clear(); + void Add(double value); + void Merge(const Histogram& other); + + std::string ToString() const; + + private: + double min_; + double max_; + double num_; + double sum_; + double sum_squares_; + + enum { kNumBuckets = 154 }; + static const double kBucketLimit[kNumBuckets]; + double buckets_[kNumBuckets]; + + double Median() const; + double Percentile(double p) const; + double Average() const; + double StandardDeviation() const; +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_HISTOGRAM_H_ diff --git a/Subtrees/hyperleveldb/util/logging.cc b/Subtrees/hyperleveldb/util/logging.cc new file mode 100644 index 0000000000..76f8231ae9 --- /dev/null +++ b/Subtrees/hyperleveldb/util/logging.cc @@ -0,0 +1,81 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/logging.h" + +#include +#include +#include +#include +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/slice.h" + +namespace hyperleveldb { + +void AppendNumberTo(std::string* str, uint64_t num) { + char buf[30]; + snprintf(buf, sizeof(buf), "%llu", (unsigned long long) num); + str->append(buf); +} + +void AppendEscapedStringTo(std::string* str, const Slice& value) { + for (size_t i = 0; i < value.size(); i++) { + char c = value[i]; + if (c >= ' ' && c <= '~') { + str->push_back(c); + } else { + char buf[10]; + snprintf(buf, sizeof(buf), "\\x%02x", + static_cast(c) & 0xff); + str->append(buf); + } + } +} + +std::string NumberToString(uint64_t num) { + std::string r; + AppendNumberTo(&r, num); + return r; +} + +std::string EscapeString(const Slice& value) { + std::string r; + AppendEscapedStringTo(&r, value); + return r; +} + +bool ConsumeChar(Slice* in, char c) { + if (!in->empty() && (*in)[0] == c) { + in->remove_prefix(1); + return true; + } else { + return false; + } +} + +bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { + uint64_t v = 0; + int digits = 0; + while (!in->empty()) { + char c = (*in)[0]; + if (c >= '0' && c <= '9') { + ++digits; + const int delta = (c - '0'); + static const uint64_t kMaxUint64 = ~static_cast(0); + if (v > kMaxUint64/10 || + (v == kMaxUint64/10 && delta > kMaxUint64%10)) { + // Overflow + return false; + } + v = (v * 10) + delta; + in->remove_prefix(1); + } else { + break; + } + } + *val = v; + return (digits > 0); +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/logging.h b/Subtrees/hyperleveldb/util/logging.h new file mode 100644 index 0000000000..3b0d6b85d8 --- /dev/null +++ b/Subtrees/hyperleveldb/util/logging.h @@ -0,0 +1,47 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Must not be included from any .h files to avoid polluting the namespace +// with macros. + +#ifndef STORAGE_HYPERLEVELDB_UTIL_LOGGING_H_ +#define STORAGE_HYPERLEVELDB_UTIL_LOGGING_H_ + +#include +#include +#include +#include "../port/port.h" + +namespace hyperleveldb { + +class Slice; +class WritableFile; + +// Append a human-readable printout of "num" to *str +extern void AppendNumberTo(std::string* str, uint64_t num); + +// Append a human-readable printout of "value" to *str. +// Escapes any non-printable characters found in "value". +extern void AppendEscapedStringTo(std::string* str, const Slice& value); + +// Return a human-readable printout of "num" +extern std::string NumberToString(uint64_t num); + +// Return a human-readable version of "value". +// Escapes any non-printable characters found in "value". +extern std::string EscapeString(const Slice& value); + +// If *in starts with "c", advances *in past the first character and +// returns true. Otherwise, returns false. +extern bool ConsumeChar(Slice* in, char c); + +// Parse a human-readable number from "*in" into *value. On success, +// advances "*in" past the consumed number and sets "*val" to the +// numeric value. Otherwise, returns false and leaves *in in an +// unspecified state. +extern bool ConsumeDecimalNumber(Slice* in, uint64_t* val); + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_LOGGING_H_ diff --git a/Subtrees/hyperleveldb/util/mutexlock.h b/Subtrees/hyperleveldb/util/mutexlock.h new file mode 100644 index 0000000000..d9a0913d2e --- /dev/null +++ b/Subtrees/hyperleveldb/util/mutexlock.h @@ -0,0 +1,41 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_UTIL_MUTEXLOCK_H_ +#define STORAGE_HYPERLEVELDB_UTIL_MUTEXLOCK_H_ + +#include "../port/port.h" +#include "../port/thread_annotations.h" + +namespace hyperleveldb { + +// Helper class that locks a mutex on construction and unlocks the mutex when +// the destructor of the MutexLock object is invoked. +// +// Typical usage: +// +// void MyClass::MyMethod() { +// MutexLock l(&mu_); // mu_ is an instance variable +// ... some complex code, possibly with multiple return paths ... +// } + +class SCOPED_LOCKABLE MutexLock { + public: + explicit MutexLock(port::Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { + this->mu_->Lock(); + } + ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } + + private: + port::Mutex *const mu_; + // No copying allowed + MutexLock(const MutexLock&); + void operator=(const MutexLock&); +}; + +} // namespace hyperleveldb + + +#endif // STORAGE_HYPERLEVELDB_UTIL_MUTEXLOCK_H_ diff --git a/Subtrees/hyperleveldb/util/options.cc b/Subtrees/hyperleveldb/util/options.cc new file mode 100644 index 0000000000..0825f54067 --- /dev/null +++ b/Subtrees/hyperleveldb/util/options.cc @@ -0,0 +1,29 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "../hyperleveldb/options.h" + +#include "../hyperleveldb/comparator.h" +#include "../hyperleveldb/env.h" + +namespace hyperleveldb { + +Options::Options() + : comparator(BytewiseComparator()), + create_if_missing(false), + error_if_exists(false), + paranoid_checks(false), + env(Env::Default()), + info_log(NULL), + write_buffer_size(4<<20), + max_open_files(1000), + block_cache(NULL), + block_size(4096), + block_restart_interval(16), + compression(kSnappyCompression), + filter_policy(NULL) { +} + + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/posix_logger.h b/Subtrees/hyperleveldb/util/posix_logger.h new file mode 100644 index 0000000000..629d756ce4 --- /dev/null +++ b/Subtrees/hyperleveldb/util/posix_logger.h @@ -0,0 +1,98 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Logger implementation that can be shared by all environments +// where enough posix functionality is available. + +#ifndef STORAGE_HYPERLEVELDB_UTIL_POSIX_LOGGER_H_ +#define STORAGE_HYPERLEVELDB_UTIL_POSIX_LOGGER_H_ + +#include +#include +#include +#include +#include "../hyperleveldb/env.h" + +namespace hyperleveldb { + +class PosixLogger : public Logger { + private: + FILE* file_; + uint64_t (*gettid_)(); // Return the thread id for the current thread + public: + PosixLogger(FILE* f, uint64_t (*gettid)()) : file_(f), gettid_(gettid) { } + virtual ~PosixLogger() { + fclose(file_); + } + virtual void Logv(const char* format, va_list ap) { + const uint64_t thread_id = (*gettid_)(); + + // We try twice: the first time with a fixed-size stack allocated buffer, + // and the second time with a much larger dynamically allocated buffer. + char buffer[500]; + for (int iter = 0; iter < 2; iter++) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } else { + bufsize = 30000; + base = new char[bufsize]; + } + char* p = base; + char* limit = base + bufsize; + + struct timeval now_tv; + gettimeofday(&now_tv, NULL); + const time_t seconds = now_tv.tv_sec; + struct tm t; + localtime_r(&seconds, &t); + p += snprintf(p, limit - p, + "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", + t.tm_year + 1900, + t.tm_mon + 1, + t.tm_mday, + t.tm_hour, + t.tm_min, + t.tm_sec, + static_cast(now_tv.tv_usec), + static_cast(thread_id)); + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + p += vsnprintf(p, limit - p, format, backup_ap); + va_end(backup_ap); + } + + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) { + continue; // Try again with larger buffer + } else { + p = limit - 1; + } + } + + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } + + assert(p <= limit); + fwrite(base, 1, p - base, file_); + fflush(file_); + if (base != buffer) { + delete[] base; + } + break; + } + } +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_POSIX_LOGGER_H_ diff --git a/Subtrees/hyperleveldb/util/random.h b/Subtrees/hyperleveldb/util/random.h new file mode 100644 index 0000000000..119b1aa1c1 --- /dev/null +++ b/Subtrees/hyperleveldb/util/random.h @@ -0,0 +1,59 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_UTIL_RANDOM_H_ +#define STORAGE_HYPERLEVELDB_UTIL_RANDOM_H_ + +#include + +namespace hyperleveldb { + +// A very simple random number generator. Not especially good at +// generating truly random bits, but good enough for our needs in this +// package. +class Random { + private: + uint32_t seed_; + public: + explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) { } + uint32_t Next() { + static const uint32_t M = 2147483647L; // 2^31-1 + static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0 + // We are computing + // seed_ = (seed_ * A) % M, where M = 2^31-1 + // + // seed_ must not be zero or M, or else all subsequent computed values + // will be zero or M respectively. For all other values, seed_ will end + // up cycling through every number in [1,M-1] + uint64_t product = seed_ * A; + + // Compute (product % M) using the fact that ((x << 31) % M) == x. + seed_ = static_cast((product >> 31) + (product & M)); + // The first reduction may overflow by 1 bit, so we may need to + // repeat. mod == M is not possible; using > allows the faster + // sign-bit-based test. + if (seed_ > M) { + seed_ -= M; + } + return seed_; + } + // Returns a uniformly distributed value in the range [0..n-1] + // REQUIRES: n > 0 + uint32_t Uniform(int n) { return Next() % n; } + + // Randomly returns true ~"1/n" of the time, and false otherwise. + // REQUIRES: n > 0 + bool OneIn(int n) { return (Next() % n) == 0; } + + // Skewed: pick "base" uniformly from range [0,max_log] and then + // return "base" random bits. The effect is to pick a number in the + // range [0,2^max_log-1] with exponential bias towards smaller numbers. + uint32_t Skewed(int max_log) { + return Uniform(1 << Uniform(max_log + 1)); + } +}; + +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_RANDOM_H_ diff --git a/Subtrees/hyperleveldb/util/status.cc b/Subtrees/hyperleveldb/util/status.cc new file mode 100644 index 0000000000..1f714195e3 --- /dev/null +++ b/Subtrees/hyperleveldb/util/status.cc @@ -0,0 +1,75 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include "../port/port.h" +#include "../hyperleveldb/status.h" + +namespace hyperleveldb { + +const char* Status::CopyState(const char* state) { + uint32_t size; + memcpy(&size, state, sizeof(size)); + char* result = new char[size + 5]; + memcpy(result, state, size + 5); + return result; +} + +Status::Status(Code code, const Slice& msg, const Slice& msg2) { + assert(code != kOk); + const uint32_t len1 = msg.size(); + const uint32_t len2 = msg2.size(); + const uint32_t size = len1 + (len2 ? (2 + len2) : 0); + char* result = new char[size + 5]; + memcpy(result, &size, sizeof(size)); + result[4] = static_cast(code); + memcpy(result + 5, msg.data(), len1); + if (len2) { + result[5 + len1] = ':'; + result[6 + len1] = ' '; + memcpy(result + 7 + len1, msg2.data(), len2); + } + state_ = result; +} + +std::string Status::ToString() const { + if (state_ == NULL) { + return "OK"; + } else { + char tmp[30]; + const char* type; + switch (code()) { + case kOk: + type = "OK"; + break; + case kNotFound: + type = "NotFound: "; + break; + case kCorruption: + type = "Corruption: "; + break; + case kNotSupported: + type = "Not implemented: "; + break; + case kInvalidArgument: + type = "Invalid argument: "; + break; + case kIOError: + type = "IO error: "; + break; + default: + snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", + static_cast(code())); + type = tmp; + break; + } + std::string result(type); + uint32_t length; + memcpy(&length, state_, sizeof(length)); + result.append(state_ + 5, length); + return result; + } +} + +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/testharness.cc b/Subtrees/hyperleveldb/util/testharness.cc new file mode 100644 index 0000000000..d0b9b254a0 --- /dev/null +++ b/Subtrees/hyperleveldb/util/testharness.cc @@ -0,0 +1,77 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "testharness.h" + +#include +#include +#include +#include + +namespace hyperleveldb { +namespace test { + +namespace { +struct Test { + const char* base; + const char* name; + void (*func)(); +}; +std::vector* tests; +} + +bool RegisterTest(const char* base, const char* name, void (*func)()) { + if (tests == NULL) { + tests = new std::vector; + } + Test t; + t.base = base; + t.name = name; + t.func = func; + tests->push_back(t); + return true; +} + +int RunAllTests() { + const char* matcher = getenv("LEVELDB_TESTS"); + + int num = 0; + if (tests != NULL) { + for (int i = 0; i < tests->size(); i++) { + const Test& t = (*tests)[i]; + if (matcher != NULL) { + std::string name = t.base; + name.push_back('.'); + name.append(t.name); + if (strstr(name.c_str(), matcher) == NULL) { + continue; + } + } + fprintf(stderr, "==== Test %s.%s\n", t.base, t.name); + (*t.func)(); + ++num; + } + } + fprintf(stderr, "==== PASSED %d tests\n", num); + return 0; +} + +std::string TmpDir() { + std::string dir; + Status s = Env::Default()->GetTestDirectory(&dir); + ASSERT_TRUE(s.ok()) << s.ToString(); + return dir; +} + +int RandomSeed() { + const char* env = getenv("TEST_RANDOM_SEED"); + int result = (env != NULL ? atoi(env) : 301); + if (result <= 0) { + result = 301; + } + return result; +} + +} // namespace test +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/testharness.h b/Subtrees/hyperleveldb/util/testharness.h new file mode 100644 index 0000000000..752ef9982d --- /dev/null +++ b/Subtrees/hyperleveldb/util/testharness.h @@ -0,0 +1,138 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_UTIL_TESTHARNESS_H_ +#define STORAGE_HYPERLEVELDB_UTIL_TESTHARNESS_H_ + +#include +#include +#include +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/slice.h" +#include "random.h" + +namespace hyperleveldb { +namespace test { + +// Run some of the tests registered by the TEST() macro. If the +// environment variable "LEVELDB_TESTS" is not set, runs all tests. +// Otherwise, runs only the tests whose name contains the value of +// "LEVELDB_TESTS" as a substring. E.g., suppose the tests are: +// TEST(Foo, Hello) { ... } +// TEST(Foo, World) { ... } +// LEVELDB_TESTS=Hello will run the first test +// LEVELDB_TESTS=o will run both tests +// LEVELDB_TESTS=Junk will run no tests +// +// Returns 0 if all tests pass. +// Dies or returns a non-zero value if some test fails. +extern int RunAllTests(); + +// Return the directory to use for temporary storage. +extern std::string TmpDir(); + +// Return a randomization seed for this run. Typically returns the +// same number on repeated invocations of this binary, but automated +// runs may be able to vary the seed. +extern int RandomSeed(); + +// An instance of Tester is allocated to hold temporary state during +// the execution of an assertion. +class Tester { + private: + bool ok_; + const char* fname_; + int line_; + std::stringstream ss_; + + public: + Tester(const char* f, int l) + : ok_(true), fname_(f), line_(l) { + } + + ~Tester() { + if (!ok_) { + fprintf(stderr, "%s:%d:%s\n", fname_, line_, ss_.str().c_str()); + exit(1); + } + } + + Tester& Is(bool b, const char* msg) { + if (!b) { + ss_ << " Assertion failure " << msg; + ok_ = false; + } + return *this; + } + + Tester& IsOk(const Status& s) { + if (!s.ok()) { + ss_ << " " << s.ToString(); + ok_ = false; + } + return *this; + } + +#define BINARY_OP(name,op) \ + template \ + Tester& name(const X& x, const Y& y) { \ + if (! (x op y)) { \ + ss_ << " failed: " << x << (" " #op " ") << y; \ + ok_ = false; \ + } \ + return *this; \ + } + + BINARY_OP(IsEq, ==) + BINARY_OP(IsNe, !=) + BINARY_OP(IsGe, >=) + BINARY_OP(IsGt, >) + BINARY_OP(IsLe, <=) + BINARY_OP(IsLt, <) +#undef BINARY_OP + + // Attach the specified value to the error message if an error has occurred + template + Tester& operator<<(const V& value) { + if (!ok_) { + ss_ << " " << value; + } + return *this; + } +}; + +#define ASSERT_TRUE(c) ::leveldb::test::Tester(__FILE__, __LINE__).Is((c), #c) +#define ASSERT_OK(s) ::leveldb::test::Tester(__FILE__, __LINE__).IsOk((s)) +#define ASSERT_EQ(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsEq((a),(b)) +#define ASSERT_NE(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsNe((a),(b)) +#define ASSERT_GE(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsGe((a),(b)) +#define ASSERT_GT(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsGt((a),(b)) +#define ASSERT_LE(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsLe((a),(b)) +#define ASSERT_LT(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsLt((a),(b)) + +#define TCONCAT(a,b) TCONCAT1(a,b) +#define TCONCAT1(a,b) a##b + +#define TEST(base,name) \ +class TCONCAT(_Test_,name) : public base { \ + public: \ + void _Run(); \ + static void _RunIt() { \ + TCONCAT(_Test_,name) t; \ + t._Run(); \ + } \ +}; \ +bool TCONCAT(_Test_ignored_,name) = \ + ::leveldb::test::RegisterTest(#base, #name, &TCONCAT(_Test_,name)::_RunIt); \ +void TCONCAT(_Test_,name)::_Run() + +// Register the specified test. Typically not used directly, but +// invoked via the macro expansion of TEST. +extern bool RegisterTest(const char* base, const char* name, void (*func)()); + + +} // namespace test +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_TESTHARNESS_H_ diff --git a/Subtrees/hyperleveldb/util/testutil.cc b/Subtrees/hyperleveldb/util/testutil.cc new file mode 100644 index 0000000000..bdd400ad23 --- /dev/null +++ b/Subtrees/hyperleveldb/util/testutil.cc @@ -0,0 +1,51 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "testutil.h" + +#include "random.h" + +namespace hyperleveldb { +namespace test { + +Slice RandomString(Random* rnd, int len, std::string* dst) { + dst->resize(len); + for (int i = 0; i < len; i++) { + (*dst)[i] = static_cast(' ' + rnd->Uniform(95)); // ' ' .. '~' + } + return Slice(*dst); +} + +std::string RandomKey(Random* rnd, int len) { + // Make sure to generate a wide variety of characters so we + // test the boundary conditions for short-key optimizations. + static const char kTestChars[] = { + '\0', '\1', 'a', 'b', 'c', 'd', 'e', '\xfd', '\xfe', '\xff' + }; + std::string result; + for (int i = 0; i < len; i++) { + result += kTestChars[rnd->Uniform(sizeof(kTestChars))]; + } + return result; +} + + +extern Slice CompressibleString(Random* rnd, double compressed_fraction, + int len, std::string* dst) { + int raw = static_cast(len * compressed_fraction); + if (raw < 1) raw = 1; + std::string raw_data; + RandomString(rnd, raw, &raw_data); + + // Duplicate the random data until we have filled "len" bytes + dst->clear(); + while (dst->size() < len) { + dst->append(raw_data); + } + dst->resize(len); + return Slice(*dst); +} + +} // namespace test +} // namespace hyperleveldb diff --git a/Subtrees/hyperleveldb/util/testutil.h b/Subtrees/hyperleveldb/util/testutil.h new file mode 100644 index 0000000000..d62f07dfac --- /dev/null +++ b/Subtrees/hyperleveldb/util/testutil.h @@ -0,0 +1,53 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_HYPERLEVELDB_UTIL_TESTUTIL_H_ +#define STORAGE_HYPERLEVELDB_UTIL_TESTUTIL_H_ + +#include "../hyperleveldb/env.h" +#include "../hyperleveldb/slice.h" +#include "random.h" + +namespace hyperleveldb { +namespace test { + +// Store in *dst a random string of length "len" and return a Slice that +// references the generated data. +extern Slice RandomString(Random* rnd, int len, std::string* dst); + +// Return a random key with the specified length that may contain interesting +// characters (e.g. \x00, \xff, etc.). +extern std::string RandomKey(Random* rnd, int len); + +// Store in *dst a string of length "len" that will compress to +// "N*compressed_fraction" bytes and return a Slice that references +// the generated data. +extern Slice CompressibleString(Random* rnd, double compressed_fraction, + int len, std::string* dst); + +// A wrapper that allows injection of errors. +class ErrorEnv : public EnvWrapper { + public: + bool writable_file_error_; + int num_writable_file_errors_; + + ErrorEnv() : EnvWrapper(Env::Default()), + writable_file_error_(false), + num_writable_file_errors_(0) { } + + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result) { + if (writable_file_error_) { + ++num_writable_file_errors_; + *result = NULL; + return Status::IOError(fname, "fake error"); + } + return target()->NewWritableFile(fname, result); + } +}; + +} // namespace test +} // namespace hyperleveldb + +#endif // STORAGE_HYPERLEVELDB_UTIL_TESTUTIL_H_ diff --git a/Subtrees/leveldb/build_detect_platform b/Subtrees/leveldb/build_detect_platform index 609cb51224..a3ad057eee 100755 --- a/Subtrees/leveldb/build_detect_platform +++ b/Subtrees/leveldb/build_detect_platform @@ -94,6 +94,12 @@ case "$TARGET_OS" in PLATFORM_LIBS="-lpthread" PORT_FILE=port/port_posix.cc ;; + GNU/kFreeBSD) + PLATFORM=OS_KFREEBSD + COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_KFREEBSD" + PLATFORM_LIBS="-lpthread" + PORT_FILE=port/port_posix.cc + ;; NetBSD) PLATFORM=OS_NETBSD COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD" diff --git a/Subtrees/websocket/src/http/parser.hpp b/Subtrees/websocket/src/http/parser.hpp index 2a4918b60a..72fb0a65de 100644 --- a/Subtrees/websocket/src/http/parser.hpp +++ b/Subtrees/websocket/src/http/parser.hpp @@ -83,7 +83,12 @@ public: } std::string header(const std::string& key) const { - header_list::const_iterator h = m_headers.find(tolower(key)); + header_list::const_iterator h = m_headers.find(key); + if (h != m_headers.end()) { + return h->second; + } + + h = m_headers.find(tolower(key)); if (h == m_headers.end()) { return ""; @@ -97,18 +102,18 @@ public: void add_header(const std::string &key, const std::string &val) { // TODO: prevent use of reserved headers? if (this->header(key) == "") { - m_headers[tolower(key)] = val; + m_headers[key] = val; } else { - m_headers[tolower(key)] += ", " + val; + m_headers[key] += ", " + val; } } void replace_header(const std::string &key, const std::string &val) { - m_headers[tolower(key)] = val; + m_headers[key] = val; } void remove_header(const std::string &key) { - m_headers.erase(tolower(key)); + m_headers.erase(key); } protected: bool parse_headers(std::istream& s) { diff --git a/Subtrees/websocket/src/rng/boost_rng.hpp b/Subtrees/websocket/src/rng/boost_rng.hpp index 8069937080..c87c20e54d 100644 --- a/Subtrees/websocket/src/rng/boost_rng.hpp +++ b/Subtrees/websocket/src/rng/boost_rng.hpp @@ -30,7 +30,9 @@ #ifndef BOOST_RNG_HPP #define BOOST_RNG_HPP -#define __STDC_LIMIT_MACROS 1 +#ifndef __STDC_LIMIT_MACROS + #define __STDC_LIMIT_MACROS +#endif #include #include diff --git a/Subtrees/websocket/src/roles/server.hpp b/Subtrees/websocket/src/roles/server.hpp index 1c2ea938f5..5f4ae1500e 100644 --- a/Subtrees/websocket/src/roles/server.hpp +++ b/Subtrees/websocket/src/roles/server.hpp @@ -551,10 +551,13 @@ void server::connection::async_init() { m_connection.register_timeout(5000,fail::status::TIMEOUT_WS, "Timeout on WebSocket handshake"); + static boost::arg<1> pl1; + static boost::arg<2> pl2; + boost::shared_ptr stringPtr = boost::make_shared(); m_connection.get_socket().async_read_until( m_connection.buffer(), - boost::bind(&match_header, stringPtr, _1, _2), + boost::bind(&match_header, stringPtr, pl1, pl2), m_connection.get_strand().wrap(boost::bind( &type::handle_read_request, m_connection.shared_from_this(), diff --git a/Subtrees/websocket/src/sha1/sha.cpp b/Subtrees/websocket/src/sha1/sha.cpp index ad860863af..d59e1a60fc 100755 --- a/Subtrees/websocket/src/sha1/sha.cpp +++ b/Subtrees/websocket/src/sha1/sha.cpp @@ -19,7 +19,7 @@ #include #include -#ifdef WIN32 +#if BEAST_WIN32 #include #endif #include @@ -87,7 +87,7 @@ int main(int argc, char *argv[]) if (argc == 1 || !strcmp(argv[i],"-")) { -#ifdef WIN32 +#if BEAST_WIN32 _setmode(_fileno(stdin), _O_BINARY); #endif fp = stdin; diff --git a/Subtrees/websocket/src/websocket_frame.hpp b/Subtrees/websocket/src/websocket_frame.hpp index 3c36e04543..1bf6a8ed2c 100644 --- a/Subtrees/websocket/src/websocket_frame.hpp +++ b/Subtrees/websocket/src/websocket_frame.hpp @@ -38,7 +38,7 @@ #include -#if defined(WIN32) +#if BEAST_WIN32 #include #else #include diff --git a/TODO.txt b/TODO.txt index 6df6f52560..2f4756596f 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,19 +1,104 @@ -------------------------------------------------------------------------------- -TODO +RIPPLE TODO -------------------------------------------------------------------------------- +- See if UniqueNodeList is really used, and if its not used remove it. If + only some small part of it is used, then delete the rest. David says + that it is broken anyway. + +- Roll a simple wrapper for sqlite relational stuff like loading the UNL + Completely hide the specifics of SQLite and/or beast::db + +- Tidy up convenience functions in RPC.h + +- Maybe rename RPCServer to RPCClientServicer + +- Take away the "I" prefix from abstract interface classes, in both the class + name and the file name. It is messing up sorting in the IDE. Use "Imp" or + suffix for implementations. + +- Profile/VTune the application to identify hot spots + * Determine why rippled has a slow startup on Windows + * Improve the performance when running all unit tests on Windows + +- Rename "fullBelow" to something like haveAllDescendants or haveAllChildren. + +- Class to represent IP and Port number, with members to print, check syntax, + etc... replace the boost calls. + +- Remove dependence on JobQueue, LoadFeeTrack, and NetworkOPs from LoadManager + by providing an observer (beast::ListenerList or Listeners). This way + LoadManager does not need stopThread() function. + +- Move everything in src/cpp/ripple into ripple_app and sort them into + subdirectories within the module as per the project filters. + * Make sure there are no pending commits from David + +- Rewrite Sustain to use Beast and work on Windows as well + * Do not enable watchdog process if a debugger is attached + +- Make separate LevelDB VS2012 project for source browsing, leave ony the unity + .cpp in the main RippleD project + +- Add LevelDB unity .cpp to the LevelDB fork + +- Make sure the leak detector output appears on Linux and FreeBSD debug builds. + +- Create SharedData , move all load related state variables currently + protected by separated mutexes in different classes into the LoadState, and + use read/write locking semantics to update the values. Later, use Listeners + to notify dependent code to resolve the dependency inversion. + +- Merge ripple_Version.h and ripple_BuildVersion.h + +- Rename LoadMonitor to LoadMeter, change LoadEvent to LoadMeter::ScopedSample + +- Rename LedgerMaster to Ledgers, create ILedgers interface. + +- Restructure the ripple sources to have this directory structure: + /Source/ripple/ripple_core/ripple_core.h + /... + /Source/Subtrees/... ? + PROBLEM: Where to put BeastConfig.h ? + +- Figure out where previous ledgers go after a call to LedgerMaster::pushLedger() + and see if it is possible to clean up the leaks on exit. + +- Replace all NULL with nullptr + +- Add ICore interface (incremental replacement for IApplication) + +- Make TxFormats a member of ICore instead of a singleton. + PROBLEM: STObject derived classes like STInt16 make direct use of the + singleton. It might have to remain a singleton. At the very least, + it should be a SharedSingleton to resolve ordering issues. + +- Rename include guards to boost style, e.g. RIPPLE_LOG_H_INCLUDED + +- Replace C11X with BEAST_COMPILER_SUPPORTS_MOVE_SEMANTICS + +- Fix all leaks on exit (!) + Say there's a leak, a ledger that can never be accessed is locked in some + structure. If the organized teardown code frees that structure, the leak + will not be reported. + Yes, so you'll detect some small subset of leaks that way. + You'll still have to be vigilant for the leaks that won't detect. + The problem is ordering. There are lots of circular dependencies. + The biggest problem is the order of destruction of global objects. (I think) + Getting rid of global objects is a good solution to that. + Vinnie Falco: Those I can resolve with my ReferenceCountedSingleton. And + yeah thats a good approach, one that I am doing slowly anyway + Yeah, that's good for other reasons too, not just the unpredictability of + creation order that can hide bugs. + There may also just be some missing destructors. + Some of it may be things being shut down in the wrong order. Like if you shut + down the cache and then something that uses the cache, objects may get + put in the cache after it was shut down. + - Remove "ENABLE_INSECURE" when the time is right. -- Put all the Ripple code in the ripple namespace - * boost unit tests create namespace problems - -- Remove references to BEASTApplication - -- Remove "using namespace" statements - -- lift bind, function, and placeholders into ripple namespace -- lift beast into the ripple namespace, remove ripple's duplicated integer types -- lift unique_ptr / auto_ptr into ripple namespace, or replace with ScopedPointer +- lift unique_ptr / auto_ptr into ripple namespace, + or replace with ScopedPointer (preferred) - Make LevelDB and Ripple code work with both Unicode and non-Unicode Windows APIs @@ -34,26 +119,68 @@ TODO - Consolidate SQLite database classes: DatabaseCon, Database, SqliteDatabase. +-------------------------------------------------------------------------------- +LEVELDB TODO +-------------------------------------------------------------------------------- + +- Add VisualStudio 2012 project file to our fork + +- Add LevelDB unity .cpp and .h to our fork + +- Replace Beast specific platform macros with universal macros so that the + unity doesn't require Beast + +- Submit LevelDB fork changes to Bitcoin upstream + +-------------------------------------------------------------------------------- +WEBSOCKET TODO +-------------------------------------------------------------------------------- + +*** Figure out how hard we want to fork websocket first ** + +- Think about stripping the ripple specifics out of AutoSocket, make AutoSocket + part of our websocketpp fork + +- Regroup all the sources together in one directory + +- Strip includes and enforce unity + +- Put a new front-end on websocket to hide ALL of their classes and templates + from the host application, make this part of the websocket fork + +-------------------------------------------------------------------------------- +PROTOCOL BUFFERS TODO +-------------------------------------------------------------------------------- + +- Create/maintain the protobuf Git repo (original uses SVN) + +- Update the subtree + +- Make a Visual Studio 2012 Project for source browsing + +-------------------------------------------------------------------------------- +NOTES -------------------------------------------------------------------------------- LoadEvent Is referenced with both a shared pointer and an auto pointer. + Should be named LoadMeter::ScopedSample. Or possibly ScopedLoadSample JobQueue getLoadEvent and getLoadEventAP differ only in the style of pointer container which is returned. Unnecessary complexity. --------------------------------------------------------------------------------- - -Naming - -Some names don't make sense. +Naming: Some names don't make sense. Index Stop using Index to refer to keys in tables. Replace with "Key" ? Index implies a small integer, or a data structure. + + This is all over the place in the Ledger API, "Index" of this and + "Index" of that, the terminology is imprecise and helps neither + understanding nor recall. Inconsistent names @@ -70,13 +197,14 @@ Verbose names Ledger "Skip List" - Is not really a skip list data structure + Is not really a skip list data structure. This is more appropriately + called an "index" although that name is currently used to identify hashes + used as keys. Duplicate Code LedgerEntryFormat and TxFormat - --------------------------------------------------------------------------------- + * Resolved with a todo item, create WireFormats<> template class. Interfaces @@ -92,15 +220,7 @@ Interfaces would be more clear: bool write (OutputStream& stream); --------------------------------------------------------------------------------- - -Implementation - - LoadManager - - What is going on in the destructor - --------------------------------------------------------------------------------- + We have beast for InputStream and OutputStream, we can use those now. boost @@ -114,7 +234,63 @@ boost boost::recursive_mutex Recursive mutexes should never be necessary. + They require the "mutable" keyword for const members to acquire the lock (yuck) - Replace recursive_mutex with juce::Mutex to remove boost dependency + Replace recursive_mutex with beast::Mutex to remove boost dependency + +-------------------------------------------------------------------------------- +Davidisms +-------------------------------------------------------------------------------- + +(Figure out a good place to record information like this permanently) + +Regarding a defect where a failing transaction was being submitted over and over + again on the network (July 3, 2013) + + The core problem was an interaction between two bits of logic. + 1) Normally, we won't relay a transaction again if we already recently relayed + it. But this is bypassed if the transaction failed in a way that could + allow it to succeed later. This way, if one server discovers a transaction + can now work, it can get all servers to retry it. + 2) Normally, we won't relay a transaction if we think it can't claim a fee. + But if we're not sure it can't claim a fee because we're in an unhealthy + state, we propagate the transaction to let other servers decide if they + think it can claim a fee. + With these two bits of logic, two unhealthy servers could infinitely propagate + a transaction back and forth between each other. + +A node is "full below" if we believe we have (either in the database or + scheduled to be stored in the database) the contents of every node below that + node in a hash tree. When trying to acquire a hash tree/map, if a node is + full below, we know not to bother with anything below that node. + +The fullBelowCache is a cache of hashes of nodes that are full below. Which means + there are no missing children + + +What we want from the unique node list: + - Some number of trusted roots (known by domain) + probably organizations whose job is to provide a list of validators + - We imagine the IRGA for example would establish some group whose job is to + maintain a list of validators. There would be a public list of criteria + that they would use to vet the validator. Things like: + * Not anonymous + * registered business + * Physical location + * Agree not to cease operations without notice / arbitrarily + * Responsive to complaints + - Identifiable jurisdiction + * Homogeneity in the jurisdiction is a business risk + * If all validators are in the same jurisdiction this is a business risk + - OpenCoin sets criteria for the organizations + - Rippled will ship with a list of trusted root "certificates" + In other words this is a list of trusted domains from which the software + can contact each trusted root and retrieve a list of "good" validators + and then do something with that information + - All the validation information would be public, including the broadcast + messages. + - The goal is to easily identify bad actors and assess network health + * Malicious intent + * Or, just hardware problems (faulty drive or memory) diff --git a/libraries/liblmdb/.gitignore b/libraries/liblmdb/.gitignore new file mode 100644 index 0000000000..0d493fe188 --- /dev/null +++ b/libraries/liblmdb/.gitignore @@ -0,0 +1,16 @@ +mtest +mtest[23456] +testdb +mdb_copy +mdb_stat +*.[ao] +*.so +*[~#] +*.bak +*.orig +*.rej +core +core.* +valgrind.* +man/ +html/ diff --git a/libraries/liblmdb/COPYRIGHT b/libraries/liblmdb/COPYRIGHT new file mode 100644 index 0000000000..4482816cf5 --- /dev/null +++ b/libraries/liblmdb/COPYRIGHT @@ -0,0 +1,20 @@ +Copyright 2011-2013 Howard Chu, Symas Corp. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted only as authorized by the OpenLDAP +Public License. + +A copy of this license is available in the file LICENSE in the +top-level directory of the distribution or, alternatively, at +. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Individual files and/or contributed packages may be copyright by +other parties and/or subject to additional restrictions. + +This work also contains materials derived from public sources. + +Additional information about OpenLDAP can be obtained at +. diff --git a/libraries/liblmdb/Doxyfile b/libraries/liblmdb/Doxyfile new file mode 100644 index 0000000000..3fd0365c7d --- /dev/null +++ b/libraries/liblmdb/Doxyfile @@ -0,0 +1,1631 @@ +# Doxyfile 1.7.1 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = MDB + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, +# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English +# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, +# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak, +# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = YES + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this +# tag. The format is ext=language, where ext is a file extension, and language +# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, +# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make +# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C +# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions +# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen to replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +INLINE_GROUPED_CLASSES = YES +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. + +TYPEDEF_HIDES_STRUCT = YES + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penality. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will rougly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols + +SYMBOL_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespace are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen +# will list include files with double quotes in the documentation +# rather than with sharp brackets. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = NO + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen +# will sort the (brief and detailed) documentation of class members so that +# constructors and destructors are listed first. If set to NO (the default) +# the constructors will appear in the respective orders defined by +# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. +# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO +# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = NO + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. +# This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. The create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. +# You can optionally specify a file name after the option, if omitted +# DoxygenLayout.xml will be used as the name of the layout file. + +LAYOUT_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = lmdb.h midl.h mdb.c midl.c + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. +# If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. +# Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. +# The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. +# Otherwise they will link to the documentation. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = YES + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. +# Doxygen will adjust the colors in the stylesheet and background images +# according to this color. Hue is specified as an angle on a colorwheel, +# see http://en.wikipedia.org/wiki/Hue for more information. +# For instance the value 0 represents red, 60 is yellow, 120 is green, +# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. +# The allowed range is 0 to 359. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of +# the colors in the HTML output. For a value of 0 the output will use +# grayscales only. A value of 255 will produce the most vivid colors. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to +# the luminance component of the colors in the HTML output. Values below +# 100 gradually make the output lighter, whereas values above 100 make +# the output darker. The value divided by 100 is the actual gamma applied, +# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, +# and 100 does not change the gamma. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting +# this to NO can help when comparing the output of multiple runs. + +HTML_TIMESTAMP = YES + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated +# that can be used as input for Qt's qhelpgenerator to generate a +# Qt Compressed Help (.qch) of the generated HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to +# add. For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see +# +# Qt Help Project / Custom Filters. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's +# filter section matches. +# +# Qt Help Project / Filter Attributes. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files +# will be generated, which together with the HTML files, form an Eclipse help +# plugin. To install this plugin and make it available under the help contents +# menu in Eclipse, the contents of the directory containing the HTML and XML +# files needs to be copied into the plugins directory of eclipse. The name of +# the directory within the plugins directory should be the same as +# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before +# the help appears. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have +# this name. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to YES, a side panel will be generated +# containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). +# Windows users are probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, +# and Class Hierarchy pages using a tree view instead of an ordered list. + +USE_INLINE_TREES = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open +# links to external symbols imported via tag files in a separate window. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are +# not supported properly for IE 6.0, but are supported on all modern browsers. +# Note that when changing this option you need to delete any form_*.png files +# in the HTML output before the changes have effect. + +FORMULA_TRANSPARENT = YES + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box +# for the HTML output. The underlying search engine uses javascript +# and DHTML and should work on any modern browser. Note that when using +# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets +# (GENERATE_DOCSET) there is already a search function so this one should +# typically be disabled. For large projects the javascript based search engine +# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a PHP enabled web server instead of at the web client +# using Javascript. Doxygen will generate the search PHP script and index +# file to put on the web server. The advantage of the server +# based approach is that it scales better to large projects and allows +# full text search. The disadvances is that it is more difficult to setup +# and does not have live searching capabilities. + +SERVER_BASED_SEARCH = NO + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. +# Note that when enabling USE_PDFLATEX this option is only used for +# generating bitmaps for formulas in the HTML output, but not in the +# Makefile that is written to the output directory. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +# If LATEX_SOURCE_CODE is set to YES then doxygen will include +# source code with syntax highlighting in the LaTeX output. +# Note that which sources are shown also depends on other settings +# such as SOURCE_BROWSER. + +LATEX_SOURCE_CODE = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = YES + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. +# This is useful +# if you want to understand what is going on. +# On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = DEBUG=2 __GNUC__=1 + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is +# allowed to run in parallel. When set to 0 (the default) doxygen will +# base this on the number of processors available in the system. You can set it +# explicitly to a value larger than 0 to get control over the balance +# between CPU load and processing speed. + +DOT_NUM_THREADS = 0 + +# By default doxygen will write a font called FreeSans.ttf to the output +# directory and reference it in all dot files that doxygen generates. This +# font does not include all possible unicode characters however, so when you need +# these (or just want a differently looking font) you can specify the font name +# using DOT_FONTNAME. You need need to make sure dot is able to find the font, +# which can be done by putting it in a standard location or by setting the +# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory +# containing the font. + +DOT_FONTNAME = FreeSans.ttf + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the output directory to look for the +# FreeSans.ttf font (which doxygen will put there itself). If you specify a +# different font using DOT_FONTNAME you can set the path where dot +# can find it using this tag. + +DOT_FONTPATH = + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = YES + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES diff --git a/libraries/liblmdb/LICENSE b/libraries/liblmdb/LICENSE new file mode 100644 index 0000000000..05ad7571e4 --- /dev/null +++ b/libraries/liblmdb/LICENSE @@ -0,0 +1,47 @@ +The OpenLDAP Public License + Version 2.8, 17 August 2003 + +Redistribution and use of this software and associated documentation +("Software"), with or without modification, are permitted provided +that the following conditions are met: + +1. Redistributions in source form must retain copyright statements + and notices, + +2. Redistributions in binary form must reproduce applicable copyright + statements and notices, this list of conditions, and the following + disclaimer in the documentation and/or other materials provided + with the distribution, and + +3. Redistributions must contain a verbatim copy of this document. + +The OpenLDAP Foundation may revise this license from time to time. +Each revision is distinguished by a version number. You may use +this Software under terms of this license revision or under the +terms of any subsequent revision of the license. + +THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS +CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) +OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +The names of the authors and copyright holders must not be used in +advertising or otherwise to promote the sale, use or other dealing +in this Software without specific, written prior permission. Title +to copyright in this Software shall at all times remain with copyright +holders. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, +California, USA. All Rights Reserved. Permission to copy and +distribute verbatim copies of this document is granted. diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile new file mode 100644 index 0000000000..25c52ada8e --- /dev/null +++ b/libraries/liblmdb/Makefile @@ -0,0 +1,87 @@ +# Makefile for liblmdb (Lightning memory-mapped database library). + +######################################################################## +# Configuration. The compiler options must enable threaded compilation. +# +# Preprocessor macros (for CPPFLAGS) of interest: +# +# To compile successfully if the default does not: +# - MDB_USE_POSIX_SEM (enabled by default on BSD, Apple) +# Define if shared mutexes are unsupported. Note that Posix +# semaphores and shared mutexes have different behaviors and +# different problems, see the Caveats section in lmdb.h. +# +# For best performence or to compile successfully: +# - MDB_DSYNC = "O_DSYNC" (default) or "O_SYNC" (less efficient) +# If O_DSYNC is undefined but exists in /usr/include, +# preferably set some compiler flag to get the definition. +# - MDB_FDATASYNC = "fdatasync" or "fsync" +# Function for flushing the data of a file. Define this to +# "fsync" if fdatasync() is not supported. fdatasync is +# default except on BSD, Apple, Android which use fsync. +# - MDB_USE_PWRITEV +# Define if the pwritev() function is supported. +# +# Data format: +# - MDB_MAXKEYSIZE +# Controls data packing and limits, see mdb.c. +# +# Debugging: +# - MDB_DEBUG, MDB_PARANOID. +# +CC = gcc +W = -W -Wall -Wno-unused-parameter -Wbad-function-cast +OPT = -O2 -g +CFLAGS = -pthread $(OPT) $(W) $(XCFLAGS) +LDLIBS = +SOLIBS = +prefix = /usr/local + +######################################################################## + +IHDRS = lmdb.h +ILIBS = liblmdb.a liblmdb.so +IPROGS = mdb_stat mdb_copy +IDOCS = mdb_stat.1 mdb_copy.1 +PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 +all: $(ILIBS) $(PROGS) + +install: $(ILIBS) $(IPROGS) $(IHDRS) + cp $(IPROGS) $(DESTDIR)$(prefix)/bin + cp $(ILIBS) $(DESTDIR)$(prefix)/lib + cp $(IHDRS) $(DESTDIR)$(prefix)/include + cp $(IDOCS) $(DESTDIR)$(prefix)/man/man1 + +clean: + rm -rf $(PROGS) *.[ao] *.so *~ testdb + +test: all + mkdir testdb + ./mtest && ./mdb_stat testdb + +liblmdb.a: mdb.o midl.o + ar rs $@ mdb.o midl.o + +liblmdb.so: mdb.o midl.o + $(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.o midl.o $(SOLIBS) + +mdb_stat: mdb_stat.o liblmdb.a +mdb_copy: mdb_copy.o liblmdb.a +mtest: mtest.o liblmdb.a +mtest2: mtest2.o liblmdb.a +mtest3: mtest3.o liblmdb.a +mtest4: mtest4.o liblmdb.a +mtest5: mtest5.o liblmdb.a +mtest6: mtest6.o liblmdb.a + +mdb.o: mdb.c lmdb.h midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c + +midl.o: midl.c midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c + +%: %.o + $(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@ + +%.o: %.c lmdb.h + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h new file mode 100644 index 0000000000..9f00a04202 --- /dev/null +++ b/libraries/liblmdb/lmdb.h @@ -0,0 +1,1297 @@ +/** @file lmdb.h + * @brief Lightning memory-mapped database library + * + * @mainpage Lightning Memory-Mapped Database Manager (MDB) + * + * @section intro_sec Introduction + * MDB is a Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. The entire database is exposed + * in a memory map, and all data fetches return data directly + * from the mapped memory, so no malloc's or memcpy's occur during + * data fetches. As such, the library is extremely simple because it + * requires no page caching layer of its own, and it is extremely high + * performance and memory-efficient. It is also fully transactional with + * full ACID semantics, and when the memory map is read-only, the + * database integrity cannot be corrupted by stray pointer writes from + * application code. + * + * The library is fully thread-aware and supports concurrent read/write + * access from multiple processes and threads. Data pages use a copy-on- + * write strategy so no active data pages are ever overwritten, which + * also provides resistance to corruption and eliminates the need of any + * special recovery procedures after a system crash. Writes are fully + * serialized; only one write transaction may be active at a time, which + * guarantees that writers can never deadlock. The database structure is + * multi-versioned so readers run with no locks; writers cannot block + * readers, and readers don't block writers. + * + * Unlike other well-known database mechanisms which use either write-ahead + * transaction logs or append-only data writes, MDB requires no maintenance + * during operation. Both write-ahead loggers and append-only databases + * require periodic checkpointing and/or compaction of their log or database + * files otherwise they grow without bound. MDB tracks free pages within + * the database and re-uses them for new write operations, so the database + * size does not grow without bound in normal use. + * + * The memory map can be used as a read-only or read-write map. It is + * read-only by default as this provides total immunity to corruption. + * Using read-write mode offers much higher write performance, but adds + * the possibility for stray application writes thru pointers to silently + * corrupt the database. Of course if your application code is known to + * be bug-free (...) then this is not an issue. + * + * @section caveats_sec Caveats + * Troubleshooting the lock file, plus semaphores on BSD systems: + * + * - A broken lockfile can cause sync issues. + * Stale reader transactions left behind by an aborted program + * cause further writes to grow the database quickly, and + * stale locks can block further operation. + * + * Fix: Terminate all programs using the database, or make + * them close it. Next database user will reset the lockfile. + * + * - On BSD systems or others configured with MDB_USE_POSIX_SEM, + * startup can fail due to semaphores owned by another userid. + * + * Fix: Open and close the database as the user which owns the + * semaphores (likely last user) or as root, while no other + * process is using the database. + * + * Restrictions/caveats (in addition to those listed for some functions): + * + * - Only the database owner should normally use the database on + * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. + * Multiple users can cause startup to fail later, as noted above. + * + * - A thread can only use one transaction at a time, plus any child + * transactions. Each transaction belongs to one thread. See below. + * The #MDB_NOTLS flag changes this for read-only transactions. + * + * - Use an MDB_env* in the process which opened it, without fork()ing. + * + * - Do not have open an MDB database twice in the same process at + * the same time. Not even from a plain open() call - close()ing it + * breaks flock() advisory locking. + * + * - Avoid long-lived transactions. Read transactions prevent + * reuse of pages freed by newer write transactions, thus the + * database can grow quickly. Write transactions prevent + * other write transactions, since writes are serialized. + * + * - Avoid suspending a process with active transactions. These + * would then be "long-lived" as above. Also read transactions + * suspended when writers commit could sometimes see wrong data. + * + * ...when several processes can use a database concurrently: + * + * - Avoid aborting a process with an active transaction. + * The transaction becomes "long-lived" as above until the lockfile + * is reset, since the process may not remove it from the lockfile. + * + * - If you do that anyway, close the environment once in a while, + * so the lockfile can get reset. + * + * - Do not use MDB databases on remote filesystems, even between + * processes on the same host. This breaks flock() on some OSes, + * possibly memory map sync, and certainly sync between programs + * on different hosts. + * + * - Opening a database can fail if another process is opening or + * closing it at exactly the same time. + * + * @author Howard Chu, Symas Corporation. + * + * @copyright Copyright 2011-2013 Howard Chu, Symas Corp. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * @par Derived From: + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _LMDB_H_ +#define _LMDB_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef _MSC_VER +typedef int mdb_mode_t; +#else +typedef mode_t mdb_mode_t; +#endif + +/** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#ifdef _WIN32 +typedef void *mdb_filehandle_t; +#else +typedef int mdb_filehandle_t; +#endif + +/** @defgroup mdb MDB API + * @{ + * @brief OpenLDAP Lightning Memory-Mapped Database Manager + */ +/** @defgroup Version Version Macros + * @{ + */ +/** Library major version */ +#define MDB_VERSION_MAJOR 0 +/** Library minor version */ +#define MDB_VERSION_MINOR 9 +/** Library patch version */ +#define MDB_VERSION_PATCH 6 + +/** Combine args a,b,c into a single integer for easy version comparisons */ +#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) + +/** The full library version as a single integer */ +#define MDB_VERSION_FULL \ + MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) + +/** The release date of this library version */ +#define MDB_VERSION_DATE "January 10, 2013" + +/** A stringifier for the version info */ +#define MDB_VERSTR(a,b,c,d) "MDB " #a "." #b "." #c ": (" d ")" + +/** A helper for the stringifier macro */ +#define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) + +/** The full library version as a C string */ +#define MDB_VERSION_STRING \ + MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) +/** @} */ + +/** @brief Opaque structure for a database environment. + * + * A DB environment supports multiple databases, all residing in the same + * shared-memory map. + */ +typedef struct MDB_env MDB_env; + +/** @brief Opaque structure for a transaction handle. + * + * All database operations require a transaction handle. Transactions may be + * read-only or read-write. + */ +typedef struct MDB_txn MDB_txn; + +/** @brief A handle for an individual database in the DB environment. */ +typedef unsigned int MDB_dbi; + +/** @brief Opaque structure for navigating through a database */ +typedef struct MDB_cursor MDB_cursor; + +/** @brief Generic structure used for passing keys and data in and out + * of the database. + * + * Key sizes must be between 1 and the liblmdb build-time constant + * #MDB_MAXKEYSIZE inclusive. This currently defaults to 511. The + * same applies to data sizes in databases with the #MDB_DUPSORT flag. + * Other data items can in theory be from 0 to 0xffffffff bytes long. + * + * Values returned from the database are valid only until a subsequent + * update operation, or the end of the transaction. + */ +typedef struct MDB_val { + size_t mv_size; /**< size of the data item */ + void *mv_data; /**< address of the data item */ +} MDB_val; + +/** @brief A callback function used to compare two keys in a database */ +typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); + +/** @brief A callback function used to relocate a position-dependent data item + * in a fixed-address database. + * + * The \b newptr gives the item's desired address in + * the memory map, and \b oldptr gives its previous address. The item's actual + * data resides at the address in \b item. This callback is expected to walk + * through the fields of the record in \b item and modify any + * values based at the \b oldptr address to be relative to the \b newptr address. + * @param[in,out] item The item that is to be relocated. + * @param[in] oldptr The previous address. + * @param[in] newptr The new address to relocate to. + * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). + * @todo This feature is currently unimplemented. + */ +typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); + +/** @defgroup mdb_env Environment Flags + * + * Values do not overlap Database Flags. + * @{ + */ + /** mmap at a fixed address (experimental) */ +#define MDB_FIXEDMAP 0x01 + /** no environment directory */ +#define MDB_NOSUBDIR 0x4000 + /** don't fsync after commit */ +#define MDB_NOSYNC 0x10000 + /** read only */ +#define MDB_RDONLY 0x20000 + /** don't fsync metapage after commit */ +#define MDB_NOMETASYNC 0x40000 + /** use writable mmap */ +#define MDB_WRITEMAP 0x80000 + /** use asynchronous msync when MDB_WRITEMAP is used */ +#define MDB_MAPASYNC 0x100000 + /** tie reader locktable slots to #MDB_txn objects instead of to threads */ +#define MDB_NOTLS 0x200000 +/** @} */ + +/** @defgroup mdb_dbi_open Database Flags + * + * Values do not overlap Environment Flags. + * @{ + */ + /** use reverse string keys */ +#define MDB_REVERSEKEY 0x02 + /** use sorted duplicates */ +#define MDB_DUPSORT 0x04 + /** numeric keys in native byte order. + * The keys must all be of the same size. */ +#define MDB_INTEGERKEY 0x08 + /** with #MDB_DUPSORT, sorted dup items have fixed size */ +#define MDB_DUPFIXED 0x10 + /** with #MDB_DUPSORT, dups are numeric in native byte order */ +#define MDB_INTEGERDUP 0x20 + /** with #MDB_DUPSORT, use reverse string dups */ +#define MDB_REVERSEDUP 0x40 + /** create DB if not already existing */ +#define MDB_CREATE 0x40000 +/** @} */ + +/** @defgroup mdb_put Write Flags + * @{ + */ +/** For put: Don't write if the key already exists. */ +#define MDB_NOOVERWRITE 0x10 +/** Only for #MDB_DUPSORT
+ * For put: don't write if the key and data pair already exist.
+ * For mdb_cursor_del: remove all duplicate data items. + */ +#define MDB_NODUPDATA 0x20 +/** For mdb_cursor_put: overwrite the current key/data pair */ +#define MDB_CURRENT 0x40 +/** For put: Just reserve space for data, don't copy it. Return a + * pointer to the reserved space. + */ +#define MDB_RESERVE 0x10000 +/** Data is being appended, don't split full pages. */ +#define MDB_APPEND 0x20000 +/** Duplicate data is being appended, don't split full pages. */ +#define MDB_APPENDDUP 0x40000 +/** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ +#define MDB_MULTIPLE 0x80000 +/* @} */ + +/** @brief Cursor Get operations. + * + * This is the set of all operations for retrieving data + * using a cursor. + */ +typedef enum MDB_cursor_op { + MDB_FIRST, /**< Position at first key/data item */ + MDB_FIRST_DUP, /**< Position at first data item of current key. + Only for #MDB_DUPSORT */ + MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ + MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ + MDB_GET_CURRENT, /**< Return key/data at current cursor position */ + MDB_GET_MULTIPLE, /**< Return all the duplicate data items at the current + cursor position. Only for #MDB_DUPFIXED */ + MDB_LAST, /**< Position at last key/data item */ + MDB_LAST_DUP, /**< Position at last data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT, /**< Position at next data item */ + MDB_NEXT_DUP, /**< Position at next data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT_MULTIPLE, /**< Return all duplicate data items at the next + cursor position. Only for #MDB_DUPFIXED */ + MDB_NEXT_NODUP, /**< Position at first data item of next key */ + MDB_PREV, /**< Position at previous data item */ + MDB_PREV_DUP, /**< Position at previous data item of current key. + Only for #MDB_DUPSORT */ + MDB_PREV_NODUP, /**< Position at last data item of previous key */ + MDB_SET, /**< Position at specified key */ + MDB_SET_KEY, /**< Position at specified key, return key + data */ + MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */ +} MDB_cursor_op; + +/** @defgroup errors Return Codes + * + * BerkeleyDB uses -30800 to -30999, we'll go under them + * @{ + */ + /** Successful result */ +#define MDB_SUCCESS 0 + /** key/data pair already exists */ +#define MDB_KEYEXIST (-30799) + /** key/data pair not found (EOF) */ +#define MDB_NOTFOUND (-30798) + /** Requested page not found - this usually indicates corruption */ +#define MDB_PAGE_NOTFOUND (-30797) + /** Located page was wrong type */ +#define MDB_CORRUPTED (-30796) + /** Update of meta page failed, probably I/O error */ +#define MDB_PANIC (-30795) + /** Environment version mismatch */ +#define MDB_VERSION_MISMATCH (-30794) + /** File is not a valid MDB file */ +#define MDB_INVALID (-30793) + /** Environment mapsize reached */ +#define MDB_MAP_FULL (-30792) + /** Environment maxdbs reached */ +#define MDB_DBS_FULL (-30791) + /** Environment maxreaders reached */ +#define MDB_READERS_FULL (-30790) + /** Too many TLS keys in use - Windows only */ +#define MDB_TLS_FULL (-30789) + /** Txn has too many dirty pages */ +#define MDB_TXN_FULL (-30788) + /** Cursor stack too deep - internal error */ +#define MDB_CURSOR_FULL (-30787) + /** Page has not enough space - internal error */ +#define MDB_PAGE_FULL (-30786) + /** Database contents grew beyond environment mapsize */ +#define MDB_MAP_RESIZED (-30785) + /** Database flags changed or would change */ +#define MDB_INCOMPATIBLE (-30784) + /** Invalid reuse of reader locktable slot */ +#define MDB_BAD_RSLOT (-30783) +#define MDB_LAST_ERRCODE MDB_BAD_RSLOT +/** @} */ + +/** @brief Statistics for a database in the environment */ +typedef struct MDB_stat { + unsigned int ms_psize; /**< Size of a database page. + This is currently the same for all databases. */ + unsigned int ms_depth; /**< Depth (height) of the B-tree */ + size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ + size_t ms_leaf_pages; /**< Number of leaf pages */ + size_t ms_overflow_pages; /**< Number of overflow pages */ + size_t ms_entries; /**< Number of data items */ +} MDB_stat; + +/** @brief Information about the environment */ +typedef struct MDB_envinfo { + void *me_mapaddr; /**< Address of map, if fixed */ + size_t me_mapsize; /**< Size of the data memory map */ + size_t me_last_pgno; /**< ID of the last used page */ + size_t me_last_txnid; /**< ID of the last committed transaction */ + unsigned int me_maxreaders; /**< max reader slots in the environment */ + unsigned int me_numreaders; /**< max reader slots used in the environment */ +} MDB_envinfo; + + /** @brief Return the mdb library version information. + * + * @param[out] major if non-NULL, the library major version number is copied here + * @param[out] minor if non-NULL, the library minor version number is copied here + * @param[out] patch if non-NULL, the library patch version number is copied here + * @retval "version string" The library version as a string + */ +char *mdb_version(int *major, int *minor, int *patch); + + /** @brief Return a string describing a given error code. + * + * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) + * function. If the error code is greater than or equal to 0, then the string + * returned by the system function strerror(3) is returned. If the error code + * is less than 0, an error string corresponding to the MDB library error is + * returned. See @ref errors for a list of MDB-specific error codes. + * @param[in] err The error code + * @retval "error message" The description of the error + */ +char *mdb_strerror(int err); + + /** @brief Create an MDB environment handle. + * + * This function allocates memory for a #MDB_env structure. To release + * the allocated memory and discard the handle, call #mdb_env_close(). + * Before the handle may be used, it must be opened using #mdb_env_open(). + * Various other options may also need to be set before opening the handle, + * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), + * depending on usage requirements. + * @param[out] env The address where the new handle will be stored + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_create(MDB_env **env); + + /** @brief Open an environment handle. + * + * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] path The directory in which the database files reside. This + * directory must already exist and be writable. + * @param[in] flags Special options for this environment. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * Flags set by mdb_env_set_flags() are also used. + *
    + *
  • #MDB_FIXEDMAP + * use a fixed address for the mmap region. This flag must be specified + * when creating the environment, and is stored persistently in the environment. + * If successful, the memory map will always reside at the same virtual address + * and pointers used to reference data items in the database will be constant + * across multiple invocations. This option may not always work, depending on + * how the operating system has allocated memory to shared libraries and other uses. + * The feature is highly experimental. + *
  • #MDB_NOSUBDIR + * By default, MDB creates its environment in a directory whose + * pathname is given in \b path, and creates its data and lock files + * under that directory. With this option, \b path is used as-is for + * the database main data file. The database lock file is the \b path + * with "-lock" appended. + *
  • #MDB_RDONLY + * Open the environment in read-only mode. No write operations will be + * allowed. MDB will still modify the lock file - except on read-only + * filesystems, where MDB does not use locks. + *
  • #MDB_WRITEMAP + * Use a writeable memory map unless MDB_RDONLY is set. This is faster + * and uses fewer mallocs, but loses protection from application bugs + * like wild pointer writes and other bad updates into the database. + * Incompatible with nested transactions. + *
  • #MDB_NOMETASYNC + * Flush system buffers to disk only once per transaction, omit the + * metadata flush. Defer that until the system flushes files to disk, + * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization + * maintains database integrity, but a system crash may undo the last + * committed transaction. I.e. it preserves the ACI (atomicity, + * consistency, isolation) but not D (durability) database property. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_NOSYNC + * Don't flush system buffers to disk when committing a transaction. + * This optimization means a system crash can corrupt the database or + * lose the last transactions if buffers are not yet flushed to disk. + * The risk is governed by how often the system flushes dirty buffers + * to disk and how often #mdb_env_sync() is called. However, if the + * filesystem preserves write order and the #MDB_WRITEMAP flag is not + * used, transactions exhibit ACI (atomicity, consistency, isolation) + * properties and only lose D (durability). I.e. database integrity + * is maintained, but a system crash may undo the final transactions. + * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no + * hint for when to write transactions to disk, unless #mdb_env_sync() + * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_MAPASYNC + * When using #MDB_WRITEMAP, use asynchronous flushes to disk. + * As with #MDB_NOSYNC, a system crash can then corrupt the + * database or lose the last transactions. Calling #mdb_env_sync() + * ensures on-disk database integrity until next commit. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_NOTLS + * Don't use Thread-Local Storage. Tie reader locktable slots to + * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps + * the slot reseved for the #MDB_txn object. A thread may use parallel + * read-only transactions. A read-only transaction may span threads if + * the user synchronizes its use. Applications that multiplex many + * user threads over individual OS threads need this option. Such an + * application must also serialize the write transactions in an OS + * thread, since MDB's write locking is unaware of the user threads. + *
+ * @param[in] mode The UNIX permissions to set on created files. This parameter + * is ignored on Windows. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_VERSION_MISMATCH - the version of the MDB library doesn't match the + * version that created the database environment. + *
  • #MDB_INVALID - the environment file headers are corrupted. + *
  • ENOENT - the directory specified by the path parameter doesn't exist. + *
  • EACCES - the user didn't have permission to access the environment files. + *
  • EAGAIN - the environment was locked by another process. + *
+ */ +int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); + + /** @brief Copy an MDB environment to the specified path. + * + * This function may be used to make a backup of an existing environment. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy(MDB_env *env, const char *path); + + /** @brief Copy an MDB environment to the specified file descriptor. + * + * This function may be used to make a backup of an existing environment. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); + + /** @brief Return statistics about the MDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + */ +int mdb_env_stat(MDB_env *env, MDB_stat *stat); + + /** @brief Return information about the MDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_envinfo structure + * where the information will be copied + */ +int mdb_env_info(MDB_env *env, MDB_envinfo *stat); + + /** @brief Flush the data buffers to disk. + * + * Data is always written to disk when #mdb_txn_commit() is called, + * but the operating system may keep it buffered. MDB always flushes + * the OS buffers upon commit as well, unless the environment was + * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] force If non-zero, force a synchronous flush. Otherwise + * if the environment has the #MDB_NOSYNC flag set the flushes + * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
  • EIO - an error occurred during synchronization. + *
+ */ +int mdb_env_sync(MDB_env *env, int force); + + /** @brief Close the environment and release the memory map. + * + * Only a single thread may call this function. All transactions, databases, + * and cursors must already be closed before calling this function. Attempts to + * use any such handles after calling this function will cause a SIGSEGV. + * The environment handle will be freed and must not be used again after this call. + * @param[in] env An environment handle returned by #mdb_env_create() + */ +void mdb_env_close(MDB_env *env); + + /** @brief Set environment flags. + * + * This may be used to set some flags in addition to those from + * #mdb_env_open(), or to unset these flags. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] flags The flags to change, bitwise OR'ed together + * @param[in] onoff A non-zero value sets the flags, zero clears them. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); + + /** @brief Get environment flags. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] flags The address of an integer to store the flags + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_flags(MDB_env *env, unsigned int *flags); + + /** @brief Return the path that was used in #mdb_env_open(). + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] path Address of a string pointer to contain the path. This + * is the actual string in the environment, not a copy. It should not be + * altered in any way. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_path(MDB_env *env, const char **path); + + /** @brief Set the size of the memory map to use for this environment. + * + * The size should be a multiple of the OS page size. The default is + * 10485760 bytes. The size of the memory map is also the maximum size + * of the database. The value should be chosen as large as possible, + * to accommodate future growth of the database. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * The size may be changed by closing and reopening the environment. + * Any attempt to set a size smaller than the space already consumed + * by the environment will be silently changed to the current size of the used space. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] size The size in bytes + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified, or the environment is already open. + *
+ */ +int mdb_env_set_mapsize(MDB_env *env, size_t size); + + /** @brief Set the maximum number of threads/reader slots for the environment. + * + * This defines the number of slots in the lock table that is used to track readers in the + * the environment. The default is 126. + * Starting a read-only transaction normally ties a lock table slot to the + * current thread until the environment closes or the thread exits. If + * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the + * MDB_txn object until it or the #MDB_env object is destroyed. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] readers The maximum number of reader lock table slots + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified, or the environment is already open. + *
+ */ +int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); + + /** @brief Get the maximum number of threads/reader slots for the environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] readers Address of an integer to store the number of readers + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); + + /** @brief Set the maximum number of named databases for the environment. + * + * This function is only needed if multiple databases will be used in the + * environment. Simpler applications that use the environment as a single + * unnamed database can ignore this option. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbs The maximum number of databases + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified, or the environment is already open. + *
+ */ +int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); + + /** @brief Create a transaction for use with the environment. + * + * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). + * @note A transaction and its cursors must only be used by a single + * thread, and a thread may only have a single transaction at a time. + * If #MDB_NOTLS is in use, this does not apply to read-only transactions. + * @note Cursors may not span transactions. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] parent If this parameter is non-NULL, the new transaction + * will be a nested transaction, with the transaction indicated by \b parent + * as its parent. Transactions may be nested to any level. A parent + * transaction may not issue any other operations besides mdb_txn_begin, + * mdb_txn_abort, or mdb_txn_commit while it has active child transactions. + * @param[in] flags Special options for this transaction. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_RDONLY + * This transaction will not perform any write operations. + *
+ * @param[out] txn Address where the new #MDB_txn handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + *
  • #MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's + * mapsize and the environment must be shut down. + *
  • #MDB_READERS_FULL - a read-only transaction was requested and + * the reader lock table is full. See #mdb_env_set_maxreaders(). + *
  • ENOMEM - out of memory. + *
+ */ +int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); + + /** @brief Commit all the operations of a transaction into the database. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
  • ENOSPC - no more disk space. + *
  • EIO - a low-level I/O error occurred while writing. + *
  • ENOMEM - out of memory. + *
+ */ +int mdb_txn_commit(MDB_txn *txn); + + /** @brief Abandon all the operations of the transaction instead of saving them. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_abort(MDB_txn *txn); + + /** @brief Reset a read-only transaction. + * + * Abort the transaction like #mdb_txn_abort(), but keep the transaction + * handle. #mdb_txn_renew() may reuse the handle. This saves allocation + * overhead if the process will start a new read-only transaction soon, + * and also locking overhead if #MDB_NOTLS is in use. The reader table + * lock is released, but the table slot stays tied to its thread or + * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free + * its lock table slot if MDB_NOTLS is in use. + * Cursors opened within the transaction must not be used + * again after this call, except with #mdb_cursor_renew(). + * Reader locks generally don't interfere with writers, but they keep old + * versions of database pages allocated. Thus they prevent the old pages + * from being reused when writers commit new data, and so under heavy load + * the database size may grow much more rapidly than otherwise. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_reset(MDB_txn *txn); + + /** @brief Renew a read-only transaction. + * + * This acquires a new reader lock for a transaction handle that had been + * released by #mdb_txn_reset(). It must be called before a reset transaction + * may be used again. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_txn_renew(MDB_txn *txn); + +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_close(env,dbi) mdb_dbi_close(env,dbi) + + /** @brief Open a database in the environment. + * + * A database handle denotes the name and parameters of a database, + * independently of whether such a database exists. + * The database handle may be discarded by calling #mdb_dbi_close(). + * The old database handle is returned if the database was already open. + * The handle must only be closed once. + * The database handle will be private to the current transaction until + * the transaction is successfully committed. If the transaction is + * aborted the handle will be closed automatically. + * After a successful commit the + * handle will reside in the shared environment, and may be used + * by other transactions. This function must not be called from + * multiple concurrent transactions. A transaction that uses this function + * must finish (either commit or abort) before any other transaction may + * use this function. + * + * To use named databases (with name != NULL), #mdb_env_set_maxdbs() + * must be called before opening the environment. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] name The name of the database to open. If only a single + * database is needed in the environment, this value may be NULL. + * @param[in] flags Special options for this database. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_REVERSEKEY + * Keys are strings to be compared in reverse order, from the end + * of the strings to the beginning. By default, Keys are treated as strings and + * compared from beginning to end. + *
  • #MDB_DUPSORT + * Duplicate keys may be used in the database. (Or, from another perspective, + * keys may have multiple data items, stored in sorted order.) By default + * keys must be unique and may have only a single data item. + *
  • #MDB_INTEGERKEY + * Keys are binary integers in native byte order. Setting this option + * requires all keys to be the same size, typically sizeof(int) + * or sizeof(size_t). + *
  • #MDB_DUPFIXED + * This flag may only be used in combination with #MDB_DUPSORT. This option + * tells the library that the data items for this database are all the same + * size, which allows further optimizations in storage and retrieval. When + * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE + * cursor operations may be used to retrieve multiple items at once. + *
  • #MDB_INTEGERDUP + * This option specifies that duplicate data items are also integers, and + * should be sorted as such. + *
  • #MDB_REVERSEDUP + * This option specifies that duplicate data items should be compared as + * strings in reverse order. + *
  • #MDB_CREATE + * Create the named database if it doesn't exist. This option is not + * allowed in a read-only transaction or a read-only environment. + *
+ * @param[out] dbi Address where the new #MDB_dbi handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_NOTFOUND - the specified database doesn't exist in the environment + * and #MDB_CREATE was not specified. + *
  • #MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). + *
+ */ +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); + + /** @brief Retrieve statistics for a database. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); + + /** @brief Close a database handle. + * + * This call is not mutex protected. Handles should only be closed by + * a single thread, and only if no other threads are going to reference + * the database handle or one of its cursors any further. Do not close + * a handle if an existing transaction has modified its database. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); + + /** @brief Delete a database and/or free all its pages. + * + * If the \b del parameter is 1, the DB handle will be closed + * and the DB will be deleted. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] del 1 to delete the DB from the environment, + * 0 to just free its pages. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); + + /** @brief Set a custom key comparison function for a database. + * + * The comparison function is called whenever it is necessary to compare a + * key specified by the application with a key currently stored in the database. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating + * before longer keys. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. + * + * This comparison function is called whenever it is necessary to compare a data + * item specified by the application with a data item currently stored in the database. + * This function only takes effect if the database was opened with the #MDB_DUPSORT + * flag. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating + * before longer items. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a relocation function for a #MDB_FIXEDMAP database. + * + * @todo The relocation function is called whenever it is necessary to move the data + * of an item to a different position in the database (e.g. through tree + * balancing operations, shifts as a result of adds or deletes, etc.). It is + * intended to allow address/position-dependent data items to be stored in + * a database in an environment opened with the #MDB_FIXEDMAP option. + * Currently the relocation feature is unimplemented and setting + * this function has no effect. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] rel A #MDB_rel_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); + + /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. + * + * See #mdb_set_relfunc and #MDB_rel_func for more details. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * It will be passed to the callback function set by #mdb_set_relfunc + * as its \b relctx parameter whenever the callback is invoked. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); + + /** @brief Get items from a database. + * + * This function retrieves key/data pairs from the database. The address + * and length of the data associated with the specified \b key are returned + * in the structure to which \b data refers. + * If the database supports duplicate keys (#MDB_DUPSORT) then the + * first data item for the key will be returned. Retrieval of other + * items requires the use of #mdb_cursor_get(). + * + * @note The memory pointed to by the returned values is owned by the + * database. The caller need not dispose of the memory, and may not + * modify it in any way. For values returned in a read-only transaction + * any modification attempts will cause a SIGSEGV. + * @note Values returned from the database are valid only until a + * subsequent update operation, or the end of the transaction. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to search for in the database + * @param[out] data The data corresponding to the key + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_NOTFOUND - the key was not in the database. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Store items into a database. + * + * This function stores key/data pairs in the database. The default behavior + * is to enter the new key/data pair, replacing any previously existing key + * if duplicates are disallowed, or adding a duplicate data item if + * duplicates are allowed (#MDB_DUPSORT). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to store in the database + * @param[in,out] data The data to store + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). The \b data + * parameter will be set to point to the existing item. + *
  • #MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later - before + * the next update operation or the transaction ends. This saves + * an extra memcpy if the data is being generated later. + *
  • #MDB_APPEND - append the given key/data pair to the end of the + * database. No key comparisons are performed. This option allows + * fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * data corruption. + *
  • #MDB_APPENDDUP - as above, but for sorted dup data. + *
+ * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete items from a database. + * + * This function removes key/data pairs from the database. + * If the database does not support sorted duplicate data items + * (#MDB_DUPSORT) the data parameter is ignored. + * If the database supports sorted duplicates and the data parameter + * is NULL, all of the duplicate data items for the key will be + * deleted. Otherwise, if the data parameter is non-NULL + * only the matching data item will be deleted. + * This function will return #MDB_NOTFOUND if the specified key/data + * pair is not in the database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to delete from the database + * @param[in] data The data to delete + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Create a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * A cursor cannot be used when its database handle is closed. Nor + * when its transaction has ended, except with #mdb_cursor_renew(). + * It can be discarded with #mdb_cursor_close(). + * A cursor in a write-transaction can be closed before its transaction + * ends, and will otherwise be closed when its transaction ends. + * A cursor in a read-only transaction must be closed explicitly, before + * or after its transaction ends. It can be reused with + * #mdb_cursor_renew() before finally closing it. + * @note Earlier documentation said that cursors in every transaction + * were closed when the transaction committed or aborted. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] cursor Address where the new #MDB_cursor handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); + + /** @brief Close a cursor handle. + * + * The cursor handle will be freed and must not be used again after this call. + * Its transaction must still be live if it is a write-transaction. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +void mdb_cursor_close(MDB_cursor *cursor); + + /** @brief Renew a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * Cursors that are only used in read-only + * transactions may be re-used, to avoid unnecessary malloc/free overhead. + * The cursor may be associated with a new read-only transaction, and + * referencing the same database handle as it was created with. + * This may be done whether the previous transaction is live or dead. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); + + /** @brief Return the cursor's transaction handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); + + /** @brief Return the cursor's database handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); + + /** @brief Retrieve by cursor. + * + * This function retrieves key/data pairs from the database. The address and length + * of the key are returned in the object to which \b key refers (except for the + * case of the #MDB_SET option, in which the \b key object is unchanged), and + * the address and length of the data are returned in the object to which \b data + * refers. + * See #mdb_get() for restrictions on using the output values. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in,out] key The key for a retrieved item + * @param[in,out] data The data of a retrieved item + * @param[in] op A cursor operation #MDB_cursor_op + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_NOTFOUND - no matching key found. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + MDB_cursor_op op); + + /** @brief Store by cursor. + * + * This function stores key/data pairs into the database. + * If the function fails for any reason, the state of the cursor will be + * unchanged. If the function succeeds and an item is inserted into the + * database, the cursor is always positioned to refer to the newly inserted item. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] key The key operated on. + * @param[in] data The data operated on. + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + *
    + *
  • #MDB_CURRENT - overwrite the data of the key/data pair to which + * the cursor refers with the specified data item. The \b key + * parameter is ignored. + *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). + *
  • #MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later. This saves + * an extra memcpy if the data is being generated later. + *
  • #MDB_APPEND - append the given key/data pair to the end of the + * database. No key comparisons are performed. This option allows + * fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * data corruption. + *
  • #MDB_APPENDDUP - as above, but for sorted dup data. + *
  • #MDB_MULTIPLE - store multiple contiguous data elements in a + * single request. This flag may only be specified if the database + * was opened with #MDB_DUPFIXED. The \b data argument must be an + * array of two MDB_vals. The mv_size of the first MDB_val must be + * the size of a single data element. The mv_data of the first MDB_val + * must point to the beginning of the array of contiguous data elements. + * The mv_size of the second MDB_val must be the count of the number + * of data elements to store. On return this field will be set to + * the count of the number of elements actually written. The mv_data + * of the second MDB_val is unused. + *
+ * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. + *
  • EACCES - an attempt was made to modify a read-only database. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete current key/data pair + * + * This function deletes the key/data pair to which the cursor refers. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + *
    + *
  • #MDB_NODUPDATA - delete all of the data items for the current key. + * This flag may only be specified if the database was opened with #MDB_DUPSORT. + *
+ * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EACCES - an attempt was made to modify a read-only database. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); + + /** @brief Return count of duplicates for current key. + * + * This call is only valid on databases that support sorted duplicate + * data items #MDB_DUPSORT. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[out] countp Address where the count will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - cursor is not initialized, or an invalid parameter was specified. + *
+ */ +int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two data items were keys in the + * specified database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two items were data items of + * the specified database. The database must have the #MDB_DUPSORT flag. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); +/** @} */ + +#ifdef __cplusplus +} +#endif +#endif /* _LMDB_H_ */ diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c new file mode 100644 index 0000000000..620e5b51ff --- /dev/null +++ b/libraries/liblmdb/mdb.c @@ -0,0 +1,7488 @@ +/** @file mdb.c + * @brief memory-mapped database library + * + * A Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. + */ +/* + * Copyright 2011-2013 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#include +#ifdef HAVE_SYS_FILE_H +#include +#endif +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) +#include +#include /* defines BYTE_ORDER on HPUX and Solaris */ +#endif + +#if defined(__APPLE__) || defined (BSD) +# define MDB_USE_POSIX_SEM 1 +# define MDB_FDATASYNC fsync +#elif defined(ANDROID) +# define MDB_FDATASYNC fsync +#endif + +#ifndef _WIN32 +#include +#ifdef MDB_USE_POSIX_SEM +#include +#endif +#endif + +#ifdef USE_VALGRIND +#include +#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) +#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) +#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) +#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) +#else +#define VGMEMP_CREATE(h,r,z) +#define VGMEMP_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) +#define VGMEMP_DESTROY(h) +#define VGMEMP_DEFINED(a,s) +#endif + +#ifndef BYTE_ORDER +# if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) +/* Solaris just defines one or the other */ +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# ifdef _LITTLE_ENDIAN +# define BYTE_ORDER LITTLE_ENDIAN +# else +# define BYTE_ORDER BIG_ENDIAN +# endif +# else +# define BYTE_ORDER __BYTE_ORDER +# endif +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN __LITTLE_ENDIAN +#endif +#ifndef BIG_ENDIAN +#define BIG_ENDIAN __BIG_ENDIAN +#endif + +#if defined(__i386) || defined(__x86_64) || defined(_M_IX86) +#define MISALIGNED_OK 1 +#endif + +#include "lmdb.h" +#include "midl.h" + +#if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) +# error "Unknown or unsupported endianness (BYTE_ORDER)" +#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +# error "Two's complement, reasonably sized integer types, please" +#endif + +/** @defgroup internal MDB Internals + * @{ + */ +/** @defgroup compat Windows Compatibility Macros + * A bunch of macros to minimize the amount of platform-specific ifdefs + * needed throughout the rest of the code. When the features this library + * needs are similar enough to POSIX to be hidden in a one-or-two line + * replacement, this macro approach is used. + * @{ + */ +#ifdef _WIN32 +#define pthread_t DWORD +#define pthread_mutex_t HANDLE +#define pthread_key_t DWORD +#define pthread_self() GetCurrentThreadId() +#define pthread_key_create(x,y) \ + ((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0) +#define pthread_key_delete(x) TlsFree(x) +#define pthread_getspecific(x) TlsGetValue(x) +#define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) +#define pthread_mutex_unlock(x) ReleaseMutex(x) +#define pthread_mutex_lock(x) WaitForSingleObject(x, INFINITE) +#define LOCK_MUTEX_R(env) pthread_mutex_lock((env)->me_rmutex) +#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock((env)->me_rmutex) +#define LOCK_MUTEX_W(env) pthread_mutex_lock((env)->me_wmutex) +#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock((env)->me_wmutex) +#define getpid() GetCurrentProcessId() +#define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) +#define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) +#define ErrCode() GetLastError() +#define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;} +#define close(fd) (CloseHandle(fd) ? 0 : -1) +#define munmap(ptr,len) UnmapViewOfFile(ptr) +#else + +#ifdef MDB_USE_POSIX_SEM + +#define LOCK_MUTEX_R(env) mdb_sem_wait((env)->me_rmutex) +#define UNLOCK_MUTEX_R(env) sem_post((env)->me_rmutex) +#define LOCK_MUTEX_W(env) mdb_sem_wait((env)->me_wmutex) +#define UNLOCK_MUTEX_W(env) sem_post((env)->me_wmutex) + +static int +mdb_sem_wait(sem_t *sem) +{ + int rc; + while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ; + return rc; +} + +#else + /** Lock the reader mutex. + */ +#define LOCK_MUTEX_R(env) pthread_mutex_lock(&(env)->me_txns->mti_mutex) + /** Unlock the reader mutex. + */ +#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock(&(env)->me_txns->mti_mutex) + + /** Lock the writer mutex. + * Only a single write transaction is allowed at a time. Other writers + * will block waiting for this mutex. + */ +#define LOCK_MUTEX_W(env) pthread_mutex_lock(&(env)->me_txns->mti_wmutex) + /** Unlock the writer mutex. + */ +#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock(&(env)->me_txns->mti_wmutex) +#endif /* MDB_USE_POSIX_SEM */ + + /** Get the error code for the last failed system function. + */ +#define ErrCode() errno + + /** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#define HANDLE int + + /** A value for an invalid file handle. + * Mainly used to initialize file variables and signify that they are + * unused. + */ +#define INVALID_HANDLE_VALUE (-1) + + /** Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. + */ +#define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) +#endif + +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) +#define MNAME_LEN 32 +#else +#define MNAME_LEN (sizeof(pthread_mutex_t)) +#endif + +/** @} */ + +#ifndef _WIN32 +/** A flag for opening a file and requesting synchronous data writes. + * This is only used when writing a meta page. It's not strictly needed; + * we could just do a normal write and then immediately perform a flush. + * But if this flag is available it saves us an extra system call. + * + * @note If O_DSYNC is undefined but exists in /usr/include, + * preferably set some compiler flag to get the definition. + * Otherwise compile with the less efficient -DMDB_DSYNC=O_SYNC. + */ +#ifndef MDB_DSYNC +# define MDB_DSYNC O_DSYNC +#endif +#endif + +/** Function for flushing the data of a file. Define this to fsync + * if fdatasync() is not supported. + */ +#ifndef MDB_FDATASYNC +# define MDB_FDATASYNC fdatasync +#endif + +#ifndef MDB_MSYNC +# define MDB_MSYNC(addr,len,flags) msync(addr,len,flags) +#endif + +#ifndef MS_SYNC +#define MS_SYNC 1 +#endif + +#ifndef MS_ASYNC +#define MS_ASYNC 0 +#endif + + /** A page number in the database. + * Note that 64 bit page numbers are overkill, since pages themselves + * already represent 12-13 bits of addressable memory, and the OS will + * always limit applications to a maximum of 63 bits of address space. + * + * @note In the #MDB_node structure, we only store 48 bits of this value, + * which thus limits us to only 60 bits of addressable data. + */ +typedef MDB_ID pgno_t; + + /** A transaction ID. + * See struct MDB_txn.mt_txnid for details. + */ +typedef MDB_ID txnid_t; + +/** @defgroup debug Debug Macros + * @{ + */ +#ifndef MDB_DEBUG + /** Enable debug output. + * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs + * read from and written to the database (used for free space management). + */ +#define MDB_DEBUG 0 +#endif + +#if !(__STDC_VERSION__ >= 199901L || defined(__GNUC__)) +# undef MDB_DEBUG +# define MDB_DEBUG 0 +# define DPRINTF (void) /* Vararg macros may be unsupported */ +#elif MDB_DEBUG +static int mdb_debug; +static txnid_t mdb_debug_start; + + /** Print a debug message with printf formatting. */ +# define DPRINTF(fmt, ...) /**< Requires 2 or more args */ \ + ((void) ((mdb_debug) && \ + fprintf(stderr, "%s:%d " fmt "\n", __func__, __LINE__, __VA_ARGS__))) +#else +# define DPRINTF(fmt, ...) ((void) 0) +# define MDB_DEBUG_SKIP +#endif + /** Print a debug string. + * The string is printed literally, with no format processing. + */ +#define DPUTS(arg) DPRINTF("%s", arg) +/** @} */ + + /** A default memory page size. + * The actual size is platform-dependent, but we use this for + * boot-strapping. We probably should not be using this any more. + * The #GET_PAGESIZE() macro is used to get the actual size. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. + */ +#define MDB_PAGESIZE 4096 + + /** The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. + */ +#define MDB_MINKEYS 2 + + /** A stamp that identifies a file as an MDB file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. + */ +#define MDB_MAGIC 0xBEEFC0DE + + /** The version number for a database's file format. */ +#define MDB_VERSION 1 + + /** @brief The maximum size of a key in the database. + * + * The library rejects bigger keys, and cannot deal with records + * with bigger keys stored by a library with bigger max keysize. + * + * We require that keys all fit onto a regular page. This limit + * could be raised a bit further if needed; to something just + * under #MDB_PAGESIZE / #MDB_MINKEYS. + * + * Note that data items in an #MDB_DUPSORT database are actually keys + * of a subDB, so they're also limited to this size. + */ +#ifndef MDB_MAXKEYSIZE +#define MDB_MAXKEYSIZE 511 +#endif + + /** @brief The maximum size of a data item. + * + * We only store a 32 bit value for node sizes. + */ +#define MAXDATASIZE 0xffffffffUL + +#if MDB_DEBUG + /** A key buffer. + * @ingroup debug + * This is used for printing a hex dump of a key's contents. + */ +#define DKBUF char kbuf[(MDB_MAXKEYSIZE*2+1)] + /** Display a key in hex. + * @ingroup debug + * Invoke a function to display a key in hex. + */ +#define DKEY(x) mdb_dkey(x, kbuf) +#else +#define DKBUF typedef int dummy_kbuf /* so we can put ';' after */ +#define DKEY(x) 0 +#endif + + /** An invalid page number. + * Mainly used to denote an empty tree. + */ +#define P_INVALID (~(pgno_t)0) + + /** Test if the flags \b f are set in a flag word \b w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + + /** Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. + */ +typedef uint16_t indx_t; + + /** Default size of memory map. + * This is certainly too small for any actual applications. Apps should always set + * the size explicitly using #mdb_env_set_mapsize(). + */ +#define DEFAULT_MAPSIZE 1048576 + +/** @defgroup readers Reader Lock Table + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent read + * transactions started by the same thread need no further locking to proceed. + * + * If #MDB_NOTLS is set, the slot address is not saved in thread-specific data. + * + * No reader table is used if the database is on a read-only filesystem. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. + * @{ + */ + /** Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. 126 readers plus a + * couple mutexes fit exactly into 8KB on my development machine. + * Applications should set the table size using #mdb_env_set_maxreaders(). + */ +#define DEFAULT_READERS 126 + + /** The size of a CPU cache line in bytes. We want our lock structures + * aligned to this size to avoid false cache line sharing in the + * lock table. + * This value works for most CPUs. For Itanium this should be 128. + */ +#ifndef CACHELINE +#define CACHELINE 64 +#endif + + /** The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * @note We currently don't check for stale records. We simply re-init + * the table when we know that we're the only process opening the + * lock file. + */ +typedef struct MDB_rxbody { + /** Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. + */ + txnid_t mrb_txnid; + /** The process ID of the process owning this reader txn. */ + pid_t mrb_pid; + /** The thread ID of the thread owning this txn. */ + pthread_t mrb_tid; +} MDB_rxbody; + + /** The actual reader record, with cacheline padding. */ +typedef struct MDB_reader { + union { + MDB_rxbody mrx; + /** shorthand for mrb_txnid */ +#define mr_txnid mru.mrx.mrb_txnid +#define mr_pid mru.mrx.mrb_pid +#define mr_tid mru.mrx.mrb_tid + /** cache line alignment */ + char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mru; +} MDB_reader; + + /** The header for the reader table. + * The table resides in a memory-mapped file. (This is a different file + * than is used for the main database.) + * + * For POSIX the actual mutexes reside in the shared memory of this + * mapped file. On Windows, mutexes are named objects allocated by the + * kernel; we store the mutex names in this mapped file so that other + * processes can grab them. This same approach is also used on + * MacOSX/Darwin (using named semaphores) since MacOSX doesn't support + * process-shared POSIX mutexes. For these cases where a named object + * is used, the object name is derived from a 64 bit FNV hash of the + * environment pathname. As such, naming collisions are extremely + * unlikely. If a collision occurs, the results are unpredictable. + */ +typedef struct MDB_txbody { + /** Stamp identifying this as an MDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mtb_magic; + /** Version number of this lock file. Must be set to #MDB_VERSION. */ + uint32_t mtb_version; +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mtb_rmname[MNAME_LEN]; +#else + /** Mutex protecting access to this table. + * This is the reader lock that #LOCK_MUTEX_R acquires. + */ + pthread_mutex_t mtb_mutex; +#endif + /** The ID of the last transaction committed to the database. + * This is recorded here only for convenience; the value can always + * be determined by reading the main database meta pages. + */ + txnid_t mtb_txnid; + /** The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. + */ + unsigned mtb_numreaders; +} MDB_txbody; + + /** The actual reader table definition. */ +typedef struct MDB_txninfo { + union { + MDB_txbody mtb; +#define mti_magic mt1.mtb.mtb_magic +#define mti_version mt1.mtb.mtb_version +#define mti_mutex mt1.mtb.mtb_mutex +#define mti_rmname mt1.mtb.mtb_rmname +#define mti_txnid mt1.mtb.mtb_txnid +#define mti_numreaders mt1.mtb.mtb_numreaders + char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mt1; + union { +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mt2_wmname[MNAME_LEN]; +#define mti_wmname mt2.mt2_wmname +#else + pthread_mutex_t mt2_wmutex; +#define mti_wmutex mt2.mt2_wmutex +#endif + char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)]; + } mt2; + MDB_reader mti_readers[1]; +} MDB_txninfo; +/** @} */ + +/** Common header for all page types. + * Overflow records occupy a number of contiguous pages with no + * headers on any page after the first. + */ +typedef struct MDB_page { +#define mp_pgno mp_p.p_pgno +#define mp_next mp_p.p_next + union { + pgno_t p_pgno; /**< page number */ + void * p_next; /**< for in-memory list of freed structs */ + } mp_p; + uint16_t mp_pad; +/** @defgroup mdb_page Page Flags + * @ingroup internal + * Flags for the page headers. + * @{ + */ +#define P_BRANCH 0x01 /**< branch page */ +#define P_LEAF 0x02 /**< leaf page */ +#define P_OVERFLOW 0x04 /**< overflow page */ +#define P_META 0x08 /**< meta page */ +#define P_DIRTY 0x10 /**< dirty page */ +#define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ +#define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ +/** @} */ + uint16_t mp_flags; /**< @ref mdb_page */ +#define mp_lower mp_pb.pb.pb_lower +#define mp_upper mp_pb.pb.pb_upper +#define mp_pages mp_pb.pb_pages + union { + struct { + indx_t pb_lower; /**< lower bound of free space */ + indx_t pb_upper; /**< upper bound of free space */ + } pb; + uint32_t pb_pages; /**< number of overflow pages */ + } mp_pb; + indx_t mp_ptrs[1]; /**< dynamic size */ +} MDB_page; + + /** Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned) offsetof(MDB_page, mp_ptrs)) + + /** Address of first usable data byte in a page, after the header */ +#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ)) + + /** Number of nodes on a page */ +#define NUMKEYS(p) (((p)->mp_lower - PAGEHDRSZ) >> 1) + + /** The amount of space remaining in the page */ +#define SIZELEFT(p) (indx_t)((p)->mp_upper - (p)->mp_lower) + + /** The percentage of space used in the page, in tenths of a percent. */ +#define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \ + ((env)->me_psize - PAGEHDRSZ)) + /** The minimum page fill factor, in tenths of a percent. + * Pages emptier than this are candidates for merging. + */ +#define FILL_THRESHOLD 250 + + /** Test if a page is a leaf page */ +#define IS_LEAF(p) F_ISSET((p)->mp_flags, P_LEAF) + /** Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) F_ISSET((p)->mp_flags, P_LEAF2) + /** Test if a page is a branch page */ +#define IS_BRANCH(p) F_ISSET((p)->mp_flags, P_BRANCH) + /** Test if a page is an overflow page */ +#define IS_OVERFLOW(p) F_ISSET((p)->mp_flags, P_OVERFLOW) + /** Test if a page is a sub page */ +#define IS_SUBP(p) F_ISSET((p)->mp_flags, P_SUBP) + + /** The number of overflow pages needed to store the given size. */ +#define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) + + /** Header for a single key/data pair within a page. + * We guarantee 2-byte alignment for nodes. + */ +typedef struct MDB_node { + /** lo and hi are used for data size on leaf nodes and for + * child pgno on branch nodes. On 64 bit platforms, flags + * is also used for pgno. (Branch nodes have no flags). + * They are in host byte order in case that lets some + * accesses be optimized into a 32-bit word access. + */ +#define mn_lo mn_offset[BYTE_ORDER!=LITTLE_ENDIAN] +#define mn_hi mn_offset[BYTE_ORDER==LITTLE_ENDIAN] /**< part of dsize or pgno */ + unsigned short mn_offset[2]; /**< storage for #mn_lo and #mn_hi */ +/** @defgroup mdb_node Node Flags + * @ingroup internal + * Flags for node headers. + * @{ + */ +#define F_BIGDATA 0x01 /**< data put on overflow page */ +#define F_SUBDATA 0x02 /**< data is a sub-database */ +#define F_DUPDATA 0x04 /**< data has duplicates */ + +/** valid flags for #mdb_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND) + +/** @} */ + unsigned short mn_flags; /**< @ref mdb_node */ + unsigned short mn_ksize; /**< key size */ + char mn_data[1]; /**< key and data are appended here */ +} MDB_node; + + /** Size of the node header, excluding dynamic data at the end */ +#define NODESIZE offsetof(MDB_node, mn_data) + + /** Bit position of top word in page number, for shifting mn_flags */ +#define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0) + + /** Size of a node in a branch page with a given key. + * This is just the node header plus the key, there is no data. + */ +#define INDXSIZE(k) (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size)) + + /** Size of a node in a leaf page with a given key and data. + * This is node header plus key plus data size. + */ +#define LEAFSIZE(k, d) (NODESIZE + (k)->mv_size + (d)->mv_size) + + /** Address of node \b i in page \b p */ +#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + (p)->mp_ptrs[i])) + + /** Address of the key for the node */ +#define NODEKEY(node) (void *)((node)->mn_data) + + /** Address of the data for a node */ +#define NODEDATA(node) (void *)((char *)(node)->mn_data + (node)->mn_ksize) + + /** Get the page number pointed to by a branch node */ +#define NODEPGNO(node) \ + ((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \ + (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0)) + /** Set the page number in a branch node */ +#define SETPGNO(node,pgno) do { \ + (node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \ + if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0) + + /** Get the size of the data in a leaf node */ +#define NODEDSZ(node) ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16)) + /** Set the size of the data for a leaf node */ +#define SETDSZ(node,size) do { \ + (node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0) + /** The size of a key in a node */ +#define NODEKSZ(node) ((node)->mn_ksize) + + /** Copy a page number from src to dst */ +#ifdef MISALIGNED_OK +#define COPY_PGNO(dst,src) dst = src +#else +#if SIZE_MAX > 4294967295UL +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d++ = *s++; \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#else +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#endif +#endif + /** The address of a key in a LEAF2 page. + * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs. + * There are no node headers, keys are stored contiguously. + */ +#define LEAF2KEY(p, i, ks) ((char *)(p) + PAGEHDRSZ + ((i)*(ks))) + + /** Set the \b node's key into \b key, if requested. */ +#define MDB_GET_KEY(node, key) { if ((key) != NULL) { \ + (key)->mv_size = NODEKSZ(node); (key)->mv_data = NODEKEY(node); } } + + /** Information about a single database in the environment. */ +typedef struct MDB_db { + uint32_t md_pad; /**< also ksize for LEAF2 pages */ + uint16_t md_flags; /**< @ref mdb_dbi_open */ + uint16_t md_depth; /**< depth of this tree */ + pgno_t md_branch_pages; /**< number of internal pages */ + pgno_t md_leaf_pages; /**< number of leaf pages */ + pgno_t md_overflow_pages; /**< number of overflow pages */ + size_t md_entries; /**< number of data items */ + pgno_t md_root; /**< the root page of this tree */ +} MDB_db; + + /** mdb_dbi_open flags */ +#define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ +#define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) +#define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ + MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) + + /** Handle for the DB used to track free pages. */ +#define FREE_DBI 0 + /** Handle for the default DB. */ +#define MAIN_DBI 1 + + /** Meta page content. */ +typedef struct MDB_meta { + /** Stamp identifying this as an MDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mm_magic; + /** Version number of this lock file. Must be set to #MDB_VERSION. */ + uint32_t mm_version; + void *mm_address; /**< address for fixed mapping */ + size_t mm_mapsize; /**< size of mmap region */ + MDB_db mm_dbs[2]; /**< first is free space, 2nd is main db */ + /** The size of pages used in this DB */ +#define mm_psize mm_dbs[0].md_pad + /** Any persistent environment flags. @ref mdb_env */ +#define mm_flags mm_dbs[0].md_flags + pgno_t mm_last_pg; /**< last used page in file */ + txnid_t mm_txnid; /**< txnid that committed this page */ +} MDB_meta; + + /** Buffer for a stack-allocated dirty page. + * The members define size and alignment, and silence type + * aliasing warnings. They are not used directly; that could + * mean incorrectly using several union members in parallel. + */ +typedef union MDB_pagebuf { + char mb_raw[MDB_PAGESIZE]; + MDB_page mb_page; + struct { + char mm_pad[PAGEHDRSZ]; + MDB_meta mm_meta; + } mb_metabuf; +} MDB_pagebuf; + + /** Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. + */ +typedef struct MDB_dbx { + MDB_val md_name; /**< name of the database */ + MDB_cmp_func *md_cmp; /**< function for comparing keys */ + MDB_cmp_func *md_dcmp; /**< function for comparing data items */ + MDB_rel_func *md_rel; /**< user relocate function */ + void *md_relctx; /**< user-provided context for md_rel */ +} MDB_dbx; + + /** A database transaction. + * Every operation requires a transaction handle. + */ +struct MDB_txn { + MDB_txn *mt_parent; /**< parent of a nested txn */ + MDB_txn *mt_child; /**< nested txn under this txn */ + pgno_t mt_next_pgno; /**< next unallocated page */ + /** The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. + */ + txnid_t mt_txnid; + MDB_env *mt_env; /**< the DB environment */ + /** The list of pages that became unused during this transaction. + */ + MDB_IDL mt_free_pgs; + union { + MDB_ID2L dirty_list; /**< for write txns: modified pages */ + MDB_reader *reader; /**< this thread's reader table slot or NULL */ + } mt_u; + /** Array of records for each DB known in the environment. */ + MDB_dbx *mt_dbxs; + /** Array of MDB_db records for each known DB */ + MDB_db *mt_dbs; +/** @defgroup mt_dbflag Transaction DB Flags + * @ingroup internal + * @{ + */ +#define DB_DIRTY 0x01 /**< DB was written in this txn */ +#define DB_STALE 0x02 /**< DB record is older than txnID */ +#define DB_NEW 0x04 /**< DB handle opened in this txn */ +#define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ +/** @} */ + /** In write txns, array of cursors for each DB */ + MDB_cursor **mt_cursors; + /** Array of flags for each DB */ + unsigned char *mt_dbflags; + /** Number of DB records in use. This number only ever increments; + * we don't decrement it when individual DB handles are closed. + */ + MDB_dbi mt_numdbs; + +/** @defgroup mdb_txn Transaction Flags + * @ingroup internal + * @{ + */ +#define MDB_TXN_RDONLY 0x01 /**< read-only transaction */ +#define MDB_TXN_ERROR 0x02 /**< an error has occurred */ +#define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ +/** @} */ + unsigned int mt_flags; /**< @ref mdb_txn */ + /** dirty_list maxsize - # of allocated pages allowed, including in parent txns */ + unsigned int mt_dirty_room; + /** Tracks which of the two meta pages was used at the start + * of this transaction. + */ + unsigned int mt_toggle; +}; + +/** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. + */ +#define CURSOR_STACK 32 + +struct MDB_xcursor; + + /** Cursors are used for all DB operations */ +struct MDB_cursor { + /** Next cursor on this DB in this txn */ + MDB_cursor *mc_next; + /** Backup of the original cursor if this cursor is a shadow */ + MDB_cursor *mc_backup; + /** Context used for databases with #MDB_DUPSORT, otherwise NULL */ + struct MDB_xcursor *mc_xcursor; + /** The transaction that owns this cursor */ + MDB_txn *mc_txn; + /** The database handle this cursor operates on */ + MDB_dbi mc_dbi; + /** The database record for this cursor */ + MDB_db *mc_db; + /** The database auxiliary record for this cursor */ + MDB_dbx *mc_dbx; + /** The @ref mt_dbflag for this database */ + unsigned char *mc_dbflag; + unsigned short mc_snum; /**< number of pushed pages */ + unsigned short mc_top; /**< index of top page, normally mc_snum-1 */ +/** @defgroup mdb_cursor Cursor Flags + * @ingroup internal + * Cursor state flags. + * @{ + */ +#define C_INITIALIZED 0x01 /**< cursor has been initialized and is valid */ +#define C_EOF 0x02 /**< No more data */ +#define C_SUB 0x04 /**< Cursor is a sub-cursor */ +#define C_SPLITTING 0x20 /**< Cursor is in page_split */ +#define C_UNTRACK 0x40 /**< Un-track cursor when closing */ +/** @} */ + unsigned int mc_flags; /**< @ref mdb_cursor */ + MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ +}; + + /** Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. + */ +typedef struct MDB_xcursor { + /** A sub-cursor for traversing the Dup DB */ + MDB_cursor mx_cursor; + /** The database record for this Dup DB */ + MDB_db mx_db; + /** The auxiliary DB record for this Dup DB */ + MDB_dbx mx_dbx; + /** The @ref mt_dbflag for this Dup DB */ + unsigned char mx_dbflag; +} MDB_xcursor; + + /** State of FreeDB old pages, stored in the MDB_env */ +typedef struct MDB_pgstate { + pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ + txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ +} MDB_pgstate; + + /** The database environment. */ +struct MDB_env { + HANDLE me_fd; /**< The main data file */ + HANDLE me_lfd; /**< The lock file */ + HANDLE me_mfd; /**< just for writing the meta pages */ + /** Failed to update the meta page. Probably an I/O error. */ +#define MDB_FATAL_ERROR 0x80000000U + /** Some fields are initialized. */ +#define MDB_ENV_ACTIVE 0x20000000U + /** me_txkey is set */ +#define MDB_ENV_TXKEY 0x10000000U + uint32_t me_flags; /**< @ref mdb_env */ + unsigned int me_psize; /**< size of a page, from #GET_PAGESIZE */ + unsigned int me_maxreaders; /**< size of the reader table */ + unsigned int me_numreaders; /**< max numreaders set by this env */ + MDB_dbi me_numdbs; /**< number of DBs opened */ + MDB_dbi me_maxdbs; /**< size of the DB table */ + pid_t me_pid; /**< process ID of this env */ + char *me_path; /**< path to the DB files */ + char *me_map; /**< the memory map of the data file */ + MDB_txninfo *me_txns; /**< the memory map of the lock file or NULL */ + MDB_meta *me_metas[2]; /**< pointers to the two meta pages */ + MDB_txn *me_txn; /**< current write transaction */ + size_t me_mapsize; /**< size of the data memory map */ + off_t me_size; /**< current file size */ + pgno_t me_maxpg; /**< me_mapsize / me_psize */ + MDB_dbx *me_dbxs; /**< array of static DB info */ + uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ + pthread_key_t me_txkey; /**< thread-key for readers */ + MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ +# define me_pglast me_pgstate.mf_pglast +# define me_pghead me_pgstate.mf_pghead + MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ + /** IDL of pages that became unused in a write txn */ + MDB_IDL me_free_pgs; + /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */ + MDB_ID2L me_dirty_list; + /** Max number of freelist items that can fit in a single overflow page */ + int me_maxfree_1pg; + /** Max size of a node on a page */ + unsigned int me_nodemax; +#ifdef _WIN32 + HANDLE me_rmutex; /* Windows mutexes don't reside in shared mem */ + HANDLE me_wmutex; +#elif defined(MDB_USE_POSIX_SEM) + sem_t *me_rmutex; /* Shared mutexes are not supported */ + sem_t *me_wmutex; +#endif +}; + + /** Nested transaction */ +typedef struct MDB_ntxn { + MDB_txn mnt_txn; /* the transaction */ + MDB_pgstate mnt_pgstate; /* parent transaction's saved freestate */ +} MDB_ntxn; + + /** max number of pages to commit in one writev() call */ +#define MDB_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES +#undef MDB_COMMIT_PAGES +#define MDB_COMMIT_PAGES IOV_MAX +#endif + + /* max bytes to write in one call */ +#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4)) + +static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); +static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); +static int mdb_page_touch(MDB_cursor *mc); + +static int mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **mp, int *lvl); +static int mdb_page_search_root(MDB_cursor *mc, + MDB_val *key, int modify); +#define MDB_PS_MODIFY 1 +#define MDB_PS_ROOTONLY 2 +static int mdb_page_search(MDB_cursor *mc, + MDB_val *key, int flags); +static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); + +#define MDB_SPLIT_REPLACE MDB_APPENDDUP /**< newkey is not new */ +static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, + pgno_t newpgno, unsigned int nflags); + +static int mdb_env_read_header(MDB_env *env, MDB_meta *meta); +static int mdb_env_pick_meta(const MDB_env *env); +static int mdb_env_write_meta(MDB_txn *txn); +#if !(defined(_WIN32) || defined(MDB_USE_POSIX_SEM)) /* Drop unused excl arg */ +# define mdb_env_close0(env, excl) mdb_env_close1(env) +#endif +static void mdb_env_close0(MDB_env *env, int excl); + +static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp); +static int mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags); +static void mdb_node_del(MDB_page *mp, indx_t indx, int ksize); +static void mdb_node_shrink(MDB_page *mp, indx_t indx); +static int mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst); +static int mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data); +static size_t mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data); +static size_t mdb_branch_size(MDB_env *env, MDB_val *key); + +static int mdb_rebalance(MDB_cursor *mc); +static int mdb_update_key(MDB_cursor *mc, MDB_val *key); + +static void mdb_cursor_pop(MDB_cursor *mc); +static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp); + +static int mdb_cursor_del0(MDB_cursor *mc, MDB_node *leaf); +static int mdb_cursor_sibling(MDB_cursor *mc, int move_right); +static int mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op, + int *exactp); +static int mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data); +static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); + +static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); +static void mdb_xcursor_init0(MDB_cursor *mc); +static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); + +static int mdb_drop0(MDB_cursor *mc, int subs); +static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); + +/** @cond */ +static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; +/** @endcond */ + +#ifdef _WIN32 +static SECURITY_DESCRIPTOR mdb_null_sd; +static SECURITY_ATTRIBUTES mdb_all_sa; +static int mdb_sec_inited; +#endif + +/** Return the library version info. */ +char * +mdb_version(int *major, int *minor, int *patch) +{ + if (major) *major = MDB_VERSION_MAJOR; + if (minor) *minor = MDB_VERSION_MINOR; + if (patch) *patch = MDB_VERSION_PATCH; + return MDB_VERSION_STRING; +} + +/** Table of descriptions for MDB @ref errors */ +static char *const mdb_errstr[] = { + "MDB_KEYEXIST: Key/data pair already exists", + "MDB_NOTFOUND: No matching key/data pair found", + "MDB_PAGE_NOTFOUND: Requested page not found", + "MDB_CORRUPTED: Located page was wrong type", + "MDB_PANIC: Update of meta page failed", + "MDB_VERSION_MISMATCH: Database environment version mismatch", + "MDB_INVALID: File is not an MDB file", + "MDB_MAP_FULL: Environment mapsize limit reached", + "MDB_DBS_FULL: Environment maxdbs limit reached", + "MDB_READERS_FULL: Environment maxreaders limit reached", + "MDB_TLS_FULL: Thread-local storage keys full - too many environments open", + "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", + "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", + "MDB_PAGE_FULL: Internal error - page has no more space", + "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", + "MDB_INCOMPATIBLE: Database flags changed or would change", + "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", +}; + +char * +mdb_strerror(int err) +{ + int i; + if (!err) + return ("Successful return: 0"); + + if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { + i = err - MDB_KEYEXIST; + return mdb_errstr[i]; + } + + return strerror(err); +} + +#if MDB_DEBUG +/** Display a key in hexadecimal and return the address of the result. + * @param[in] key the key to display + * @param[in] buf the buffer to write into. Should always be #DKBUF. + * @return The key in hexadecimal form. + */ +char * +mdb_dkey(MDB_val *key, char *buf) +{ + char *ptr = buf; + unsigned char *c = key->mv_data; + unsigned int i; + + if (!key) + return ""; + + if (key->mv_size > MDB_MAXKEYSIZE) + return "MDB_MAXKEYSIZE"; + /* may want to make this a dynamic check: if the key is mostly + * printable characters, print it as-is instead of converting to hex. + */ +#if 1 + buf[0] = '\0'; + for (i=0; imv_size; i++) + ptr += sprintf(ptr, "%02x", *c++); +#else + sprintf(buf, "%.*s", key->mv_size, key->mv_data); +#endif + return buf; +} + +/** Display all the keys in the page. */ +void +mdb_page_list(MDB_page *mp) +{ + MDB_node *node; + unsigned int i, nkeys, nsize; + MDB_val key; + DKBUF; + + nkeys = NUMKEYS(mp); + fprintf(stderr, "Page %zu numkeys %d\n", mp->mp_pgno, nkeys); + for (i=0; imn_ksize; + key.mv_data = node->mn_data; + nsize = NODESIZE + NODEKSZ(node) + sizeof(indx_t); + if (IS_BRANCH(mp)) { + fprintf(stderr, "key %d: page %zu, %s\n", i, NODEPGNO(node), + DKEY(&key)); + } else { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + nsize += sizeof(pgno_t); + else + nsize += NODEDSZ(node); + fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key)); + } + } +} + +void +mdb_cursor_chk(MDB_cursor *mc) +{ + unsigned int i; + MDB_node *node; + MDB_page *mp; + + if (!mc->mc_snum && !(mc->mc_flags & C_INITIALIZED)) return; + for (i=0; imc_top; i++) { + mp = mc->mc_pg[i]; + node = NODEPTR(mp, mc->mc_ki[i]); + if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno) + printf("oops!\n"); + } + if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) + printf("ack!\n"); +} +#endif + +#if MDB_DEBUG > 2 +/** Count all the pages in each DB and in the freelist + * and make sure it matches the actual number of pages + * being used. + */ +static void mdb_audit(MDB_txn *txn) +{ + MDB_cursor mc; + MDB_val key, data; + MDB_ID freecount, count; + MDB_dbi i; + int rc; + + freecount = 0; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + + count = 0; + for (i = 0; imt_numdbs; i++) { + MDB_xcursor mx; + mdb_cursor_init(&mc, txn, i, &mx); + if (txn->mt_dbs[i].md_root == P_INVALID) + continue; + count += txn->mt_dbs[i].md_branch_pages + + txn->mt_dbs[i].md_leaf_pages + + txn->mt_dbs[i].md_overflow_pages; + if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) { + mdb_page_search(&mc, NULL, 0); + do { + unsigned j; + MDB_page *mp; + mp = mc.mc_pg[mc.mc_top]; + for (j=0; jmn_flags & F_SUBDATA) { + MDB_db db; + memcpy(&db, NODEDATA(leaf), sizeof(db)); + count += db.md_branch_pages + db.md_leaf_pages + + db.md_overflow_pages; + } + } + } + while (mdb_cursor_sibling(&mc, 1) == 0); + } + } + if (freecount + count + 2 /* metapages */ != txn->mt_next_pgno) { + fprintf(stderr, "audit: %lu freecount: %lu count: %lu total: %lu next_pgno: %lu\n", + txn->mt_txnid, freecount, count+2, freecount+count+2, txn->mt_next_pgno); + } +} +#endif + +int +mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_cmp(a, b); +} + +int +mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_dcmp(a, b); +} + +/** Allocate a page. + * Re-use old malloc'd pages first for singletons, otherwise just malloc. + */ +static MDB_page * +mdb_page_malloc(MDB_txn *txn, unsigned num) +{ + MDB_env *env = txn->mt_env; + MDB_page *ret = env->me_dpages; + size_t sz = env->me_psize; + if (num == 1) { + if (ret) { + VGMEMP_ALLOC(env, ret, sz); + VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); + env->me_dpages = ret->mp_next; + return ret; + } + } else { + sz *= num; + } + if ((ret = malloc(sz)) != NULL) { + VGMEMP_ALLOC(env, ret, sz); + } + return ret; +} + +/** Free a single page. + * Saves single pages to a list, for future reuse. + * (This is not used for multi-page overflow pages.) + */ +static void +mdb_page_free(MDB_env *env, MDB_page *mp) +{ + mp->mp_next = env->me_dpages; + VGMEMP_FREE(env, mp); + env->me_dpages = mp; +} + +/* Free a dirty page */ +static void +mdb_dpage_free(MDB_env *env, MDB_page *dp) +{ + if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { + mdb_page_free(env, dp); + } else { + /* large pages just get freed directly */ + VGMEMP_FREE(env, dp); + free(dp); + } +} + +/* Return all dirty pages to dpage list */ +static void +mdb_dlist_free(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned i, n = dl[0].mid; + + for (i = 1; i <= n; i++) { + mdb_dpage_free(env, dl[i].mptr); + } + dl[0].mid = 0; +} + +/** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */ +static txnid_t +mdb_find_oldest(MDB_txn *txn) +{ + int i; + txnid_t mr, oldest = txn->mt_txnid - 1; + MDB_reader *r = txn->mt_env->me_txns->mti_readers; + for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) { + if (r[i].mr_pid) { + mr = r[i].mr_txnid; + if (oldest > mr) + oldest = mr; + } + } + return oldest; +} + +/** Allocate pages for writing. + * If there are free pages available from older transactions, they + * will be re-used first. Otherwise a new page will be allocated. + * @param[in] mc cursor A cursor handle identifying the transaction and + * database for which we are allocating. + * @param[in] num the number of pages to allocate. + * @param[out] mp Address of the allocated page(s). Requests for multiple pages + * will always be satisfied by a single contiguous chunk of memory. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) +{ +#ifdef MDB_PARANOID /* Seems like we can ignore this now */ + /* Get at most more freeDB records once me_pghead + * has enough pages. If not enough, use new pages from the map. + * If and mc is updating the freeDB, only get new + * records if me_pghead is empty. Then the freelist cannot play + * catch-up with itself by growing while trying to save it. + */ + enum { Paranoid = 1, Max_retries = 500 }; +#else + enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; +#endif + int rc, n2 = num-1, retry = Max_retries; + MDB_txn *txn = mc->mc_txn; + MDB_env *env = txn->mt_env; + pgno_t pgno, *mop = env->me_pghead; + unsigned i, j, k, mop_len = mop ? mop[0] : 0; + MDB_page *np; + MDB_ID2 mid; + txnid_t oldest = 0, last; + MDB_cursor_op op; + MDB_cursor m2; + int (*insert)(MDB_ID2L, MDB_ID2 *); + + *mp = NULL; + + /* If our dirty list is already full, we can't do anything */ + if (txn->mt_dirty_room == 0) + return MDB_TXN_FULL; + + for (op = MDB_FIRST;; op = MDB_NEXT) { + MDB_val key, data; + MDB_node *leaf; + pgno_t *idl, old_id, new_id; + + /* Seek a big enough contiguous page range. Prefer + * pages at the tail, just truncating the list. + */ + if (mop_len >= (unsigned)num) { + i = mop_len; + do { + pgno = mop[i]; + if (mop[i-n2] == pgno+n2) + goto search_done; + } while (--i >= (unsigned)num); + if (Max_retries < INT_MAX && --retry < 0) + break; + } + + if (op == MDB_FIRST) { /* 1st iteration */ + /* Prepare to fetch more and coalesce */ + oldest = mdb_find_oldest(txn); + last = env->me_pglast; + mdb_cursor_init(&m2, txn, FREE_DBI, NULL); + if (last) { + op = MDB_SET_RANGE; + key.mv_data = &last; /* will loop up last+1 */ + key.mv_size = sizeof(last); + } + if (Paranoid && mc->mc_dbi == FREE_DBI) + retry = -1; + } + if (Paranoid && retry < 0 && mop_len) + break; + + last++; + /* Do not fetch more if the record will be too recent */ + if (oldest <= last) + break; + rc = mdb_cursor_get(&m2, &key, NULL, op); + if (rc) { + if (rc == MDB_NOTFOUND) + break; + return rc; + } + last = *(txnid_t*)key.mv_data; + if (oldest <= last) + break; + np = m2.mc_pg[m2.mc_top]; + leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); + if ((rc = mdb_node_read(txn, leaf, &data)) != MDB_SUCCESS) + return rc; + + idl = (MDB_ID *) data.mv_data; + i = idl[0]; + if (!mop) { + if (!(env->me_pghead = mop = mdb_midl_alloc(i))) + return ENOMEM; + } else { + if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) + return rc; + mop = env->me_pghead; + } + env->me_pglast = last; +#if MDB_DEBUG > 1 + DPRINTF("IDL read txn %zu root %zu num %u", + last, txn->mt_dbs[FREE_DBI].md_root, i); + for (k = i; k; k--) + DPRINTF("IDL %zu", idl[k]); +#endif + /* Merge in descending sorted order */ + j = mop_len; + k = mop_len += i; + mop[0] = (pgno_t)-1; + old_id = mop[j]; + while (i) { + new_id = idl[i--]; + for (; old_id < new_id; old_id = mop[--j]) + mop[k--] = old_id; + mop[k--] = new_id; + } + mop[0] = mop_len; + } + + /* Use new pages from the map when nothing suitable in the freeDB */ + i = 0; + pgno = txn->mt_next_pgno; + if (pgno + num >= env->me_maxpg) { + DPUTS("DB size maxed out"); + return MDB_MAP_FULL; + } + +search_done: + if (env->me_flags & MDB_WRITEMAP) { + np = (MDB_page *)(env->me_map + env->me_psize * pgno); + insert = mdb_mid2l_append; + } else { + if (!(np = mdb_page_malloc(txn, num))) + return ENOMEM; + insert = mdb_mid2l_insert; + } + if (i) { + mop[0] = mop_len -= num; + /* Move any stragglers down */ + for (j = i-num; j < mop_len; ) + mop[++j] = mop[++i]; + } else { + txn->mt_next_pgno = pgno + num; + } + mid.mid = np->mp_pgno = pgno; + mid.mptr = np; + insert(txn->mt_u.dirty_list, &mid); + txn->mt_dirty_room--; + *mp = np; + + return MDB_SUCCESS; +} + +/** Copy the used portions of a non-overflow page. + * @param[in] dst page to copy into + * @param[in] src page to copy from + * @param[in] psize size of a page + */ +static void +mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize) +{ + enum { Align = sizeof(pgno_t) }; + indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower; + + /* If page isn't full, just copy the used portion. Adjust + * alignment so memcpy may copy words instead of bytes. + */ + if ((unused &= -Align) && !IS_LEAF2(src)) { + upper &= -Align; + memcpy(dst, src, (lower + (Align-1)) & -Align); + memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper), + psize - upper); + } else { + memcpy(dst, src, psize - unused); + } +} + +/** Touch a page: make it dirty and re-insert into tree with updated pgno. + * @param[in] mc cursor pointing to the page to be touched + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_touch(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top], *np; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m2, *m3; + MDB_dbi dbi; + pgno_t pgno; + int rc; + + if (!F_ISSET(mp->mp_flags, P_DIRTY)) { + if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || + (rc = mdb_page_alloc(mc, 1, &np))) + return rc; + pgno = np->mp_pgno; + DPRINTF("touched db %u page %zu -> %zu", mc->mc_dbi,mp->mp_pgno,pgno); + assert(mp->mp_pgno != pgno); + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* Update the parent page, if any, to point to the new page */ + if (mc->mc_top) { + MDB_page *parent = mc->mc_pg[mc->mc_top-1]; + MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]); + SETPGNO(node, pgno); + } else { + mc->mc_db->md_root = pgno; + } + } else if (txn->mt_parent && !IS_SUBP(mp)) { + MDB_ID2 mid, *dl = txn->mt_u.dirty_list; + pgno = mp->mp_pgno; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + return MDB_CORRUPTED; + } + return 0; + } + } + assert(dl[0].mid < MDB_IDL_UM_MAX); + /* No - copy it */ + np = mdb_page_malloc(txn, 1); + if (!np) + return ENOMEM; + mid.mid = pgno; + mid.mptr = np; + mdb_mid2l_insert(dl, &mid); + } else { + return 0; + } + + mdb_page_copy(np, mp, txn->mt_env->me_psize); + np->mp_pgno = pgno; + np->mp_flags |= P_DIRTY; + + /* Adjust cursors pointing to mp */ + mc->mc_pg[mc->mc_top] = np; + dbi = mc->mc_dbi; + if (mc->mc_flags & C_SUB) { + dbi--; + for (m2 = txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + m3 = &m2->mc_xcursor->mx_cursor; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[mc->mc_top] == mp) + m3->mc_pg[mc->mc_top] = np; + } + } else { + for (m2 = txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (m2->mc_snum < mc->mc_snum) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + m2->mc_pg[mc->mc_top] = np; + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) + { + MDB_node *leaf = NODEPTR(np, mc->mc_ki[mc->mc_top]); + if (!(leaf->mn_flags & F_SUBDATA)) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + } + } + } + return 0; +} + +int +mdb_env_sync(MDB_env *env, int force) +{ + int rc = 0; + if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { + if (env->me_flags & MDB_WRITEMAP) { + int flags = ((env->me_flags & MDB_MAPASYNC) && !force) + ? MS_ASYNC : MS_SYNC; + if (MDB_MSYNC(env->me_map, env->me_mapsize, flags)) + rc = ErrCode(); +#ifdef _WIN32 + else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); +#endif + } else { + if (MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); + } + } + return rc; +} + +/** Back up parent txn's cursors, then grab the originals for tracking */ +static int +mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) +{ + MDB_cursor *mc, *bk; + MDB_xcursor *mx; + size_t size; + int i; + + for (i = src->mt_numdbs; --i >= 0; ) { + if ((mc = src->mt_cursors[i]) != NULL) { + size = sizeof(MDB_cursor); + if (mc->mc_xcursor) + size += sizeof(MDB_xcursor); + for (; mc; mc = bk->mc_next) { + bk = malloc(size); + if (!bk) + return ENOMEM; + *bk = *mc; + mc->mc_backup = bk; + mc->mc_db = &dst->mt_dbs[i]; + /* Kill pointers into src - and dst to reduce abuse: The + * user may not use mc until dst ends. Otherwise we'd... + */ + mc->mc_txn = NULL; /* ...set this to dst */ + mc->mc_dbflag = NULL; /* ...and &dst->mt_dbflags[i] */ + if ((mx = mc->mc_xcursor) != NULL) { + *(MDB_xcursor *)(bk+1) = *mx; + mx->mx_cursor.mc_txn = NULL; /* ...and dst. */ + } + mc->mc_next = dst->mt_cursors[i]; + dst->mt_cursors[i] = mc; + } + } + } + return MDB_SUCCESS; +} + +/** Close this write txn's cursors, give parent txn's cursors back to parent. + * @param[in] txn the transaction handle. + * @param[in] merge true to keep changes to parent cursors, false to revert. + * @return 0 on success, non-zero on failure. + */ +static void +mdb_cursors_close(MDB_txn *txn, unsigned merge) +{ + MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; + MDB_xcursor *mx; + int i; + + for (i = txn->mt_numdbs; --i >= 0; ) { + for (mc = cursors[i]; mc; mc = next) { + next = mc->mc_next; + if ((bk = mc->mc_backup) != NULL) { + if (merge) { + /* Commit changes to parent txn */ + mc->mc_next = bk->mc_next; + mc->mc_backup = bk->mc_backup; + mc->mc_txn = bk->mc_txn; + mc->mc_db = bk->mc_db; + mc->mc_dbflag = bk->mc_dbflag; + if ((mx = mc->mc_xcursor) != NULL) + mx->mx_cursor.mc_txn = bk->mc_txn; + } else { + /* Abort nested txn */ + *mc = *bk; + if ((mx = mc->mc_xcursor) != NULL) + *mx = *(MDB_xcursor *)(bk+1); + } + mc = bk; + } + free(mc); + } + cursors[i] = NULL; + } +} + +#ifdef MDB_DEBUG_SKIP +#define mdb_txn_reset0(txn, act) mdb_txn_reset0(txn) +#endif +static void +mdb_txn_reset0(MDB_txn *txn, const char *act); + +/** Common code for #mdb_txn_begin() and #mdb_txn_renew(). + * @param[in] txn the transaction handle to initialize + * @return 0 on success, non-zero on failure. + */ +static int +mdb_txn_renew0(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + unsigned int i; + uint16_t x; + int rc, new_notls = 0; + + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; + txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ + + if (txn->mt_flags & MDB_TXN_RDONLY) { + if (!env->me_txns) { + i = mdb_env_pick_meta(env); + txn->mt_txnid = env->me_metas[i]->mm_txnid; + txn->mt_u.reader = NULL; + } else { + MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader : + pthread_getspecific(env->me_txkey); + if (r) { + if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1) + return MDB_BAD_RSLOT; + } else { + pid_t pid = env->me_pid; + pthread_t tid = pthread_self(); + + LOCK_MUTEX_R(env); + for (i=0; ime_txns->mti_numreaders; i++) + if (env->me_txns->mti_readers[i].mr_pid == 0) + break; + if (i == env->me_maxreaders) { + UNLOCK_MUTEX_R(env); + return MDB_READERS_FULL; + } + env->me_txns->mti_readers[i].mr_pid = pid; + env->me_txns->mti_readers[i].mr_tid = tid; + if (i >= env->me_txns->mti_numreaders) + env->me_txns->mti_numreaders = i+1; + /* Save numreaders for un-mutexed mdb_env_close() */ + env->me_numreaders = env->me_txns->mti_numreaders; + UNLOCK_MUTEX_R(env); + r = &env->me_txns->mti_readers[i]; + new_notls = (env->me_flags & MDB_NOTLS); + if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { + r->mr_pid = 0; + return rc; + } + } + txn->mt_txnid = r->mr_txnid = env->me_txns->mti_txnid; + txn->mt_u.reader = r; + } + txn->mt_toggle = txn->mt_txnid & 1; + } else { + LOCK_MUTEX_W(env); + + txn->mt_txnid = env->me_txns->mti_txnid; + txn->mt_toggle = txn->mt_txnid & 1; + txn->mt_txnid++; +#if MDB_DEBUG + if (txn->mt_txnid == mdb_debug_start) + mdb_debug = 1; +#endif + txn->mt_dirty_room = MDB_IDL_UM_MAX; + txn->mt_u.dirty_list = env->me_dirty_list; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_free_pgs = env->me_free_pgs; + txn->mt_free_pgs[0] = 0; + env->me_txn = txn; + } + + /* Copy the DB info and flags */ + memcpy(txn->mt_dbs, env->me_metas[txn->mt_toggle]->mm_dbs, 2 * sizeof(MDB_db)); + + /* Moved to here to avoid a data race in read TXNs */ + txn->mt_next_pgno = env->me_metas[txn->mt_toggle]->mm_last_pg+1; + + for (i=2; imt_numdbs; i++) { + x = env->me_dbflags[i]; + txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; + txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_STALE : 0; + } + txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; + + if (env->me_maxpg < txn->mt_next_pgno) { + mdb_txn_reset0(txn, "renew0-mapfail"); + if (new_notls) { + txn->mt_u.reader->mr_pid = 0; + txn->mt_u.reader = NULL; + } + return MDB_MAP_RESIZED; + } + + return MDB_SUCCESS; +} + +int +mdb_txn_renew(MDB_txn *txn) +{ + int rc; + + if (!txn || txn->mt_dbxs) /* A reset txn has mt_dbxs==NULL */ + return EINVAL; + + if (txn->mt_env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + return MDB_PANIC; + } + + rc = mdb_txn_renew0(txn); + if (rc == MDB_SUCCESS) { + DPRINTF("renew txn %zu%c %p on mdbenv %p, root page %zu", + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root); + } + return rc; +} + +int +mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) +{ + MDB_txn *txn; + MDB_ntxn *ntxn; + int rc, size, tsize = sizeof(MDB_txn); + + if (env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + return MDB_PANIC; + } + if ((env->me_flags & MDB_RDONLY) && !(flags & MDB_RDONLY)) + return EACCES; + if (parent) { + /* Nested transactions: Max 1 child, write txns only, no writemap */ + if (parent->mt_child || + (flags & MDB_RDONLY) || (parent->mt_flags & MDB_TXN_RDONLY) || + (env->me_flags & MDB_WRITEMAP)) + { + return EINVAL; + } + tsize = sizeof(MDB_ntxn); + } + size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1); + if (!(flags & MDB_RDONLY)) + size += env->me_maxdbs * sizeof(MDB_cursor *); + + if ((txn = calloc(1, size)) == NULL) { + DPRINTF("calloc: %s", strerror(ErrCode())); + return ENOMEM; + } + txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); + if (flags & MDB_RDONLY) { + txn->mt_flags |= MDB_TXN_RDONLY; + txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); + } else { + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); + } + txn->mt_env = env; + + if (parent) { + unsigned int i; + txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); + if (!txn->mt_u.dirty_list || + !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) + { + free(txn->mt_u.dirty_list); + free(txn); + return ENOMEM; + } + txn->mt_txnid = parent->mt_txnid; + txn->mt_toggle = parent->mt_toggle; + txn->mt_dirty_room = parent->mt_dirty_room; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_next_pgno = parent->mt_next_pgno; + parent->mt_child = txn; + txn->mt_parent = parent; + txn->mt_numdbs = parent->mt_numdbs; + txn->mt_flags = parent->mt_flags; + txn->mt_dbxs = parent->mt_dbxs; + memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + /* Copy parent's mt_dbflags, but clear DB_NEW */ + for (i=0; imt_numdbs; i++) + txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW; + rc = 0; + ntxn = (MDB_ntxn *)txn; + ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */ + if (env->me_pghead) { + size = MDB_IDL_SIZEOF(env->me_pghead); + env->me_pghead = mdb_midl_alloc(env->me_pghead[0]); + if (env->me_pghead) + memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size); + else + rc = ENOMEM; + } + if (!rc) + rc = mdb_cursor_shadow(parent, txn); + if (rc) + mdb_txn_reset0(txn, "beginchild-fail"); + } else { + rc = mdb_txn_renew0(txn); + } + if (rc) + free(txn); + else { + *ret = txn; + DPRINTF("begin txn %zu%c %p on mdbenv %p, root page %zu", + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root); + } + + return rc; +} + +/** Export or close DBI handles opened in this txn. */ +static void +mdb_dbis_update(MDB_txn *txn, int keep) +{ + int i; + MDB_dbi n = txn->mt_numdbs; + MDB_env *env = txn->mt_env; + unsigned char *tdbflags = txn->mt_dbflags; + + for (i = n; --i >= 2;) { + if (tdbflags[i] & DB_NEW) { + if (keep) { + env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; + } else { + char *ptr = env->me_dbxs[i].md_name.mv_data; + env->me_dbxs[i].md_name.mv_data = NULL; + env->me_dbxs[i].md_name.mv_size = 0; + env->me_dbflags[i] = 0; + free(ptr); + } + } + } + if (keep && env->me_numdbs < n) + env->me_numdbs = n; +} + +/** Common code for #mdb_txn_reset() and #mdb_txn_abort(). + * May be called twice for readonly txns: First reset it, then abort. + * @param[in] txn the transaction handle to reset + */ +static void +mdb_txn_reset0(MDB_txn *txn, const char *act) +{ + MDB_env *env = txn->mt_env; + + /* Close any DBI handles opened in this txn */ + mdb_dbis_update(txn, 0); + + DPRINTF("%s txn %zu%c %p on mdbenv %p, root page %zu", + act, txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root); + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + if (txn->mt_u.reader) { + txn->mt_u.reader->mr_txnid = (txnid_t)-1; + if (!(env->me_flags & MDB_NOTLS)) + txn->mt_u.reader = NULL; /* txn does not own reader */ + } + txn->mt_numdbs = 0; /* close nothing if called again */ + txn->mt_dbxs = NULL; /* mark txn as reset */ + } else { + mdb_cursors_close(txn, 0); + + if (!(env->me_flags & MDB_WRITEMAP)) { + mdb_dlist_free(txn); + } + mdb_midl_free(env->me_pghead); + + if (txn->mt_parent) { + txn->mt_parent->mt_child = NULL; + env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; + mdb_midl_free(txn->mt_free_pgs); + free(txn->mt_u.dirty_list); + return; + } + + if (mdb_midl_shrink(&txn->mt_free_pgs)) + env->me_free_pgs = txn->mt_free_pgs; + env->me_pghead = NULL; + env->me_pglast = 0; + + env->me_txn = NULL; + /* The writer mutex was locked in mdb_txn_begin. */ + UNLOCK_MUTEX_W(env); + } +} + +void +mdb_txn_reset(MDB_txn *txn) +{ + if (txn == NULL) + return; + + /* This call is only valid for read-only txns */ + if (!(txn->mt_flags & MDB_TXN_RDONLY)) + return; + + mdb_txn_reset0(txn, "reset"); +} + +void +mdb_txn_abort(MDB_txn *txn) +{ + if (txn == NULL) + return; + + if (txn->mt_child) + mdb_txn_abort(txn->mt_child); + + mdb_txn_reset0(txn, "abort"); + /* Free reader slot tied to this txn (if MDB_NOTLS && writable FS) */ + if ((txn->mt_flags & MDB_TXN_RDONLY) && txn->mt_u.reader) + txn->mt_u.reader->mr_pid = 0; + + free(txn); +} + +/** Save the freelist as of this transaction to the freeDB. + * This changes the freelist. Keep trying until it stabilizes. + */ +static int +mdb_freelist_save(MDB_txn *txn) +{ + /* env->me_pghead[] can grow and shrink during this call. + * env->me_pglast and txn->mt_free_pgs[] can only grow. + * Page numbers cannot disappear from txn->mt_free_pgs[]. + */ + MDB_cursor mc; + MDB_env *env = txn->mt_env; + int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; + txnid_t pglast = 0, head_id = 0; + pgno_t freecnt = 0, *free_pgs, *mop; + ssize_t head_room = 0, total_room = 0, mop_len; + + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + + if (env->me_pghead) { + /* Make sure first page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + for (;;) { + /* Come back here after each Put() in case freelist changed */ + MDB_val key, data; + + /* If using records from freeDB which we have not yet + * deleted, delete them and any we reserved for me_pghead. + */ + while (pglast < env->me_pglast) { + rc = mdb_cursor_first(&mc, &key, NULL); + if (rc) + return rc; + pglast = head_id = *(txnid_t *)key.mv_data; + total_room = head_room = 0; + assert(pglast <= env->me_pglast); + rc = mdb_cursor_del(&mc, 0); + if (rc) + return rc; + } + + /* Save the IDL of pages freed by this txn, to a single record */ + if (freecnt < txn->mt_free_pgs[0]) { + if (!freecnt) { + /* Make sure last page of freeDB is touched and on freelist */ + key.mv_size = MDB_MAXKEYSIZE+1; + key.mv_data = NULL; + rc = mdb_page_search(&mc, &key, MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + free_pgs = txn->mt_free_pgs; + /* Write to last page of freeDB */ + key.mv_size = sizeof(txn->mt_txnid); + key.mv_data = &txn->mt_txnid; + do { + freecnt = free_pgs[0]; + data.mv_size = MDB_IDL_SIZEOF(free_pgs); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* Retry if mt_free_pgs[] grew during the Put() */ + free_pgs = txn->mt_free_pgs; + } while (freecnt < free_pgs[0]); + mdb_midl_sort(free_pgs); + memcpy(data.mv_data, free_pgs, data.mv_size); +#if MDB_DEBUG > 1 + { + unsigned int i = free_pgs[0]; + DPRINTF("IDL write txn %zu root %zu num %u", + txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i); + for (; i; i--) + DPRINTF("IDL %zu", free_pgs[i]); + } +#endif + continue; + } + + mop = env->me_pghead; + mop_len = mop ? mop[0] : 0; + + /* Reserve records for me_pghead[]. Split it if multi-page, + * to avoid searching freeDB for a page range. Use keys in + * range [1,me_pglast]: Smaller than txnid of oldest reader. + */ + if (total_room >= mop_len) { + if (total_room == mop_len || --more < 0) + break; + } else if (head_room >= maxfree_1pg && head_id > 1) { + /* Keep current record (overflow page), add a new one */ + head_id--; + head_room = 0; + } + /* (Re)write {key = head_id, IDL length = head_room} */ + total_room -= head_room; + head_room = mop_len - total_room; + if (head_room > maxfree_1pg && head_id > 1) { + /* Overflow multi-page for part of me_pghead */ + head_room /= head_id; /* amortize page sizes */ + head_room += maxfree_1pg - head_room % (maxfree_1pg + 1); + } else if (head_room < 0) { + /* Rare case, not bothering to delete this record */ + head_room = 0; + } + key.mv_size = sizeof(head_id); + key.mv_data = &head_id; + data.mv_size = (head_room + 1) * sizeof(pgno_t); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + *(MDB_ID *)data.mv_data = 0; /* IDL is initially empty */ + total_room += head_room; + } + + /* Fill in the reserved, touched me_pghead records. Avoid write ops + * so they cannot rearrange anything, just read the destinations. + */ + rc = MDB_SUCCESS; + if (mop_len) { + MDB_val key, data; + + mop += mop_len + 1; + rc = mdb_cursor_first(&mc, &key, &data); + for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { + MDB_IDL dest = data.mv_data; + ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; + + assert(len >= 0 && *(txnid_t*)key.mv_data <= env->me_pglast); + if (len > mop_len) + len = mop_len; + *dest++ = len; + memcpy(dest, mop -= len, len * sizeof(MDB_ID)); + if (! (mop_len -= len)) + break; + } + } + return rc; +} + +/** Flush dirty pages to the map, after clearing their dirty flag. + */ +static int +mdb_page_flush(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned psize = env->me_psize; + int i, pagecount = dl[0].mid, rc; + size_t size = 0, pos = 0; + pgno_t pgno = 0; + MDB_page *dp = NULL; +#ifdef _WIN32 + OVERLAPPED ov; +#else + struct iovec iov[MDB_COMMIT_PAGES]; + ssize_t wpos = 0, wsize = 0, wres; + size_t next_pos = 1; /* impossible pos, so pos != next_pos */ + int n = 0; +#endif + + if (env->me_flags & MDB_WRITEMAP) { + /* Clear dirty flags */ + for (i = pagecount; i; i--) { + dp = dl[i].mptr; + dp->mp_flags &= ~P_DIRTY; + } + dl[0].mid = 0; + return MDB_SUCCESS; + } + + /* Write the pages */ + for (i = 1;; i++) { + if (i <= pagecount) { + dp = dl[i].mptr; + pgno = dl[i].mid; + /* clear dirty flag */ + dp->mp_flags &= ~P_DIRTY; + pos = pgno * psize; + size = psize; + if (IS_OVERFLOW(dp)) size *= dp->mp_pages; + } +#ifdef _WIN32 + else break; + + /* Windows actually supports scatter/gather I/O, but only on + * unbuffered file handles. Since we're relying on the OS page + * cache for all our data, that's self-defeating. So we just + * write pages one at a time. We use the ov structure to set + * the write offset, to at least save the overhead of a Seek + * system call. + */ + DPRINTF("committing page %zu", pgno); + memset(&ov, 0, sizeof(ov)); + ov.Offset = pos & 0xffffffff; + ov.OffsetHigh = pos >> 16 >> 16; + if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) { + rc = ErrCode(); + DPRINTF("WriteFile: %d", rc); + return rc; + } +#else + /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ + if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { + if (n) { + /* Write previous page(s) */ +#ifdef MDB_USE_PWRITEV + wres = pwritev(env->me_fd, iov, n, wpos); +#else + if (n == 1) { + wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos); + } else { + if (lseek(env->me_fd, wpos, SEEK_SET) == -1) { + rc = ErrCode(); + DPRINTF("lseek: %s", strerror(rc)); + return rc; + } + wres = writev(env->me_fd, iov, n); + } +#endif + if (wres != wsize) { + if (wres < 0) { + rc = ErrCode(); + DPRINTF("Write error: %s", strerror(rc)); + } else { + rc = EIO; /* TODO: Use which error code? */ + DPUTS("short write, filesystem full?"); + } + return rc; + } + n = 0; + } + if (i > pagecount) + break; + wpos = pos; + wsize = 0; + } + DPRINTF("committing page %zu", pgno); + next_pos = pos + size; + iov[n].iov_len = size; + iov[n].iov_base = (char *)dp; + wsize += size; + n++; +#endif /* _WIN32 */ + } + + mdb_dlist_free(txn); + + return MDB_SUCCESS; +} + +int +mdb_txn_commit(MDB_txn *txn) +{ + int rc; + unsigned int i; + MDB_env *env; + + assert(txn != NULL); + assert(txn->mt_env != NULL); + + if (txn->mt_child) { + rc = mdb_txn_commit(txn->mt_child); + txn->mt_child = NULL; + if (rc) + goto fail; + } + + env = txn->mt_env; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + mdb_dbis_update(txn, 1); + txn->mt_numdbs = 2; /* so txn_abort() doesn't close any new handles */ + mdb_txn_abort(txn); + return MDB_SUCCESS; + } + + if (F_ISSET(txn->mt_flags, MDB_TXN_ERROR)) { + DPUTS("error flag is set, can't commit"); + if (txn->mt_parent) + txn->mt_parent->mt_flags |= MDB_TXN_ERROR; + rc = EINVAL; + goto fail; + } + + if (txn->mt_parent) { + MDB_txn *parent = txn->mt_parent; + unsigned x, y, len; + MDB_ID2L dst, src; + + /* Append our free list to parent's */ + rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs); + if (rc) + goto fail; + mdb_midl_free(txn->mt_free_pgs); + + parent->mt_next_pgno = txn->mt_next_pgno; + parent->mt_flags = txn->mt_flags; + + /* Merge our cursors into parent's and close them */ + mdb_cursors_close(txn, 1); + + /* Update parent's DB table. */ + memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + txn->mt_parent->mt_numdbs = txn->mt_numdbs; + txn->mt_parent->mt_dbflags[0] = txn->mt_dbflags[0]; + txn->mt_parent->mt_dbflags[1] = txn->mt_dbflags[1]; + for (i=2; imt_numdbs; i++) { + /* preserve parent's DB_NEW status */ + x = txn->mt_parent->mt_dbflags[i] & DB_NEW; + txn->mt_parent->mt_dbflags[i] = txn->mt_dbflags[i] | x; + } + + dst = txn->mt_parent->mt_u.dirty_list; + src = txn->mt_u.dirty_list; + /* Find len = length of merging our dirty list with parent's */ + x = dst[0].mid; + dst[0].mid = 0; /* simplify loops */ + if (parent->mt_parent) { + len = x + src[0].mid; + y = mdb_mid2l_search(src, dst[x].mid + 1) - 1; + for (i = x; y && i; y--) { + pgno_t yp = src[y].mid; + while (yp < dst[i].mid) + i--; + if (yp == dst[i].mid) { + i--; + len--; + } + } + } else { /* Simplify the above for single-ancestor case */ + len = MDB_IDL_UM_MAX - txn->mt_dirty_room; + } + /* Merge our dirty list with parent's */ + y = src[0].mid; + for (i = len; y; dst[i--] = src[y--]) { + pgno_t yp = src[y].mid; + while (yp < dst[x].mid) + dst[i--] = dst[x--]; + if (yp == dst[x].mid) + free(dst[x--].mptr); + } + assert(i == x); + dst[0].mid = len; + free(txn->mt_u.dirty_list); + parent->mt_dirty_room = txn->mt_dirty_room; + + txn->mt_parent->mt_child = NULL; + mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); + free(txn); + return MDB_SUCCESS; + } + + if (txn != env->me_txn) { + DPUTS("attempt to commit unknown transaction"); + rc = EINVAL; + goto fail; + } + + mdb_cursors_close(txn, 0); + + if (!txn->mt_u.dirty_list[0].mid && !(txn->mt_flags & MDB_TXN_DIRTY)) + goto done; + + DPRINTF("committing txn %zu %p on mdbenv %p, root page %zu", + txn->mt_txnid, (void *)txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root); + + /* Update DB root pointers */ + if (txn->mt_numdbs > 2) { + MDB_cursor mc; + MDB_dbi i; + MDB_val data; + data.mv_size = sizeof(MDB_db); + + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + for (i = 2; i < txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + data.mv_data = &txn->mt_dbs[i]; + rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, 0); + if (rc) + goto fail; + } + } + } + + rc = mdb_freelist_save(txn); + if (rc) + goto fail; + + mdb_midl_free(env->me_pghead); + env->me_pghead = NULL; + if (mdb_midl_shrink(&txn->mt_free_pgs)) + env->me_free_pgs = txn->mt_free_pgs; + +#if MDB_DEBUG > 2 + mdb_audit(txn); +#endif + + if ((rc = mdb_page_flush(txn)) || + (rc = mdb_env_sync(env, 0)) || + (rc = mdb_env_write_meta(txn))) + goto fail; + +done: + env->me_pglast = 0; + env->me_txn = NULL; + mdb_dbis_update(txn, 1); + + UNLOCK_MUTEX_W(env); + free(txn); + + return MDB_SUCCESS; + +fail: + mdb_txn_abort(txn); + return rc; +} + +/** Read the environment parameters of a DB environment before + * mapping it into memory. + * @param[in] env the environment handle + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int +mdb_env_read_header(MDB_env *env, MDB_meta *meta) +{ + MDB_pagebuf pbuf; + MDB_page *p; + MDB_meta *m; + int i, rc, off; + + /* We don't know the page size yet, so use a minimum value. + * Read both meta pages so we can use the latest one. + */ + + for (i=off=0; i<2; i++, off = meta->mm_psize) { +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + rc = ReadFile(env->me_fd,&pbuf,MDB_PAGESIZE,&len,&ov) ? (int)len : -1; +#else + rc = pread(env->me_fd, &pbuf, MDB_PAGESIZE, off); +#endif + if (rc != MDB_PAGESIZE) { + if (rc == 0 && off == 0) + return ENOENT; + rc = rc < 0 ? (int) ErrCode() : MDB_INVALID; + DPRINTF("read: %s", mdb_strerror(rc)); + return rc; + } + + p = (MDB_page *)&pbuf; + + if (!F_ISSET(p->mp_flags, P_META)) { + DPRINTF("page %zu not a meta page", p->mp_pgno); + return MDB_INVALID; + } + + m = METADATA(p); + if (m->mm_magic != MDB_MAGIC) { + DPUTS("meta has invalid magic"); + return MDB_INVALID; + } + + if (m->mm_version != MDB_VERSION) { + DPRINTF("database is version %u, expected version %u", + m->mm_version, MDB_VERSION); + return MDB_VERSION_MISMATCH; + } + + if (off == 0 || m->mm_txnid > meta->mm_txnid) + *meta = *m; + } + return 0; +} + +/** Write the environment parameters of a freshly created DB environment. + * @param[in] env the environment handle + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int +mdb_env_init_meta(MDB_env *env, MDB_meta *meta) +{ + MDB_page *p, *q; + int rc; + unsigned int psize; + + DPUTS("writing new meta page"); + + GET_PAGESIZE(psize); + + meta->mm_magic = MDB_MAGIC; + meta->mm_version = MDB_VERSION; + meta->mm_mapsize = env->me_mapsize; + meta->mm_psize = psize; + meta->mm_last_pg = 1; + meta->mm_flags = env->me_flags & 0xffff; + meta->mm_flags |= MDB_INTEGERKEY; + meta->mm_dbs[0].md_root = P_INVALID; + meta->mm_dbs[1].md_root = P_INVALID; + + p = calloc(2, psize); + p->mp_pgno = 0; + p->mp_flags = P_META; + *(MDB_meta *)METADATA(p) = *meta; + + q = (MDB_page *)((char *)p + psize); + q->mp_pgno = 1; + q->mp_flags = P_META; + *(MDB_meta *)METADATA(q) = *meta; + +#ifdef _WIN32 + { + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + rc = WriteFile(env->me_fd, p, psize * 2, &len, &ov); + rc = rc ? (len == psize * 2 ? MDB_SUCCESS : EIO) : ErrCode(); + } +#else + rc = pwrite(env->me_fd, p, psize * 2, 0); + rc = (rc == (int)psize * 2) ? MDB_SUCCESS : rc < 0 ? ErrCode() : EIO; +#endif + free(p); + return rc; +} + +/** Update the environment info to commit a transaction. + * @param[in] txn the transaction that's being committed + * @return 0 on success, non-zero on failure. + */ +static int +mdb_env_write_meta(MDB_txn *txn) +{ + MDB_env *env; + MDB_meta meta, metab, *mp; + off_t off; + int rc, len, toggle; + char *ptr; + HANDLE mfd; +#ifdef _WIN32 + OVERLAPPED ov; +#else + int r2; +#endif + + assert(txn != NULL); + assert(txn->mt_env != NULL); + + toggle = !txn->mt_toggle; + DPRINTF("writing meta page %d for root page %zu", + toggle, txn->mt_dbs[MAIN_DBI].md_root); + + env = txn->mt_env; + mp = env->me_metas[toggle]; + + if (env->me_flags & MDB_WRITEMAP) { + /* Persist any increases of mapsize config */ + if (env->me_mapsize > mp->mm_mapsize) + mp->mm_mapsize = env->me_mapsize; + mp->mm_dbs[0] = txn->mt_dbs[0]; + mp->mm_dbs[1] = txn->mt_dbs[1]; + mp->mm_last_pg = txn->mt_next_pgno - 1; + mp->mm_txnid = txn->mt_txnid; + if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { + rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; + ptr = env->me_map; + if (toggle) + ptr += env->me_psize; + if (MDB_MSYNC(ptr, env->me_psize, rc)) { + rc = ErrCode(); + goto fail; + } + } + goto done; + } + metab.mm_txnid = env->me_metas[toggle]->mm_txnid; + metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg; + + ptr = (char *)&meta; + if (env->me_mapsize > mp->mm_mapsize) { + /* Persist any increases of mapsize config */ + meta.mm_mapsize = env->me_mapsize; + off = offsetof(MDB_meta, mm_mapsize); + } else { + off = offsetof(MDB_meta, mm_dbs[0].md_depth); + } + len = sizeof(MDB_meta) - off; + + ptr += off; + meta.mm_dbs[0] = txn->mt_dbs[0]; + meta.mm_dbs[1] = txn->mt_dbs[1]; + meta.mm_last_pg = txn->mt_next_pgno - 1; + meta.mm_txnid = txn->mt_txnid; + + if (toggle) + off += env->me_psize; + off += PAGEHDRSZ; + + /* Write to the SYNC fd */ + mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ? + env->me_fd : env->me_mfd; +#ifdef _WIN32 + { + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov)) + rc = -1; + } +#else + rc = pwrite(mfd, ptr, len, off); +#endif + if (rc != len) { + rc = rc < 0 ? ErrCode() : EIO; + DPUTS("write failed, disk error?"); + /* On a failure, the pagecache still contains the new data. + * Write some old data back, to prevent it from being used. + * Use the non-SYNC fd; we know it will fail anyway. + */ + meta.mm_last_pg = metab.mm_last_pg; + meta.mm_txnid = metab.mm_txnid; +#ifdef _WIN32 + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + WriteFile(env->me_fd, ptr, len, NULL, &ov); +#else + r2 = pwrite(env->me_fd, ptr, len, off); +#endif +fail: + env->me_flags |= MDB_FATAL_ERROR; + return rc; + } +done: + /* Memory ordering issues are irrelevant; since the entire writer + * is wrapped by wmutex, all of these changes will become visible + * after the wmutex is unlocked. Since the DB is multi-version, + * readers will get consistent data regardless of how fresh or + * how stale their view of these values is. + */ + env->me_txns->mti_txnid = txn->mt_txnid; + + return MDB_SUCCESS; +} + +/** Check both meta pages to see which one is newer. + * @param[in] env the environment handle + * @return meta toggle (0 or 1). + */ +static int +mdb_env_pick_meta(const MDB_env *env) +{ + return (env->me_metas[0]->mm_txnid < env->me_metas[1]->mm_txnid); +} + +int +mdb_env_create(MDB_env **env) +{ + MDB_env *e; + + e = calloc(1, sizeof(MDB_env)); + if (!e) + return ENOMEM; + + e->me_maxreaders = DEFAULT_READERS; + e->me_maxdbs = e->me_numdbs = 2; + e->me_fd = INVALID_HANDLE_VALUE; + e->me_lfd = INVALID_HANDLE_VALUE; + e->me_mfd = INVALID_HANDLE_VALUE; +#ifdef MDB_USE_POSIX_SEM + e->me_rmutex = SEM_FAILED; + e->me_wmutex = SEM_FAILED; +#endif + e->me_pid = getpid(); + VGMEMP_CREATE(e,0,0); + *env = e; + return MDB_SUCCESS; +} + +int +mdb_env_set_mapsize(MDB_env *env, size_t size) +{ + if (env->me_map) + return EINVAL; + env->me_mapsize = size; + if (env->me_psize) + env->me_maxpg = env->me_mapsize / env->me_psize; + return MDB_SUCCESS; +} + +int +mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs) +{ + if (env->me_map) + return EINVAL; + env->me_maxdbs = dbs + 2; /* Named databases + main and free DB */ + return MDB_SUCCESS; +} + +int +mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) +{ + if (env->me_map || readers < 1) + return EINVAL; + env->me_maxreaders = readers; + return MDB_SUCCESS; +} + +int +mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) +{ + if (!env || !readers) + return EINVAL; + *readers = env->me_maxreaders; + return MDB_SUCCESS; +} + +/** Further setup required for opening an MDB environment + */ +static int +mdb_env_open2(MDB_env *env) +{ + unsigned int flags = env->me_flags; + int i, newenv = 0; + MDB_meta meta; + MDB_page *p; +#ifndef _WIN32 + int prot; +#endif + + memset(&meta, 0, sizeof(meta)); + + if ((i = mdb_env_read_header(env, &meta)) != 0) { + if (i != ENOENT) + return i; + DPUTS("new mdbenv"); + newenv = 1; + } + + /* Was a mapsize configured? */ + if (!env->me_mapsize) { + /* If this is a new environment, take the default, + * else use the size recorded in the existing env. + */ + env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize; + } else if (env->me_mapsize < meta.mm_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; + if (env->me_mapsize < minsize) + env->me_mapsize = minsize; + } + +#ifdef _WIN32 + { + int rc; + HANDLE mh; + LONG sizelo, sizehi; + sizelo = env->me_mapsize & 0xffffffff; + sizehi = env->me_mapsize >> 16 >> 16; /* only needed on Win64 */ + /* Windows won't create mappings for zero length files. + * Just allocate the maxsize right now. + */ + if (newenv) { + if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo + || !SetEndOfFile(env->me_fd) + || SetFilePointer(env->me_fd, 0, NULL, 0) != 0) + return ErrCode(); + } + mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ? + PAGE_READWRITE : PAGE_READONLY, + sizehi, sizelo, NULL); + if (!mh) + return ErrCode(); + env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ? + FILE_MAP_WRITE : FILE_MAP_READ, + 0, 0, env->me_mapsize, meta.mm_address); + rc = env->me_map ? 0 : ErrCode(); + CloseHandle(mh); + if (rc) + return rc; + } +#else + i = MAP_SHARED; + prot = PROT_READ; + if (flags & MDB_WRITEMAP) { + prot |= PROT_WRITE; + if (ftruncate(env->me_fd, env->me_mapsize) < 0) + return ErrCode(); + } + env->me_map = mmap(meta.mm_address, env->me_mapsize, prot, i, + env->me_fd, 0); + if (env->me_map == MAP_FAILED) { + env->me_map = NULL; + return ErrCode(); + } + /* Turn off readahead. It's harmful when the DB is larger than RAM. */ +#ifdef MADV_RANDOM + madvise(env->me_map, env->me_mapsize, MADV_RANDOM); +#else +#ifdef POSIX_MADV_RANDOM + posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM); +#endif /* POSIX_MADV_RANDOM */ +#endif /* MADV_RANDOM */ +#endif /* _WIN32 */ + + if (newenv) { + if (flags & MDB_FIXEDMAP) + meta.mm_address = env->me_map; + i = mdb_env_init_meta(env, &meta); + if (i != MDB_SUCCESS) { + return i; + } + } else if (meta.mm_address && env->me_map != meta.mm_address) { + /* Can happen because the address argument to mmap() is just a + * hint. mmap() can pick another, e.g. if the range is in use. + * The MAP_FIXED flag would prevent that, but then mmap could + * instead unmap existing pages to make room for the new map. + */ + return EBUSY; /* TODO: Make a new MDB_* error code? */ + } + env->me_psize = meta.mm_psize; + env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; + env->me_nodemax = (env->me_psize - PAGEHDRSZ) / MDB_MINKEYS; + + env->me_maxpg = env->me_mapsize / env->me_psize; + + p = (MDB_page *)env->me_map; + env->me_metas[0] = METADATA(p); + env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + meta.mm_psize); + +#if MDB_DEBUG + { + int toggle = mdb_env_pick_meta(env); + MDB_db *db = &env->me_metas[toggle]->mm_dbs[MAIN_DBI]; + + DPRINTF("opened database version %u, pagesize %u", + env->me_metas[0]->mm_version, env->me_psize); + DPRINTF("using meta page %d", toggle); + DPRINTF("depth: %u", db->md_depth); + DPRINTF("entries: %zu", db->md_entries); + DPRINTF("branch pages: %zu", db->md_branch_pages); + DPRINTF("leaf pages: %zu", db->md_leaf_pages); + DPRINTF("overflow pages: %zu", db->md_overflow_pages); + DPRINTF("root: %zu", db->md_root); + } +#endif + + return MDB_SUCCESS; +} + + +/** Release a reader thread's slot in the reader lock table. + * This function is called automatically when a thread exits. + * @param[in] ptr This points to the slot in the reader lock table. + */ +static void +mdb_env_reader_dest(void *ptr) +{ + MDB_reader *reader = ptr; + + reader->mr_pid = 0; +} + +#ifdef _WIN32 +/** Junk for arranging thread-specific callbacks on Windows. This is + * necessarily platform and compiler-specific. Windows supports up + * to 1088 keys. Let's assume nobody opens more than 64 environments + * in a single process, for now. They can override this if needed. + */ +#ifndef MAX_TLS_KEYS +#define MAX_TLS_KEYS 64 +#endif +static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS]; +static int mdb_tls_nkeys; + +static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr) +{ + int i; + switch(reason) { + case DLL_PROCESS_ATTACH: break; + case DLL_THREAD_ATTACH: break; + case DLL_THREAD_DETACH: + for (i=0; ime_txns->mti_txnid = env->me_metas[toggle]->mm_txnid; + +#ifdef _WIN32 + { + OVERLAPPED ov; + /* First acquire a shared lock. The Unlock will + * then release the existing exclusive lock. + */ + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + rc = ErrCode(); + } else { + UnlockFile(env->me_lfd, 0, 0, 1, 0); + *excl = 0; + } + } +#else + { + struct flock lock_info; + /* The shared lock replaces the existing lock */ + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_RDLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + *excl = rc ? -1 : 0; /* error may mean we lost the lock */ + } +#endif + + return rc; +} + +/** Try to get exlusive lock, otherwise shared. + * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. + */ +static int +mdb_env_excl_lock(MDB_env *env, int *excl) +{ + int rc = 0; +#ifdef _WIN32 + if (LockFile(env->me_lfd, 0, 0, 1, 0)) { + *excl = 1; + } else { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + *excl = 0; + } else { + rc = ErrCode(); + } + } +#else + struct flock lock_info; + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (!rc) { + *excl = 1; + } else +# ifdef MDB_USE_POSIX_SEM + if (*excl < 0) /* always true when !MDB_USE_POSIX_SEM */ +# endif + { + lock_info.l_type = F_RDLCK; + while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (rc == 0) + *excl = 0; + } +#endif + return rc; +} + +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) +/* + * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code + * + * @(#) $Revision: 5.1 $ + * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ + * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ + * + * http://www.isthe.com/chongo/tech/comp/fnv/index.html + * + *** + * + * Please do not copyright this code. This code is in the public domain. + * + * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO + * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + * By: + * chongo /\oo/\ + * http://www.isthe.com/chongo/ + * + * Share and Enjoy! :-) + */ + +typedef unsigned long long mdb_hash_t; +#define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL) + +/** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer + * @param[in] str string to hash + * @param[in] hval initial value for hash + * @return 64 bit hash + * + * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the + * hval arg on the first call. + */ +static mdb_hash_t +mdb_hash_val(MDB_val *val, mdb_hash_t hval) +{ + unsigned char *s = (unsigned char *)val->mv_data; /* unsigned string */ + unsigned char *end = s + val->mv_size; + /* + * FNV-1a hash each octet of the string + */ + while (s < end) { + /* xor the bottom with the current octet */ + hval ^= (mdb_hash_t)*s++; + + /* multiply by the 64 bit FNV magic prime mod 2^64 */ + hval += (hval << 1) + (hval << 4) + (hval << 5) + + (hval << 7) + (hval << 8) + (hval << 40); + } + /* return our new hash value */ + return hval; +} + +/** Hash the string and output the hash in hex. + * @param[in] str string to hash + * @param[out] hexbuf an array of 17 chars to hold the hash + */ +static void +mdb_hash_hex(MDB_val *val, char *hexbuf) +{ + int i; + mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT); + for (i=0; i<8; i++) { + hexbuf += sprintf(hexbuf, "%02x", (unsigned int)h & 0xff); + h >>= 8; + } +} +#endif + +/** Open and/or initialize the lock region for the environment. + * @param[in] env The MDB environment. + * @param[in] lpath The pathname of the file used for the lock region. + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[out] excl Resulting file lock type: -1 none, 0 shared, 1 exclusive + * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive + * @return 0 on success, non-zero on failure. + */ +static int +mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) +{ +#ifdef _WIN32 +# define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT +#else +# define MDB_ERRCODE_ROFS EROFS +#ifdef O_CLOEXEC /* Linux: Open file and set FD_CLOEXEC atomically */ +# define MDB_CLOEXEC O_CLOEXEC +#else + int fdflags; +# define MDB_CLOEXEC 0 +#endif +#endif + int rc; + off_t size, rsize; + +#ifdef _WIN32 + env->me_lfd = CreateFile(lpath, GENERIC_READ|GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, NULL); +#else + env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode); +#endif + if (env->me_lfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { + return MDB_SUCCESS; + } + goto fail_errno; + } +#if ! ((MDB_CLOEXEC) || defined(_WIN32)) + /* Lose record locks when exec*() */ + if ((fdflags = fcntl(env->me_lfd, F_GETFD) | FD_CLOEXEC) >= 0) + fcntl(env->me_lfd, F_SETFD, fdflags); +#endif + + if (!(env->me_flags & MDB_NOTLS)) { + rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); + if (rc) + goto fail; + env->me_flags |= MDB_ENV_TXKEY; +#ifdef _WIN32 + /* Windows TLS callbacks need help finding their TLS info. */ + if (mdb_tls_nkeys >= MAX_TLS_KEYS) { + rc = MDB_TLS_FULL; + goto fail; + } + mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey; +#endif + } + + /* Try to get exclusive lock. If we succeed, then + * nobody is using the lock region and we should initialize it. + */ + if ((rc = mdb_env_excl_lock(env, excl))) goto fail; + +#ifdef _WIN32 + size = GetFileSize(env->me_lfd, NULL); +#else + size = lseek(env->me_lfd, 0, SEEK_END); + if (size == -1) goto fail_errno; +#endif + rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo); + if (size < rsize && *excl > 0) { +#ifdef _WIN32 + if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != rsize + || !SetEndOfFile(env->me_lfd)) + goto fail_errno; +#else + if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno; +#endif + } else { + rsize = size; + size = rsize - sizeof(MDB_txninfo); + env->me_maxreaders = size/sizeof(MDB_reader) + 1; + } + { +#ifdef _WIN32 + HANDLE mh; + mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE, + 0, 0, NULL); + if (!mh) goto fail_errno; + env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL); + CloseHandle(mh); + if (!env->me_txns) goto fail_errno; +#else + void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED, + env->me_lfd, 0); + if (m == MAP_FAILED) goto fail_errno; + env->me_txns = m; +#endif + } + if (*excl > 0) { +#ifdef _WIN32 + BY_HANDLE_FILE_INFORMATION stbuf; + struct { + DWORD volume; + DWORD nhigh; + DWORD nlow; + } idbuf; + MDB_val val; + char hexbuf[17]; + + if (!mdb_sec_inited) { + InitializeSecurityDescriptor(&mdb_null_sd, + SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE); + mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES); + mdb_all_sa.bInheritHandle = FALSE; + mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd; + mdb_sec_inited = 1; + } + if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.volume = stbuf.dwVolumeSerialNumber; + idbuf.nhigh = stbuf.nFileIndexHigh; + idbuf.nlow = stbuf.nFileIndexLow; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_hex(&val, hexbuf); + sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", hexbuf); + sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", hexbuf); + env->me_rmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + struct stat stbuf; + struct { + dev_t dev; + ino_t ino; + } idbuf; + MDB_val val; + char hexbuf[17]; + + if (fstat(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.dev = stbuf.st_dev; + idbuf.ino = stbuf.st_ino; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_hex(&val, hexbuf); + sprintf(env->me_txns->mti_rmname, "/MDBr%s", hexbuf); + sprintf(env->me_txns->mti_wmname, "/MDBw%s", hexbuf); + /* Clean up after a previous run, if needed: Try to + * remove both semaphores before doing anything else. + */ + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + env->me_rmutex = sem_open(env->me_txns->mti_rmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#else /* MDB_USE_POSIX_SEM */ + pthread_mutexattr_t mattr; + + if ((rc = pthread_mutexattr_init(&mattr)) + || (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) + || (rc = pthread_mutex_init(&env->me_txns->mti_mutex, &mattr)) + || (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr))) + goto fail; + pthread_mutexattr_destroy(&mattr); +#endif /* _WIN32 || MDB_USE_POSIX_SEM */ + + env->me_txns->mti_version = MDB_VERSION; + env->me_txns->mti_magic = MDB_MAGIC; + env->me_txns->mti_txnid = 0; + env->me_txns->mti_numreaders = 0; + + } else { + if (env->me_txns->mti_magic != MDB_MAGIC) { + DPUTS("lock region has invalid magic"); + rc = MDB_INVALID; + goto fail; + } + if (env->me_txns->mti_version != MDB_VERSION) { + DPRINTF("lock region is version %u, expected version %u", + env->me_txns->mti_version, MDB_VERSION); + rc = MDB_VERSION_MISMATCH; + goto fail; + } + rc = ErrCode(); + if (rc && rc != EACCES && rc != EAGAIN) { + goto fail; + } +#ifdef _WIN32 + env->me_rmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#endif + } + return MDB_SUCCESS; + +fail_errno: + rc = ErrCode(); +fail: + return rc; +} + + /** The name of the lock file in the DB environment */ +#define LOCKNAME "/lock.mdb" + /** The name of the data file in the DB environment */ +#define DATANAME "/data.mdb" + /** The suffix of the lock file when no subdir is used */ +#define LOCKSUFF "-lock" + /** Only a subset of the @ref mdb_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. + */ +#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC) +#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP|MDB_NOTLS) + +int +mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) +{ + int oflags, rc, len, excl = -1; + char *lpath, *dpath; + + if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) + return EINVAL; + + len = strlen(path); + if (flags & MDB_NOSUBDIR) { + rc = len + sizeof(LOCKSUFF) + len + 1; + } else { + rc = len + sizeof(LOCKNAME) + len + sizeof(DATANAME); + } + lpath = malloc(rc); + if (!lpath) + return ENOMEM; + if (flags & MDB_NOSUBDIR) { + dpath = lpath + len + sizeof(LOCKSUFF); + sprintf(lpath, "%s" LOCKSUFF, path); + strcpy(dpath, path); + } else { + dpath = lpath + len + sizeof(LOCKNAME); + sprintf(lpath, "%s" LOCKNAME, path); + sprintf(dpath, "%s" DATANAME, path); + } + + rc = MDB_SUCCESS; + flags |= env->me_flags; + if (flags & MDB_RDONLY) { + /* silently ignore WRITEMAP when we're only getting read access */ + flags &= ~MDB_WRITEMAP; + } else { + if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) && + (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) + rc = ENOMEM; + } + env->me_flags = flags |= MDB_ENV_ACTIVE; + if (rc) + goto leave; + + env->me_path = strdup(path); + env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); + env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); + if (!(env->me_dbxs && env->me_path && env->me_dbflags)) { + rc = ENOMEM; + goto leave; + } + + rc = mdb_env_setup_locks(env, lpath, mode, &excl); + if (rc) + goto leave; + +#ifdef _WIN32 + if (F_ISSET(flags, MDB_RDONLY)) { + oflags = GENERIC_READ; + len = OPEN_EXISTING; + } else { + oflags = GENERIC_READ|GENERIC_WRITE; + len = OPEN_ALWAYS; + } + mode = FILE_ATTRIBUTE_NORMAL; + env->me_fd = CreateFile(dpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, + NULL, len, mode, NULL); +#else + if (F_ISSET(flags, MDB_RDONLY)) + oflags = O_RDONLY; + else + oflags = O_RDWR | O_CREAT; + + env->me_fd = open(dpath, oflags, mode); +#endif + if (env->me_fd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + + if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) { + if (flags & (MDB_RDONLY|MDB_WRITEMAP)) { + env->me_mfd = env->me_fd; + } else { + /* Synchronous fd for meta writes. Needed even with + * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. + */ +#ifdef _WIN32 + env->me_mfd = CreateFile(dpath, oflags, + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, + mode | FILE_FLAG_WRITE_THROUGH, NULL); +#else + env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode); +#endif + if (env->me_mfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + } + DPRINTF("opened dbenv %p", (void *) env); + if (excl > 0) { + rc = mdb_env_share_locks(env, &excl); + } + } + +leave: + if (rc) { + mdb_env_close0(env, excl); + } + free(lpath); + return rc; +} + +/** Destroy resources from mdb_env_open(), clear our readers & DBIs */ +static void +mdb_env_close0(MDB_env *env, int excl) +{ + int i; + + if (!(env->me_flags & MDB_ENV_ACTIVE)) + return; + + /* Doing this here since me_dbxs may not exist during mdb_env_close */ + for (i = env->me_maxdbs; --i > MAIN_DBI; ) + free(env->me_dbxs[i].md_name.mv_data); + + free(env->me_dbflags); + free(env->me_dbxs); + free(env->me_path); + free(env->me_dirty_list); + mdb_midl_free(env->me_free_pgs); + + if (env->me_flags & MDB_ENV_TXKEY) { + pthread_key_delete(env->me_txkey); +#ifdef _WIN32 + /* Delete our key from the global list */ + for (i=0; ime_txkey) { + mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1]; + mdb_tls_nkeys--; + break; + } +#endif + } + + if (env->me_map) { + munmap(env->me_map, env->me_mapsize); + } + if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE) + (void) close(env->me_mfd); + if (env->me_fd != INVALID_HANDLE_VALUE) + (void) close(env->me_fd); + if (env->me_txns) { + pid_t pid = env->me_pid; + /* Clearing readers is done in this function because + * me_txkey with its destructor must be disabled first. + */ + for (i = env->me_numreaders; --i >= 0; ) + if (env->me_txns->mti_readers[i].mr_pid == pid) + env->me_txns->mti_readers[i].mr_pid = 0; +#ifdef _WIN32 + if (env->me_rmutex) { + CloseHandle(env->me_rmutex); + if (env->me_wmutex) CloseHandle(env->me_wmutex); + } + /* Windows automatically destroys the mutexes when + * the last handle closes. + */ +#elif defined(MDB_USE_POSIX_SEM) + if (env->me_rmutex != SEM_FAILED) { + sem_close(env->me_rmutex); + if (env->me_wmutex != SEM_FAILED) + sem_close(env->me_wmutex); + /* If we have the filelock: If we are the + * only remaining user, clean up semaphores. + */ + if (excl == 0) + mdb_env_excl_lock(env, &excl); + if (excl > 0) { + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + } + } +#endif + munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); + } + if (env->me_lfd != INVALID_HANDLE_VALUE) { +#ifdef _WIN32 + if (excl >= 0) { + /* Unlock the lockfile. Windows would have unlocked it + * after closing anyway, but not necessarily at once. + */ + UnlockFile(env->me_lfd, 0, 0, 1, 0); + } +#endif + (void) close(env->me_lfd); + } + + env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); +} + +int +mdb_env_copyfd(MDB_env *env, HANDLE fd) +{ + MDB_txn *txn = NULL; + int rc; + size_t wsize; + char *ptr; + + /* Do the lock/unlock of the reader mutex before starting the + * write txn. Otherwise other read txns could block writers. + */ + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + return rc; + + if (env->me_txns) { + /* We must start the actual read txn after blocking writers */ + mdb_txn_reset0(txn, "reset-stage1"); + + /* Temporarily block writers until we snapshot the meta pages */ + LOCK_MUTEX_W(env); + + rc = mdb_txn_renew0(txn); + if (rc) { + UNLOCK_MUTEX_W(env); + goto leave; + } + } + + wsize = env->me_psize * 2; +#ifdef _WIN32 + { + DWORD len; + rc = WriteFile(fd, env->me_map, wsize, &len, NULL); + rc = rc ? (len == wsize ? MDB_SUCCESS : EIO) : ErrCode(); + } +#else + rc = write(fd, env->me_map, wsize); + rc = rc == (int)wsize ? MDB_SUCCESS : rc < 0 ? ErrCode() : EIO; +#endif + if (env->me_txns) + UNLOCK_MUTEX_W(env); + + if (rc) + goto leave; + + ptr = env->me_map + wsize; + wsize = txn->mt_next_pgno * env->me_psize - wsize; +#ifdef _WIN32 + while (wsize > 0) { + DWORD len, w2; + if (wsize > MAX_WRITE) + w2 = MAX_WRITE; + else + w2 = wsize; + rc = WriteFile(fd, ptr, w2, &len, NULL); + rc = rc ? (len == w2 ? MDB_SUCCESS : EIO) : ErrCode(); + if (rc) break; + wsize -= w2; + ptr += w2; + } +#else + while (wsize > 0) { + size_t w2; + ssize_t wres; + if (wsize > MAX_WRITE) + w2 = MAX_WRITE; + else + w2 = wsize; + wres = write(fd, ptr, w2); + rc = wres == (ssize_t)w2 ? MDB_SUCCESS : wres < 0 ? ErrCode() : EIO; + if (rc) break; + wsize -= wres; + ptr += wres; + } +#endif + +leave: + mdb_txn_abort(txn); + return rc; +} + +int +mdb_env_copy(MDB_env *env, const char *path) +{ + int rc, len; + char *lpath; + HANDLE newfd = INVALID_HANDLE_VALUE; + + if (env->me_flags & MDB_NOSUBDIR) { + lpath = (char *)path; + } else { + len = strlen(path); + len += sizeof(DATANAME); + lpath = malloc(len); + if (!lpath) + return ENOMEM; + sprintf(lpath, "%s" DATANAME, path); + } + + /* The destination path must exist, but the destination file must not. + * We don't want the OS to cache the writes, since the source data is + * already in the OS cache. + */ +#ifdef _WIN32 + newfd = CreateFile(lpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, + FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); +#else + newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL +#ifdef O_DIRECT + |O_DIRECT +#endif + , 0666); +#endif + if (newfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + +#ifdef F_NOCACHE /* __APPLE__ */ + rc = fcntl(newfd, F_NOCACHE, 1); + if (rc) { + rc = ErrCode(); + goto leave; + } +#endif + + rc = mdb_env_copyfd(env, newfd); + +leave: + if (!(env->me_flags & MDB_NOSUBDIR)) + free(lpath); + if (newfd != INVALID_HANDLE_VALUE) + if (close(newfd) < 0 && rc == MDB_SUCCESS) + rc = ErrCode(); + + return rc; +} + +void +mdb_env_close(MDB_env *env) +{ + MDB_page *dp; + + if (env == NULL) + return; + + VGMEMP_DESTROY(env); + while ((dp = env->me_dpages) != NULL) { + VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); + env->me_dpages = dp->mp_next; + free(dp); + } + + mdb_env_close0(env, 0); + free(env); +} + +/** Compare two items pointing at aligned size_t's */ +static int +mdb_cmp_long(const MDB_val *a, const MDB_val *b) +{ + return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 : + *(size_t *)a->mv_data > *(size_t *)b->mv_data; +} + +/** Compare two items pointing at aligned int's */ +static int +mdb_cmp_int(const MDB_val *a, const MDB_val *b) +{ + return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 : + *(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data; +} + +/** Compare two items pointing at ints of unknown alignment. + * Nodes and keys are guaranteed to be 2-byte aligned. + */ +static int +mdb_cmp_cint(const MDB_val *a, const MDB_val *b) +{ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short *u, *c; + int x; + + u = (unsigned short *) ((char *) a->mv_data + a->mv_size); + c = (unsigned short *) ((char *) b->mv_data + a->mv_size); + do { + x = *--u - *--c; + } while(!x && u > (unsigned short *)a->mv_data); + return x; +#else + return memcmp(a->mv_data, b->mv_data, a->mv_size); +#endif +} + +/** Compare two items lexically */ +static int +mdb_cmp_memn(const MDB_val *a, const MDB_val *b) +{ + int diff; + ssize_t len_diff; + unsigned int len; + + len = a->mv_size; + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + len = b->mv_size; + len_diff = 1; + } + + diff = memcmp(a->mv_data, b->mv_data, len); + return diff ? diff : len_diff<0 ? -1 : len_diff; +} + +/** Compare two items in reverse byte order */ +static int +mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) +{ + const unsigned char *p1, *p2, *p1_lim; + ssize_t len_diff; + int diff; + + p1_lim = (const unsigned char *)a->mv_data; + p1 = (const unsigned char *)a->mv_data + a->mv_size; + p2 = (const unsigned char *)b->mv_data + b->mv_size; + + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + p1_lim += len_diff; + len_diff = 1; + } + + while (p1 > p1_lim) { + diff = *--p1 - *--p2; + if (diff) + return diff; + } + return len_diff<0 ? -1 : len_diff; +} + +/** Search for key within a page, using binary search. + * Returns the smallest entry larger or equal to the key. + * If exactp is non-null, stores whether the found entry was an exact match + * in *exactp (1 or 0). + * Updates the cursor index with the index of the found entry. + * If no entry larger or equal to the key is found, returns NULL. + */ +static MDB_node * +mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) +{ + unsigned int i = 0, nkeys; + int low, high; + int rc = 0; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NULL; + MDB_val nodekey; + MDB_cmp_func *cmp; + DKBUF; + + nkeys = NUMKEYS(mp); + +#if MDB_DEBUG + { + pgno_t pgno; + COPY_PGNO(pgno, mp->mp_pgno); + DPRINTF("searching %u keys in %s %spage %zu", + nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", + pgno); + } +#endif + + assert(nkeys > 0); + + low = IS_LEAF(mp) ? 0 : 1; + high = nkeys - 1; + cmp = mc->mc_dbx->md_cmp; + + /* Branch pages have no data, so if using integer keys, + * alignment is guaranteed. Use faster mdb_cmp_int. + */ + if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { + if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t)) + cmp = mdb_cmp_long; + else + cmp = mdb_cmp_int; + } + + if (IS_LEAF2(mp)) { + nodekey.mv_size = mc->mc_db->md_pad; + node = NODEPTR(mp, 0); /* fake */ + while (low <= high) { + i = (low + high) >> 1; + nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size); + rc = cmp(key, &nodekey); + DPRINTF("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc); + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } else { + while (low <= high) { + i = (low + high) >> 1; + + node = NODEPTR(mp, i); + nodekey.mv_size = NODEKSZ(node); + nodekey.mv_data = NODEKEY(node); + + rc = cmp(key, &nodekey); +#if MDB_DEBUG + if (IS_LEAF(mp)) + DPRINTF("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc); + else + DPRINTF("found branch index %u [%s -> %zu], rc = %i", + i, DKEY(&nodekey), NODEPGNO(node), rc); +#endif + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } + + if (rc > 0) { /* Found entry is less than the key. */ + i++; /* Skip to get the smallest entry larger than key. */ + if (!IS_LEAF2(mp)) + node = NODEPTR(mp, i); + } + if (exactp) + *exactp = (rc == 0); + /* store the key index */ + mc->mc_ki[mc->mc_top] = i; + if (i >= nkeys) + /* There is no entry larger or equal to the key. */ + return NULL; + + /* nodeptr is fake for LEAF2 */ + return node; +} + +#if 0 +static void +mdb_cursor_adjust(MDB_cursor *mc, func) +{ + MDB_cursor *m2; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { + func(mc, m2); + } + } +} +#endif + +/** Pop a page off the top of the cursor's stack. */ +static void +mdb_cursor_pop(MDB_cursor *mc) +{ + if (mc->mc_snum) { +#ifndef MDB_DEBUG_SKIP + MDB_page *top = mc->mc_pg[mc->mc_top]; +#endif + mc->mc_snum--; + if (mc->mc_snum) + mc->mc_top--; + + DPRINTF("popped page %zu off db %u cursor %p", top->mp_pgno, + mc->mc_dbi, (void *) mc); + } +} + +/** Push a page onto the top of the cursor's stack. */ +static int +mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) +{ + DPRINTF("pushing page %zu on db %u cursor %p", mp->mp_pgno, + mc->mc_dbi, (void *) mc); + + if (mc->mc_snum >= CURSOR_STACK) { + assert(mc->mc_snum < CURSOR_STACK); + return MDB_CURSOR_FULL; + } + + mc->mc_top = mc->mc_snum++; + mc->mc_pg[mc->mc_top] = mp; + mc->mc_ki[mc->mc_top] = 0; + + return MDB_SUCCESS; +} + +/** Find the address of the page corresponding to a given page number. + * @param[in] txn the transaction for this access. + * @param[in] pgno the page number for the page to retrieve. + * @param[out] ret address of a pointer where the page's address will be stored. + * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl) +{ + MDB_page *p = NULL; + int level; + + if (!((txn->mt_flags & MDB_TXN_RDONLY) | + (txn->mt_env->me_flags & MDB_WRITEMAP))) + { + MDB_txn *tx2 = txn; + level = 1; + do { + MDB_ID2L dl = tx2->mt_u.dirty_list; + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + p = dl[x].mptr; + goto done; + } + } + level++; + } while ((tx2 = tx2->mt_parent) != NULL); + } + + if (pgno < txn->mt_next_pgno) { + level = 0; + p = (MDB_page *)(txn->mt_env->me_map + txn->mt_env->me_psize * pgno); + } else { + DPRINTF("page %zu not found", pgno); + assert(p != NULL); + return MDB_PAGE_NOTFOUND; + } + +done: + *ret = p; + if (lvl) + *lvl = level; + return MDB_SUCCESS; +} + +/** Search for the page a given key should be in. + * Pushes parent pages on the cursor stack. This function continues a + * search on a cursor that has already been initialized. (Usually by + * #mdb_page_search() but also by #mdb_node_move().) + * @param[in,out] mc the cursor for this operation. + * @param[in] key the key to search for. If NULL, search for the lowest + * page. (This is used by #mdb_cursor_first().) + * @param[in] modify If true, visited pages are updated with new page numbers. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int modify) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + DKBUF; + int rc; + + + while (IS_BRANCH(mp)) { + MDB_node *node; + indx_t i; + + DPRINTF("branch page %zu has %u keys", mp->mp_pgno, NUMKEYS(mp)); + assert(NUMKEYS(mp) > 1); + DPRINTF("found index 0 to page %zu", NODEPGNO(NODEPTR(mp, 0))); + + if (key == NULL) /* Initialize cursor to first page. */ + i = 0; + else if (key->mv_size > MDB_MAXKEYSIZE && key->mv_data == NULL) { + /* cursor to last page */ + i = NUMKEYS(mp)-1; + } else { + int exact; + node = mdb_node_search(mc, key, &exact); + if (node == NULL) + i = NUMKEYS(mp) - 1; + else { + i = mc->mc_ki[mc->mc_top]; + if (!exact) { + assert(i > 0); + i--; + } + } + } + + if (key) + DPRINTF("following index %u for key [%s]", + i, DKEY(key)); + assert(i < NUMKEYS(mp)); + node = NODEPTR(mp, i); + + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = i; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + + if (modify) { + if ((rc = mdb_page_touch(mc)) != 0) + return rc; + mp = mc->mc_pg[mc->mc_top]; + } + } + + if (!IS_LEAF(mp)) { + DPRINTF("internal error, index points to a %02X page!?", + mp->mp_flags); + return MDB_CORRUPTED; + } + + DPRINTF("found leaf page %zu for key [%s]", mp->mp_pgno, + key ? DKEY(key) : NULL); + + return MDB_SUCCESS; +} + +/** Search for the lowest key under the current branch page. + * This just bypasses a NUMKEYS check in the current page + * before calling mdb_page_search_root(), because the callers + * are all in situations where the current page is known to + * be underfilled. + */ +static int +mdb_page_search_lowest(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NODEPTR(mp, 0); + int rc; + + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = 0; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + return mdb_page_search_root(mc, NULL, 0); +} + +/** Search for the page a given key should be in. + * Pushes parent pages on the cursor stack. This function just sets up + * the search; it finds the root page for \b mc's database and sets this + * as the root of the cursor's stack. Then #mdb_page_search_root() is + * called to complete the search. + * @param[in,out] mc the cursor for this operation. + * @param[in] key the key to search for. If NULL, search for the lowest + * page. (This is used by #mdb_cursor_first().) + * @param[in] flags If MDB_PS_MODIFY set, visited pages are updated with new page numbers. + * If MDB_PS_ROOTONLY set, just fetch root node, no further lookups. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) +{ + int rc; + pgno_t root; + + /* Make sure the txn is still viable, then find the root from + * the txn's db table. + */ + if (F_ISSET(mc->mc_txn->mt_flags, MDB_TXN_ERROR)) { + DPUTS("transaction has failed, must abort"); + return EINVAL; + } else { + /* Make sure we're using an up-to-date root */ + if (mc->mc_dbi > MAIN_DBI) { + if ((*mc->mc_dbflag & DB_STALE) || + ((flags & MDB_PS_MODIFY) && !(*mc->mc_dbflag & DB_DIRTY))) { + MDB_cursor mc2; + unsigned char dbflag = 0; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, flags & MDB_PS_MODIFY); + if (rc) + return rc; + if (*mc->mc_dbflag & DB_STALE) { + MDB_val data; + int exact = 0; + uint16_t flags; + MDB_node *leaf = mdb_node_search(&mc2, + &mc->mc_dbx->md_name, &exact); + if (!exact) + return MDB_NOTFOUND; + rc = mdb_node_read(mc->mc_txn, leaf, &data); + if (rc) + return rc; + memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)), + sizeof(uint16_t)); + /* The txn may not know this DBI, or another process may + * have dropped and recreated the DB with other flags. + */ + if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags) + return MDB_INCOMPATIBLE; + memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db)); + } + if (flags & MDB_PS_MODIFY) + dbflag = DB_DIRTY; + *mc->mc_dbflag &= ~DB_STALE; + *mc->mc_dbflag |= dbflag; + } + } + root = mc->mc_db->md_root; + + if (root == P_INVALID) { /* Tree is empty. */ + DPUTS("tree is empty"); + return MDB_NOTFOUND; + } + } + + assert(root > 1); + if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) + if ((rc = mdb_page_get(mc->mc_txn, root, &mc->mc_pg[0], NULL)) != 0) + return rc; + + mc->mc_snum = 1; + mc->mc_top = 0; + + DPRINTF("db %u root page %zu has flags 0x%X", + mc->mc_dbi, root, mc->mc_pg[0]->mp_flags); + + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc))) + return rc; + } + + if (flags & MDB_PS_ROOTONLY) + return MDB_SUCCESS; + + return mdb_page_search_root(mc, key, flags); +} + +static int +mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) +{ + MDB_txn *txn = mc->mc_txn; + pgno_t pg = mp->mp_pgno; + unsigned i, ovpages = mp->mp_pages; + MDB_env *env = txn->mt_env; + int rc; + + DPRINTF("free ov page %zu (%d)", pg, ovpages); + /* If the page is dirty we just acquired it, so we should + * give it back to our current free list, if any. + * Not currently supported in nested txns. + * Otherwise put it onto the list of pages we freed in this txn. + */ + if ((mp->mp_flags & P_DIRTY) && !txn->mt_parent && env->me_pghead) { + unsigned j, x; + pgno_t *mop; + MDB_ID2 *dl, ix, iy; + rc = mdb_midl_need(&env->me_pghead, ovpages); + if (rc) + return rc; + /* Remove from dirty list */ + dl = txn->mt_u.dirty_list; + x = dl[0].mid--; + for (ix = dl[x]; ix.mptr != mp; ix = iy) { + if (x > 1) { + x--; + iy = dl[x]; + dl[x] = ix; + } else { + assert(x > 1); + j = ++(dl[0].mid); + dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + } + if (!(env->me_flags & MDB_WRITEMAP)) + mdb_dpage_free(env, mp); + /* Insert in me_pghead */ + mop = env->me_pghead; + j = mop[0] + ovpages; + for (i = mop[0]; i && mop[i] < pg; i--) + mop[j--] = mop[i]; + while (j>i) + mop[j--] = pg++; + mop[0] += ovpages; + } else { + rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); + if (rc) + return rc; + } + mc->mc_db->md_overflow_pages -= ovpages; + return 0; +} + +/** Return the data associated with a given node. + * @param[in] txn The transaction for this operation. + * @param[in] leaf The node being read. + * @param[out] data Updated to point to the node's data. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data) +{ + MDB_page *omp; /* overflow page */ + pgno_t pgno; + int rc; + + if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { + data->mv_size = NODEDSZ(leaf); + data->mv_data = NODEDATA(leaf); + return MDB_SUCCESS; + } + + /* Read overflow data. + */ + data->mv_size = NODEDSZ(leaf); + memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); + if ((rc = mdb_page_get(txn, pgno, &omp, NULL)) != 0) { + DPRINTF("read overflow page %zu failed", pgno); + return rc; + } + data->mv_data = METADATA(omp); + + return MDB_SUCCESS; +} + +int +mdb_get(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + MDB_cursor mc; + MDB_xcursor mx; + int exact = 0; + DKBUF; + + assert(key); + assert(data); + DPRINTF("===> get db %u key [%s]", dbi, DKEY(key)); + + if (txn == NULL || !dbi || dbi >= txn->mt_numdbs || !(txn->mt_dbflags[dbi] & DB_VALID)) + return EINVAL; + + if (key->mv_size == 0 || key->mv_size > MDB_MAXKEYSIZE) { + return EINVAL; + } + + mdb_cursor_init(&mc, txn, dbi, &mx); + return mdb_cursor_set(&mc, key, data, MDB_SET, &exact); +} + +/** Find a sibling for a page. + * Replaces the page at the top of the cursor's stack with the + * specified sibling, if one exists. + * @param[in] mc The cursor for this operation. + * @param[in] move_right Non-zero if the right sibling is requested, + * otherwise the left sibling. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_cursor_sibling(MDB_cursor *mc, int move_right) +{ + int rc; + MDB_node *indx; + MDB_page *mp; + + if (mc->mc_snum < 2) { + return MDB_NOTFOUND; /* root has no siblings */ + } + + mdb_cursor_pop(mc); + DPRINTF("parent page is page %zu, index %u", + mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]); + + if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) + : (mc->mc_ki[mc->mc_top] == 0)) { + DPRINTF("no more keys left, moving to %s sibling", + move_right ? "right" : "left"); + if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) { + /* undo cursor_pop before returning */ + mc->mc_top++; + mc->mc_snum++; + return rc; + } + } else { + if (move_right) + mc->mc_ki[mc->mc_top]++; + else + mc->mc_ki[mc->mc_top]--; + DPRINTF("just moving to %s index key %u", + move_right ? "right" : "left", mc->mc_ki[mc->mc_top]); + } + assert(IS_BRANCH(mc->mc_pg[mc->mc_top])); + + indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(indx), &mp, NULL) != 0)) + return rc; + + mdb_cursor_push(mc, mp); + if (!move_right) + mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1; + + return MDB_SUCCESS; +} + +/** Move the cursor to the next data item. */ +static int +mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + if (mc->mc_flags & C_EOF) { + return MDB_NOTFOUND; + } + + assert(mc->mc_flags & C_INITIALIZED); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_NEXT || op == MDB_NEXT_DUP) { + rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT); + if (op != MDB_NEXT || rc != MDB_NOTFOUND) + return rc; + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_NEXT_DUP) + return MDB_NOTFOUND; + } + } + + DPRINTF("cursor_next: top page is %zu in cursor %p", mp->mp_pgno, (void *) mc); + + if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { + DPUTS("=====> move to next sibling page"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { + mc->mc_flags |= C_EOF; + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + DPRINTF("next page is %zu, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]); + } else + mc->mc_ki[mc->mc_top]++; + + DPRINTF("==> cursor points to page %zu with %u keys, key index %u", + mp->mp_pgno, NUMKEYS(mp), mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + assert(IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the previous data item. */ +static int +mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + assert(mc->mc_flags & C_INITIALIZED); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (op == MDB_PREV || op == MDB_PREV_DUP) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV); + if (op != MDB_PREV || rc != MDB_NOTFOUND) + return rc; + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_PREV_DUP) + return MDB_NOTFOUND; + } + } + } + + DPRINTF("cursor_prev: top page is %zu in cursor %p", mp->mp_pgno, (void *) mc); + + if (mc->mc_ki[mc->mc_top] == 0) { + DPUTS("=====> move to prev sibling page"); + if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; + DPRINTF("prev page is %zu, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]); + } else + mc->mc_ki[mc->mc_top]--; + + mc->mc_flags &= ~C_EOF; + + DPRINTF("==> cursor points to page %zu with %u keys, key index %u", + mp->mp_pgno, NUMKEYS(mp), mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + assert(IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Set the cursor on a specific data item. */ +static int +mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op, int *exactp) +{ + int rc; + MDB_page *mp; + MDB_node *leaf = NULL; + DKBUF; + + assert(mc); + assert(key); + assert(key->mv_size > 0); + + /* See if we're already on the right page */ + if (mc->mc_flags & C_INITIALIZED) { + MDB_val nodekey; + + mp = mc->mc_pg[mc->mc_top]; + if (!NUMKEYS(mp)) { + mc->mc_ki[mc->mc_top] = 0; + return MDB_NOTFOUND; + } + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_size = mc->mc_db->md_pad; + nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, 0); + MDB_GET_KEY(leaf, &nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* Probably happens rarely, but first node on the page + * was the one we wanted. + */ + mc->mc_ki[mc->mc_top] = 0; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc > 0) { + unsigned int i; + unsigned int nkeys = NUMKEYS(mp); + if (nkeys > 1) { + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + nkeys-1, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, nkeys-1); + MDB_GET_KEY(leaf, &nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* last node was the one we wanted */ + mc->mc_ki[mc->mc_top] = nkeys-1; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc < 0) { + if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { + /* This is definitely the right page, skip search_page */ + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + mc->mc_ki[mc->mc_top], nodekey.mv_size); + } else { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY(leaf, &nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* current node was the one we wanted */ + if (exactp) + *exactp = 1; + goto set1; + } + } + rc = 0; + goto set2; + } + } + /* If any parents have right-sibs, search. + * Otherwise, there's nothing further. + */ + for (i=0; imc_top; i++) + if (mc->mc_ki[i] < + NUMKEYS(mc->mc_pg[i])-1) + break; + if (i == mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = nkeys; + return MDB_NOTFOUND; + } + } + if (!mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = 0; + return MDB_NOTFOUND; + } + } + + rc = mdb_page_search(mc, key, 0); + if (rc != MDB_SUCCESS) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + assert(IS_LEAF(mp)); + +set2: + leaf = mdb_node_search(mc, key, exactp); + if (exactp != NULL && !*exactp) { + /* MDB_SET specified and not an exact match. */ + return MDB_NOTFOUND; + } + + if (leaf == NULL) { + DPUTS("===> inexact leaf not found, goto sibling"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) + return rc; /* no entries matched */ + mp = mc->mc_pg[mc->mc_top]; + assert(IS_LEAF(mp)); + leaf = NODEPTR(mp, 0); + } + +set1: + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + } else { + int ex2, *ex2p; + if (op == MDB_GET_BOTH) { + ex2p = &ex2; + ex2 = 0; + } else { + ex2p = NULL; + } + rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p); + if (rc != MDB_SUCCESS) + return rc; + } + } else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { + MDB_val d2; + if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS) + return rc; + rc = mc->mc_dbx->md_dcmp(data, &d2); + if (rc) { + if (op == MDB_GET_BOTH || rc > 0) + return MDB_NOTFOUND; + } + + } else { + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + /* The key already matches in all other cases */ + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) + MDB_GET_KEY(leaf, key); + DPRINTF("==> cursor placed on key [%s]", DKEY(key)); + + return rc; +} + +/** Move the cursor to the first item in the database. */ +static int +mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, 0); + if (rc != MDB_SUCCESS) + return rc; + } + assert(IS_LEAF(mc->mc_pg[mc->mc_top])); + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + mc->mc_ki[mc->mc_top] = 0; + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size); + return MDB_SUCCESS; + } + + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else { + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the last item in the database. */ +static int +mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (!(mc->mc_flags & C_EOF)) { + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + MDB_val lkey; + + lkey.mv_size = MDB_MAXKEYSIZE+1; + lkey.mv_data = NULL; + rc = mdb_page_search(mc, &lkey, 0); + if (rc != MDB_SUCCESS) + return rc; + } + assert(IS_LEAF(mc->mc_pg[mc->mc_top])); + + } + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; + mc->mc_flags |= C_INITIALIZED|C_EOF; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else { + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +int +mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op) +{ + int rc; + int exact = 0; + + assert(mc); + + switch (op) { + case MDB_GET_CURRENT: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + } else { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + if (!NUMKEYS(mp)) { + mc->mc_ki[mc->mc_top] = 0; + rc = MDB_NOTFOUND; + break; + } + rc = MDB_SUCCESS; + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } else { + MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY(leaf, key); + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); + } else { + rc = mdb_node_read(mc->mc_txn, leaf, data); + } + } + } + } + break; + case MDB_GET_BOTH: + case MDB_GET_BOTH_RANGE: + if (data == NULL || mc->mc_xcursor == NULL) { + rc = EINVAL; + break; + } + /* FALLTHRU */ + case MDB_SET: + case MDB_SET_KEY: + case MDB_SET_RANGE: + if (key == NULL || key->mv_size == 0 || key->mv_size > MDB_MAXKEYSIZE) { + rc = EINVAL; + } else if (op == MDB_SET_RANGE) + rc = mdb_cursor_set(mc, key, data, op, NULL); + else + rc = mdb_cursor_set(mc, key, data, op, &exact); + break; + case MDB_GET_MULTIPLE: + if (data == NULL || + !(mc->mc_db->md_flags & MDB_DUPFIXED) || + !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + rc = MDB_SUCCESS; + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || + (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) + break; + goto fetchm; + case MDB_NEXT_MULTIPLE: + if (data == NULL || + !(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = EINVAL; + break; + } + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_first(mc, key, data); + else + rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); + if (rc == MDB_SUCCESS) { + if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + MDB_cursor *mx; +fetchm: + mx = &mc->mc_xcursor->mx_cursor; + data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) * + mx->mc_db->md_pad; + data->mv_data = METADATA(mx->mc_pg[mx->mc_top]); + mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1; + } else { + rc = MDB_NOTFOUND; + } + } + break; + case MDB_NEXT: + case MDB_NEXT_DUP: + case MDB_NEXT_NODUP: + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_first(mc, key, data); + else + rc = mdb_cursor_next(mc, key, data, op); + break; + case MDB_PREV: + case MDB_PREV_DUP: + case MDB_PREV_NODUP: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = mdb_cursor_last(mc, key, data); + if (rc) + break; + mc->mc_flags |= C_INITIALIZED; + mc->mc_ki[mc->mc_top]++; + } + rc = mdb_cursor_prev(mc, key, data, op); + break; + case MDB_FIRST: + rc = mdb_cursor_first(mc, key, data); + break; + case MDB_FIRST_DUP: + if (data == NULL || + !(mc->mc_db->md_flags & MDB_DUPSORT) || + !(mc->mc_flags & C_INITIALIZED) || + !(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + break; + case MDB_LAST: + rc = mdb_cursor_last(mc, key, data); + break; + case MDB_LAST_DUP: + if (data == NULL || + !(mc->mc_db->md_flags & MDB_DUPSORT) || + !(mc->mc_flags & C_INITIALIZED) || + !(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + break; + default: + DPRINTF("unhandled/unimplemented cursor operation %u", op); + rc = EINVAL; + break; + } + + return rc; +} + +/** Touch all the pages in the cursor stack. + * Makes sure all the pages are writable, before attempting a write operation. + * @param[in] mc The cursor to operate on. + */ +static int +mdb_cursor_touch(MDB_cursor *mc) +{ + int rc; + + if (mc->mc_dbi > MAIN_DBI && !(*mc->mc_dbflag & DB_DIRTY)) { + MDB_cursor mc2; + MDB_xcursor mcx; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); + if (rc) + return rc; + *mc->mc_dbflag |= DB_DIRTY; + } + for (mc->mc_top = 0; mc->mc_top < mc->mc_snum; mc->mc_top++) { + rc = mdb_page_touch(mc); + if (rc) + return rc; + } + mc->mc_top = mc->mc_snum-1; + return MDB_SUCCESS; +} + +int +mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, + unsigned int flags) +{ + MDB_node *leaf = NULL; + MDB_val xdata, *rdata, dkey; + MDB_page *fp; + MDB_db dummy; + int do_sub = 0, insert = 0; + unsigned int mcount = 0, dcount = 0; + size_t nsize; + int rc, rc2; + MDB_pagebuf pbuf; + char dbuf[MDB_MAXKEYSIZE+1]; + unsigned int nflags; + DKBUF; + + /* Check this first so counter will always be zero on any + * early failures. + */ + if (flags & MDB_MULTIPLE) { + dcount = data[1].mv_size; + data[1].mv_size = 0; + if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) + return EINVAL; + } + + if (F_ISSET(mc->mc_txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; + + if (flags != MDB_CURRENT && (key->mv_size == 0 || key->mv_size > MDB_MAXKEYSIZE)) + return EINVAL; + + if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT) && data->mv_size > MDB_MAXKEYSIZE) + return EINVAL; + +#if SIZE_MAX > MAXDATASIZE + if (data->mv_size > MAXDATASIZE) + return EINVAL; +#endif + + DPRINTF("==> put db %u key [%s], size %zu, data size %zu", + mc->mc_dbi, DKEY(key), key ? key->mv_size:0, data->mv_size); + + dkey.mv_size = 0; + + if (flags == MDB_CURRENT) { + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + rc = MDB_SUCCESS; + } else if (mc->mc_db->md_root == P_INVALID) { + MDB_page *np; + /* new database, write a root leaf page */ + DPUTS("allocating new root leaf page"); + if ((rc = mdb_page_new(mc, P_LEAF, 1, &np))) { + return rc; + } + mc->mc_snum = 0; + mdb_cursor_push(mc, np); + mc->mc_db->md_root = np->mp_pgno; + mc->mc_db->md_depth++; + *mc->mc_dbflag |= DB_DIRTY; + if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) + == MDB_DUPFIXED) + np->mp_flags |= P_LEAF2; + mc->mc_flags |= C_INITIALIZED; + rc = MDB_NOTFOUND; + goto top; + } else { + int exact = 0; + MDB_val d2; + if (flags & MDB_APPEND) { + MDB_val k2; + rc = mdb_cursor_last(mc, &k2, &d2); + if (rc == 0) { + rc = mc->mc_dbx->md_cmp(key, &k2); + if (rc > 0) { + rc = MDB_NOTFOUND; + mc->mc_ki[mc->mc_top]++; + } else { + /* new key is <= last key */ + rc = MDB_KEYEXIST; + } + } + } else { + rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); + } + if ((flags & MDB_NOOVERWRITE) && rc == 0) { + DPRINTF("duplicate key [%s]", DKEY(key)); + *data = d2; + return MDB_KEYEXIST; + } + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + /* Cursor is positioned, now make sure all pages are writable */ + rc2 = mdb_cursor_touch(mc); + if (rc2) + return rc2; + +top: + /* The key already exists */ + if (rc == MDB_SUCCESS) { + /* there's only a key anyway, so this is a no-op */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + unsigned int ksize = mc->mc_db->md_pad; + if (key->mv_size != ksize) + return EINVAL; + if (flags == MDB_CURRENT) { + char *ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); + memcpy(ptr, key->mv_data, ksize); + } + return MDB_SUCCESS; + } + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + + /* DB has dups? */ + if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) { + /* Was a single item before, must convert now */ +more: + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + /* Just overwrite the current item */ + if (flags == MDB_CURRENT) + goto current; + + dkey.mv_size = NODEDSZ(leaf); + dkey.mv_data = NODEDATA(leaf); +#if UINT_MAX < SIZE_MAX + if (mc->mc_dbx->md_dcmp == mdb_cmp_int && dkey.mv_size == sizeof(size_t)) +#ifdef MISALIGNED_OK + mc->mc_dbx->md_dcmp = mdb_cmp_long; +#else + mc->mc_dbx->md_dcmp = mdb_cmp_cint; +#endif +#endif + /* if data matches, ignore it */ + if (!mc->mc_dbx->md_dcmp(data, &dkey)) + return (flags == MDB_NODUPDATA) ? MDB_KEYEXIST : MDB_SUCCESS; + + /* create a fake page for the dup items */ + memcpy(dbuf, dkey.mv_data, dkey.mv_size); + dkey.mv_data = dbuf; + fp = (MDB_page *)&pbuf; + fp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; + fp->mp_flags = P_LEAF|P_DIRTY|P_SUBP; + fp->mp_lower = PAGEHDRSZ; + fp->mp_upper = PAGEHDRSZ + dkey.mv_size + data->mv_size; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + fp->mp_flags |= P_LEAF2; + fp->mp_pad = data->mv_size; + fp->mp_upper += 2 * data->mv_size; /* leave space for 2 more */ + } else { + fp->mp_upper += 2 * sizeof(indx_t) + 2 * NODESIZE + + (dkey.mv_size & 1) + (data->mv_size & 1); + } + mdb_node_del(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], 0); + do_sub = 1; + rdata = &xdata; + xdata.mv_size = fp->mp_upper; + xdata.mv_data = fp; + flags |= F_DUPDATA; + goto new_sub; + } + if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { + /* See if we need to convert from fake page to subDB */ + MDB_page *mp; + unsigned int offset; + unsigned int i; + uint16_t fp_flags; + + fp = NODEDATA(leaf); + if (flags == MDB_CURRENT) { +reuse: + fp->mp_flags |= P_DIRTY; + COPY_PGNO(fp->mp_pgno, mc->mc_pg[mc->mc_top]->mp_pgno); + mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; + flags |= F_DUPDATA; + goto put_sub; + } + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + offset = fp->mp_pad; + if (SIZELEFT(fp) >= offset) + goto reuse; + offset *= 4; /* space for 4 more */ + } else { + offset = NODESIZE + sizeof(indx_t) + data->mv_size; + } + offset += offset & 1; + fp_flags = fp->mp_flags; + if (NODESIZE + sizeof(indx_t) + NODEKSZ(leaf) + NODEDSZ(leaf) + + offset >= mc->mc_txn->mt_env->me_nodemax) { + /* yes, convert it */ + dummy.md_flags = 0; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + dummy.md_pad = fp->mp_pad; + dummy.md_flags = MDB_DUPFIXED; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + dummy.md_flags |= MDB_INTEGERKEY; + } + dummy.md_depth = 1; + dummy.md_branch_pages = 0; + dummy.md_leaf_pages = 1; + dummy.md_overflow_pages = 0; + dummy.md_entries = NUMKEYS(fp); + rdata = &xdata; + xdata.mv_size = sizeof(MDB_db); + xdata.mv_data = &dummy; + if ((rc = mdb_page_alloc(mc, 1, &mp))) + return rc; + offset = mc->mc_txn->mt_env->me_psize - NODEDSZ(leaf); + flags |= F_DUPDATA|F_SUBDATA; + dummy.md_root = mp->mp_pgno; + fp_flags &= ~P_SUBP; + } else { + /* no, just grow it */ + rdata = &xdata; + xdata.mv_size = NODEDSZ(leaf) + offset; + xdata.mv_data = &pbuf; + mp = (MDB_page *)&pbuf; + mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; + flags |= F_DUPDATA; + } + mp->mp_flags = fp_flags | P_DIRTY; + mp->mp_pad = fp->mp_pad; + mp->mp_lower = fp->mp_lower; + mp->mp_upper = fp->mp_upper + offset; + if (IS_LEAF2(fp)) { + memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); + } else { + nsize = NODEDSZ(leaf) - fp->mp_upper; + memcpy((char *)mp + mp->mp_upper, (char *)fp + fp->mp_upper, nsize); + for (i=0; imp_ptrs[i] = fp->mp_ptrs[i] + offset; + } + mdb_node_del(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], 0); + do_sub = 1; + goto new_sub; + } + /* data is on sub-DB, just store it */ + flags |= F_DUPDATA|F_SUBDATA; + goto put_sub; + } +current: + /* overflow page overwrites need special handling */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + unsigned psize = mc->mc_txn->mt_env->me_psize; + int level, ovpages, dpages = OVPAGES(data->mv_size, psize); + + memcpy(&pg, NODEDATA(leaf), sizeof(pg)); + if ((rc2 = mdb_page_get(mc->mc_txn, pg, &omp, &level)) != 0) + return rc2; + ovpages = omp->mp_pages; + + /* Is the ov page writable and large enough? */ + if ((omp->mp_flags & P_DIRTY) && ovpages >= dpages) { + /* yes, overwrite it. Note in this case we don't + * bother to try shrinking the page if the new data + * is smaller than the overflow threshold. + */ + if (level > 1) { + /* It is writable only in a parent txn */ + size_t sz = (size_t) psize * ovpages, off; + MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages); + MDB_ID2 id2; + if (!np) + return ENOMEM; + id2.mid = pg; + id2.mptr = np; + mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); + if (!(flags & MDB_RESERVE)) { + /* Copy end of page, adjusting alignment so + * compiler may copy words instead of bytes. + */ + off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); + memcpy((size_t *)((char *)np + off), + (size_t *)((char *)omp + off), sz - off); + sz = PAGEHDRSZ; + } + memcpy(np, omp, sz); /* Copy beginning of page */ + omp = np; + } + SETDSZ(leaf, data->mv_size); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(omp); + else + memcpy(METADATA(omp), data->mv_data, data->mv_size); + goto done; + } else { + if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS) + return rc2; + } + } else if (NODEDSZ(leaf) == data->mv_size) { + /* same size, just replace it. Note that we could + * also reuse this node if the new data is smaller, + * but instead we opt to shrink the node in that case. + */ + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = NODEDATA(leaf); + else if (data->mv_size) + memcpy(NODEDATA(leaf), data->mv_data, data->mv_size); + else + memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); + goto done; + } + mdb_node_del(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], 0); + mc->mc_db->md_entries--; + } else { + DPRINTF("inserting key at index %i", mc->mc_ki[mc->mc_top]); + insert = 1; + } + + rdata = data; + +new_sub: + nflags = flags & NODE_ADD_FLAGS; + nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(mc->mc_txn->mt_env, key, rdata); + if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) { + if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA ) + nflags &= ~MDB_APPEND; + if (!insert) + nflags |= MDB_SPLIT_REPLACE; + rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags); + } else { + /* There is room already in this leaf page. */ + rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); + if (rc == 0 && !do_sub && insert) { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + if (mc->mc_flags & C_SUB) + dbi--; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[i] == mp && m3->mc_ki[i] >= mc->mc_ki[i]) { + m3->mc_ki[i]++; + } + } + } + } + + if (rc != MDB_SUCCESS) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + else { + /* Now store the actual data in the child DB. Note that we're + * storing the user data in the keys field, so there are strict + * size limits on dupdata. The actual data fields of the child + * DB are all zero size. + */ + if (do_sub) { + int xflags; +put_sub: + xdata.mv_size = 0; + xdata.mv_data = ""; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (flags & MDB_CURRENT) { + xflags = MDB_CURRENT; + } else { + mdb_xcursor_init1(mc, leaf); + xflags = (flags & MDB_NODUPDATA) ? MDB_NOOVERWRITE : 0; + } + /* converted, write the original data first */ + if (dkey.mv_size) { + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); + if (rc) + return rc; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { + mdb_xcursor_init1(m2, leaf); + } + } + } + /* we've done our job */ + dkey.mv_size = 0; + } + if (flags & MDB_APPENDDUP) + xflags |= MDB_APPEND; + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); + if (flags & F_SUBDATA) { + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } + } + /* sub-writes might have failed so check rc again. + * Don't increment count if we just replaced an existing item. + */ + if (!rc && !(flags & MDB_CURRENT)) + mc->mc_db->md_entries++; + if (flags & MDB_MULTIPLE) { + if (!rc) { + mcount++; + if (mcount < dcount) { + data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + goto more; + } + } + /* let caller know how many succeeded, if any */ + data[1].mv_size = mcount; + } + } +done: + /* If we succeeded and the key didn't exist before, make sure + * the cursor is marked valid. + */ + if (!rc && insert) + mc->mc_flags |= C_INITIALIZED; + return rc; +} + +int +mdb_cursor_del(MDB_cursor *mc, unsigned int flags) +{ + MDB_node *leaf; + int rc; + + if (F_ISSET(mc->mc_txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + rc = mdb_cursor_touch(mc); + if (rc) + return rc; + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + + if (!IS_LEAF2(mc->mc_pg[mc->mc_top]) && F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (flags != MDB_NODUPDATA) { + if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, 0); + /* If sub-DB still has entries, we're done */ + if (mc->mc_xcursor->mx_db.md_entries) { + if (leaf->mn_flags & F_SUBDATA) { + /* update subDB info */ + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } else { + MDB_cursor *m2; + /* shrink fake page */ + mdb_node_shrink(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + /* fix other sub-DB cursors pointed at this fake page */ + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (m2->mc_pg[mc->mc_top] == mc->mc_pg[mc->mc_top] && + m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + } + mc->mc_db->md_entries--; + return rc; + } + /* otherwise fall thru and delete the sub-DB */ + } + + if (leaf->mn_flags & F_SUBDATA) { + /* add all the child DB's pages to the free list */ + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc == MDB_SUCCESS) { + mc->mc_db->md_entries -= + mc->mc_xcursor->mx_db.md_entries; + } + } + } + + return mdb_cursor_del0(mc, leaf); +} + +/** Allocate and initialize new pages for a database. + * @param[in] mc a cursor on the database being added to. + * @param[in] flags flags defining what type of page is being allocated. + * @param[in] num the number of pages to allocate. This is usually 1, + * unless allocating overflow pages for a large record. + * @param[out] mp Address of a page, or NULL on failure. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) +{ + MDB_page *np; + int rc; + + if ((rc = mdb_page_alloc(mc, num, &np))) + return rc; + DPRINTF("allocated new mpage %zu, page size %u", + np->mp_pgno, mc->mc_txn->mt_env->me_psize); + np->mp_flags = flags | P_DIRTY; + np->mp_lower = PAGEHDRSZ; + np->mp_upper = mc->mc_txn->mt_env->me_psize; + + if (IS_BRANCH(np)) + mc->mc_db->md_branch_pages++; + else if (IS_LEAF(np)) + mc->mc_db->md_leaf_pages++; + else if (IS_OVERFLOW(np)) { + mc->mc_db->md_overflow_pages += num; + np->mp_pages = num; + } + *mp = np; + + return 0; +} + +/** Calculate the size of a leaf node. + * The size depends on the environment's page size; if a data item + * is too large it will be put onto an overflow page and the node + * size will only include the key and not the data. Sizes are always + * rounded up to an even number of bytes, to guarantee 2-byte alignment + * of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @param[in] data The data for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) +{ + size_t sz; + + sz = LEAFSIZE(key, data); + if (sz >= env->me_nodemax) { + /* put on overflow page */ + sz -= data->mv_size - sizeof(pgno_t); + } + sz += sz & 1; + + return sz + sizeof(indx_t); +} + +/** Calculate the size of a branch node. + * The size should depend on the environment's page size but since + * we currently don't support spilling large keys onto overflow + * pages, it's simply the size of the #MDB_node header plus the + * size of the key. Sizes are always rounded up to an even number + * of bytes, to guarantee 2-byte alignment of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_branch_size(MDB_env *env, MDB_val *key) +{ + size_t sz; + + sz = INDXSIZE(key); + if (sz >= env->me_nodemax) { + /* put on overflow page */ + /* not implemented */ + /* sz -= key->size - sizeof(pgno_t); */ + } + + return sz + sizeof(indx_t); +} + +/** Add a node to the page pointed to by the cursor. + * @param[in] mc The cursor for this operation. + * @param[in] indx The index on the page where the new node should be added. + * @param[in] key The key for the new node. + * @param[in] data The data for the new node, if any. + * @param[in] pgno The page number, if adding a branch node. + * @param[in] flags Flags for the node. + * @return 0 on success, non-zero on failure. Possible errors are: + *
    + *
  • ENOMEM - failed to allocate overflow pages for the node. + *
  • MDB_PAGE_FULL - there is insufficient room in the page. This error + * should never happen since all callers already calculate the + * page's free space before calling this function. + *
+ */ +static int +mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags) +{ + unsigned int i; + size_t node_size = NODESIZE; + indx_t ofs; + MDB_node *node; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_page *ofp = NULL; /* overflow page */ + DKBUF; + + assert(mp->mp_upper >= mp->mp_lower); + + DPRINTF("add to %s %spage %zu index %i, data size %zu key size %zu [%s]", + IS_LEAF(mp) ? "leaf" : "branch", + IS_SUBP(mp) ? "sub-" : "", + mp->mp_pgno, indx, data ? data->mv_size : 0, + key ? key->mv_size : 0, key ? DKEY(key) : NULL); + + if (IS_LEAF2(mp)) { + /* Move higher keys up one slot. */ + int ksize = mc->mc_db->md_pad, dif; + char *ptr = LEAF2KEY(mp, indx, ksize); + dif = NUMKEYS(mp) - indx; + if (dif > 0) + memmove(ptr+ksize, ptr, dif*ksize); + /* insert new key */ + memcpy(ptr, key->mv_data, ksize); + + /* Just using these for counting */ + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + return MDB_SUCCESS; + } + + if (key != NULL) + node_size += key->mv_size; + + if (IS_LEAF(mp)) { + assert(data); + if (F_ISSET(flags, F_BIGDATA)) { + /* Data already on overflow page. */ + node_size += sizeof(pgno_t); + } else if (node_size + data->mv_size >= mc->mc_txn->mt_env->me_nodemax) { + int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize); + int rc; + /* Put data on overflow page. */ + DPRINTF("data size is %zu, node would be %zu, put data on overflow page", + data->mv_size, node_size+data->mv_size); + node_size += sizeof(pgno_t); + if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) + return rc; + DPRINTF("allocated overflow page %zu", ofp->mp_pgno); + flags |= F_BIGDATA; + } else { + node_size += data->mv_size; + } + } + node_size += node_size & 1; + + if (node_size + sizeof(indx_t) > SIZELEFT(mp)) { + DPRINTF("not enough room in page %zu, got %u ptrs", + mp->mp_pgno, NUMKEYS(mp)); + DPRINTF("upper - lower = %u - %u = %u", mp->mp_upper, mp->mp_lower, + mp->mp_upper - mp->mp_lower); + DPRINTF("node size = %zu", node_size); + return MDB_PAGE_FULL; + } + + /* Move higher pointers up one slot. */ + for (i = NUMKEYS(mp); i > indx; i--) + mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; + + /* Adjust free space offsets. */ + ofs = mp->mp_upper - node_size; + assert(ofs >= mp->mp_lower + sizeof(indx_t)); + mp->mp_ptrs[indx] = ofs; + mp->mp_upper = ofs; + mp->mp_lower += sizeof(indx_t); + + /* Write the node data. */ + node = NODEPTR(mp, indx); + node->mn_ksize = (key == NULL) ? 0 : key->mv_size; + node->mn_flags = flags; + if (IS_LEAF(mp)) + SETDSZ(node,data->mv_size); + else + SETPGNO(node,pgno); + + if (key) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + if (IS_LEAF(mp)) { + assert(key); + if (ofp == NULL) { + if (F_ISSET(flags, F_BIGDATA)) + memcpy(node->mn_data + key->mv_size, data->mv_data, + sizeof(pgno_t)); + else if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = node->mn_data + key->mv_size; + else + memcpy(node->mn_data + key->mv_size, data->mv_data, + data->mv_size); + } else { + memcpy(node->mn_data + key->mv_size, &ofp->mp_pgno, + sizeof(pgno_t)); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(ofp); + else + memcpy(METADATA(ofp), data->mv_data, data->mv_size); + } + } + + return MDB_SUCCESS; +} + +/** Delete the specified node from a page. + * @param[in] mp The page to operate on. + * @param[in] indx The index of the node to delete. + * @param[in] ksize The size of a node. Only used if the page is + * part of a #MDB_DUPFIXED database. + */ +static void +mdb_node_del(MDB_page *mp, indx_t indx, int ksize) +{ + unsigned int sz; + indx_t i, j, numkeys, ptr; + MDB_node *node; + char *base; + +#if MDB_DEBUG + { + pgno_t pgno; + COPY_PGNO(pgno, mp->mp_pgno); + DPRINTF("delete node %u on %s page %zu", indx, + IS_LEAF(mp) ? "leaf" : "branch", pgno); + } +#endif + assert(indx < NUMKEYS(mp)); + + if (IS_LEAF2(mp)) { + int x = NUMKEYS(mp) - 1 - indx; + base = LEAF2KEY(mp, indx, ksize); + if (x) + memmove(base, base + ksize, x * ksize); + mp->mp_lower -= sizeof(indx_t); + mp->mp_upper += ksize - sizeof(indx_t); + return; + } + + node = NODEPTR(mp, indx); + sz = NODESIZE + node->mn_ksize; + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + sz += sizeof(pgno_t); + else + sz += NODEDSZ(node); + } + sz += sz & 1; + + ptr = mp->mp_ptrs[indx]; + numkeys = NUMKEYS(mp); + for (i = j = 0; i < numkeys; i++) { + if (i != indx) { + mp->mp_ptrs[j] = mp->mp_ptrs[i]; + if (mp->mp_ptrs[i] < ptr) + mp->mp_ptrs[j] += sz; + j++; + } + } + + base = (char *)mp + mp->mp_upper; + memmove(base + sz, base, ptr - mp->mp_upper); + + mp->mp_lower -= sizeof(indx_t); + mp->mp_upper += sz; +} + +/** Compact the main page after deleting a node on a subpage. + * @param[in] mp The main page to operate on. + * @param[in] indx The index of the subpage on the main page. + */ +static void +mdb_node_shrink(MDB_page *mp, indx_t indx) +{ + MDB_node *node; + MDB_page *sp, *xp; + char *base; + int osize, nsize; + int delta; + indx_t i, numkeys, ptr; + + node = NODEPTR(mp, indx); + sp = (MDB_page *)NODEDATA(node); + osize = NODEDSZ(node); + + delta = sp->mp_upper - sp->mp_lower; + SETDSZ(node, osize - delta); + xp = (MDB_page *)((char *)sp + delta); + + /* shift subpage upward */ + if (IS_LEAF2(sp)) { + nsize = NUMKEYS(sp) * sp->mp_pad; + memmove(METADATA(xp), METADATA(sp), nsize); + } else { + int i; + nsize = osize - sp->mp_upper; + numkeys = NUMKEYS(sp); + for (i=numkeys-1; i>=0; i--) + xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta; + } + xp->mp_upper = sp->mp_lower; + xp->mp_lower = sp->mp_lower; + xp->mp_flags = sp->mp_flags; + xp->mp_pad = sp->mp_pad; + COPY_PGNO(xp->mp_pgno, mp->mp_pgno); + + /* shift lower nodes upward */ + ptr = mp->mp_ptrs[indx]; + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] += delta; + } + + base = (char *)mp + mp->mp_upper; + memmove(base + delta, base, ptr - mp->mp_upper + NODESIZE + NODEKSZ(node)); + mp->mp_upper += delta; +} + +/** Initial setup of a sorted-dups cursor. + * Sorted duplicates are implemented as a sub-database for the given key. + * The duplicate data items are actually keys of the sub-database. + * Operations on the duplicate data items are performed using a sub-cursor + * initialized when the sub-database is first accessed. This function does + * the preliminary setup of the sub-cursor, filling in the fields that + * depend only on the parent DB. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + */ +static void +mdb_xcursor_init0(MDB_cursor *mc) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + mx->mx_cursor.mc_xcursor = NULL; + mx->mx_cursor.mc_txn = mc->mc_txn; + mx->mx_cursor.mc_db = &mx->mx_db; + mx->mx_cursor.mc_dbx = &mx->mx_dbx; + mx->mx_cursor.mc_dbi = mc->mc_dbi+1; + mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; + mx->mx_dbx.md_dcmp = NULL; + mx->mx_dbx.md_rel = mc->mc_dbx->md_rel; +} + +/** Final setup of a sorted-dups cursor. + * Sets up the fields that depend on the data from the main cursor. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + * @param[in] node The data containing the #MDB_db record for the + * sorted-dup database. + */ +static void +mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + if (node->mn_flags & F_SUBDATA) { + memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); + mx->mx_cursor.mc_pg[0] = 0; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_flags = C_SUB; + } else { + MDB_page *fp = NODEDATA(node); + mx->mx_db.md_pad = mc->mc_pg[mc->mc_top]->mp_pad; + mx->mx_db.md_flags = 0; + mx->mx_db.md_depth = 1; + mx->mx_db.md_branch_pages = 0; + mx->mx_db.md_leaf_pages = 1; + mx->mx_db.md_overflow_pages = 0; + mx->mx_db.md_entries = NUMKEYS(fp); + COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_pg[0] = fp; + mx->mx_cursor.mc_ki[0] = 0; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + mx->mx_db.md_flags = MDB_DUPFIXED; + mx->mx_db.md_pad = fp->mp_pad; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + mx->mx_db.md_flags |= MDB_INTEGERKEY; + } + } + DPRINTF("Sub-db %u for db %u root page %zu", mx->mx_cursor.mc_dbi, mc->mc_dbi, + mx->mx_db.md_root); + mx->mx_dbflag = DB_VALID | (F_ISSET(mc->mc_pg[mc->mc_top]->mp_flags, P_DIRTY) ? + DB_DIRTY : 0); + mx->mx_dbx.md_name.mv_data = NODEKEY(node); + mx->mx_dbx.md_name.mv_size = node->mn_ksize; +#if UINT_MAX < SIZE_MAX + if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) +#ifdef MISALIGNED_OK + mx->mx_dbx.md_cmp = mdb_cmp_long; +#else + mx->mx_dbx.md_cmp = mdb_cmp_cint; +#endif +#endif +} + +/** Initialize a cursor for a given transaction and database. */ +static void +mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) +{ + mc->mc_backup = NULL; + mc->mc_dbi = dbi; + mc->mc_txn = txn; + mc->mc_db = &txn->mt_dbs[dbi]; + mc->mc_dbx = &txn->mt_dbxs[dbi]; + mc->mc_dbflag = &txn->mt_dbflags[dbi]; + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_pg[0] = 0; + mc->mc_flags = 0; + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { + assert(mx != NULL); + mc->mc_xcursor = mx; + mdb_xcursor_init0(mc); + } else { + mc->mc_xcursor = NULL; + } + if (*mc->mc_dbflag & DB_STALE) { + mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); + } +} + +int +mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) +{ + MDB_cursor *mc; + size_t size = sizeof(MDB_cursor); + + if (txn == NULL || ret == NULL || dbi >= txn->mt_numdbs || !(txn->mt_dbflags[dbi] & DB_VALID)) + return EINVAL; + + /* Allow read access to the freelist */ + if (!dbi && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EINVAL; + + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) + size += sizeof(MDB_xcursor); + + if ((mc = malloc(size)) != NULL) { + mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1)); + if (txn->mt_cursors) { + mc->mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = mc; + mc->mc_flags |= C_UNTRACK; + } + } else { + return ENOMEM; + } + + *ret = mc; + + return MDB_SUCCESS; +} + +int +mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) +{ + if (txn == NULL || mc == NULL || mc->mc_dbi >= txn->mt_numdbs) + return EINVAL; + + if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) + return EINVAL; + + mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor); + return MDB_SUCCESS; +} + +/* Return the count of duplicate data items for the current key */ +int +mdb_cursor_count(MDB_cursor *mc, size_t *countp) +{ + MDB_node *leaf; + + if (mc == NULL || countp == NULL) + return EINVAL; + + if (!(mc->mc_db->md_flags & MDB_DUPSORT)) + return EINVAL; + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + *countp = 1; + } else { + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + return EINVAL; + + *countp = mc->mc_xcursor->mx_db.md_entries; + } + return MDB_SUCCESS; +} + +void +mdb_cursor_close(MDB_cursor *mc) +{ + if (mc && !mc->mc_backup) { + /* remove from txn, if tracked */ + if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { + MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; + while (*prev && *prev != mc) prev = &(*prev)->mc_next; + if (*prev == mc) + *prev = mc->mc_next; + } + free(mc); + } +} + +MDB_txn * +mdb_cursor_txn(MDB_cursor *mc) +{ + if (!mc) return NULL; + return mc->mc_txn; +} + +MDB_dbi +mdb_cursor_dbi(MDB_cursor *mc) +{ + assert(mc != NULL); + return mc->mc_dbi; +} + +/** Replace the key for a node with a new key. + * @param[in] mc Cursor pointing to the node to operate on. + * @param[in] key The new key to use. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_update_key(MDB_cursor *mc, MDB_val *key) +{ + MDB_page *mp; + MDB_node *node; + char *base; + size_t len; + int delta, delta0; + indx_t ptr, i, numkeys, indx; + DKBUF; + + indx = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + node = NODEPTR(mp, indx); + ptr = mp->mp_ptrs[indx]; +#if MDB_DEBUG + { + MDB_val k2; + char kbuf2[(MDB_MAXKEYSIZE*2+1)]; + k2.mv_data = NODEKEY(node); + k2.mv_size = node->mn_ksize; + DPRINTF("update key %u (ofs %u) [%s] to [%s] on page %zu", + indx, ptr, + mdb_dkey(&k2, kbuf2), + DKEY(key), + mp->mp_pgno); + } +#endif + + delta0 = delta = key->mv_size - node->mn_ksize; + + /* Must be 2-byte aligned. If new key is + * shorter by 1, the shift will be skipped. + */ + delta += (delta & 1); + if (delta) { + if (delta > 0 && SIZELEFT(mp) < delta) { + pgno_t pgno; + /* not enough space left, do a delete and split */ + DPRINTF("Not enough room, delta = %d, splitting...", delta); + pgno = NODEPGNO(node); + mdb_node_del(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], 0); + return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE); + } + + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] -= delta; + } + + base = (char *)mp + mp->mp_upper; + len = ptr - mp->mp_upper + NODESIZE; + memmove(base - delta, base, len); + mp->mp_upper -= delta; + + node = NODEPTR(mp, indx); + } + + /* But even if no shift was needed, update ksize */ + if (delta0) + node->mn_ksize = key->mv_size; + + if (key->mv_size) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + return MDB_SUCCESS; +} + +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); + +/** Move a node from csrc to cdst. + */ +static int +mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) +{ + MDB_node *srcnode; + MDB_val key, data; + pgno_t srcpg; + MDB_cursor mn; + int rc; + unsigned short flags; + + DKBUF; + + /* Mark src and dst as dirty. */ + if ((rc = mdb_page_touch(csrc)) || + (rc = mdb_page_touch(cdst))) + return rc; + + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); /* fake */ + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); + data.mv_size = 0; + data.mv_data = NULL; + srcpg = 0; + flags = 0; + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]); + assert(!((long)srcnode&1)); + srcpg = NODEPGNO(srcnode); + flags = srcnode->mn_flags; + if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + unsigned int snum = csrc->mc_snum; + MDB_node *s2; + /* must find the lowest key below src */ + mdb_page_search_lowest(csrc); + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + csrc->mc_snum = snum--; + csrc->mc_top = snum; + } else { + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { + unsigned int snum = cdst->mc_snum; + MDB_node *s2; + MDB_val bkey; + /* must find the lowest key below dst */ + mdb_page_search_lowest(cdst); + if (IS_LEAF2(cdst->mc_pg[cdst->mc_top])) { + bkey.mv_size = cdst->mc_db->md_pad; + bkey.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, bkey.mv_size); + } else { + s2 = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); + bkey.mv_size = NODEKSZ(s2); + bkey.mv_data = NODEKEY(s2); + } + cdst->mc_snum = snum--; + cdst->mc_top = snum; + mdb_cursor_copy(cdst, &mn); + mn.mc_ki[snum] = 0; + rc = mdb_update_key(&mn, &bkey); + if (rc) + return rc; + } + + DPRINTF("moving %s node %u [%s] on page %zu to node %u on page %zu", + IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", + csrc->mc_ki[csrc->mc_top], + DKEY(&key), + csrc->mc_pg[csrc->mc_top]->mp_pgno, + cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno); + + /* Add the node to the destination page. + */ + rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags); + if (rc != MDB_SUCCESS) + return rc; + + /* Delete the node from the source page. + */ + mdb_node_del(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + MDB_page *mp = csrc->mc_pg[csrc->mc_top]; + + if (csrc->mc_flags & C_SUB) + dbi--; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] == + csrc->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + } + } + } + + /* Update the parent separators. + */ + if (csrc->mc_ki[csrc->mc_top] == 0) { + if (csrc->mc_ki[csrc->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF("update separator for source page %zu to [%s]", + csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key)); + mdb_cursor_copy(csrc, &mn); + mn.mc_snum--; + mn.mc_top--; + if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) + return rc; + } + if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + MDB_val nullkey; + indx_t ix = csrc->mc_ki[csrc->mc_top]; + nullkey.mv_size = 0; + csrc->mc_ki[csrc->mc_top] = 0; + rc = mdb_update_key(csrc, &nullkey); + csrc->mc_ki[csrc->mc_top] = ix; + assert(rc == MDB_SUCCESS); + } + } + + if (cdst->mc_ki[cdst->mc_top] == 0) { + if (cdst->mc_ki[cdst->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF("update separator for destination page %zu to [%s]", + cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key)); + mdb_cursor_copy(cdst, &mn); + mn.mc_snum--; + mn.mc_top--; + if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) + return rc; + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) { + MDB_val nullkey; + indx_t ix = cdst->mc_ki[cdst->mc_top]; + nullkey.mv_size = 0; + cdst->mc_ki[cdst->mc_top] = 0; + rc = mdb_update_key(cdst, &nullkey); + cdst->mc_ki[cdst->mc_top] = ix; + assert(rc == MDB_SUCCESS); + } + } + + return MDB_SUCCESS; +} + +/** Merge one page into another. + * The nodes from the page pointed to by \b csrc will + * be copied to the page pointed to by \b cdst and then + * the \b csrc page will be freed. + * @param[in] csrc Cursor pointing to the source page. + * @param[in] cdst Cursor pointing to the destination page. + */ +static int +mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) +{ + int rc; + indx_t i, j; + MDB_node *srcnode; + MDB_val key, data; + unsigned nkeys; + + DPRINTF("merging page %zu into %zu", csrc->mc_pg[csrc->mc_top]->mp_pgno, + cdst->mc_pg[cdst->mc_top]->mp_pgno); + + assert(csrc->mc_snum > 1); /* can't merge root page */ + assert(cdst->mc_snum > 1); + + /* Mark dst as dirty. */ + if ((rc = mdb_page_touch(cdst))) + return rc; + + /* Move all nodes from src to dst. + */ + j = nkeys = NUMKEYS(cdst->mc_pg[cdst->mc_top]); + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = METADATA(csrc->mc_pg[csrc->mc_top]); + for (i = 0; i < NUMKEYS(csrc->mc_pg[csrc->mc_top]); i++, j++) { + rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); + if (rc != MDB_SUCCESS) + return rc; + key.mv_data = (char *)key.mv_data + key.mv_size; + } + } else { + for (i = 0; i < NUMKEYS(csrc->mc_pg[csrc->mc_top]); i++, j++) { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], i); + if (i == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + unsigned int snum = csrc->mc_snum; + MDB_node *s2; + /* must find the lowest key below src */ + mdb_page_search_lowest(csrc); + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + csrc->mc_snum = snum--; + csrc->mc_top = snum; + } else { + key.mv_size = srcnode->mn_ksize; + key.mv_data = NODEKEY(srcnode); + } + + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags); + if (rc != MDB_SUCCESS) + return rc; + } + } + + DPRINTF("dst page %zu now has %u keys (%.1f%% filled)", + cdst->mc_pg[cdst->mc_top]->mp_pgno, NUMKEYS(cdst->mc_pg[cdst->mc_top]), (float)PAGEFILL(cdst->mc_txn->mt_env, cdst->mc_pg[cdst->mc_top]) / 10); + + /* Unlink the src page from parent and add to free list. + */ + mdb_node_del(csrc->mc_pg[csrc->mc_top-1], csrc->mc_ki[csrc->mc_top-1], 0); + if (csrc->mc_ki[csrc->mc_top-1] == 0) { + key.mv_size = 0; + csrc->mc_top--; + rc = mdb_update_key(csrc, &key); + csrc->mc_top++; + if (rc) + return rc; + } + + rc = mdb_midl_append(&csrc->mc_txn->mt_free_pgs, + csrc->mc_pg[csrc->mc_top]->mp_pgno); + if (rc) + return rc; + if (IS_LEAF(csrc->mc_pg[csrc->mc_top])) + csrc->mc_db->md_leaf_pages--; + else + csrc->mc_db->md_branch_pages--; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + MDB_page *mp = cdst->mc_pg[cdst->mc_top]; + + if (csrc->mc_flags & C_SUB) + dbi--; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_snum < csrc->mc_snum) continue; + if (m3->mc_pg[csrc->mc_top] == csrc->mc_pg[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = mp; + m3->mc_ki[csrc->mc_top] += nkeys; + } + } + } + mdb_cursor_pop(csrc); + + return mdb_rebalance(csrc); +} + +/** Copy the contents of a cursor. + * @param[in] csrc The cursor to copy from. + * @param[out] cdst The cursor to copy to. + */ +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) +{ + unsigned int i; + + cdst->mc_txn = csrc->mc_txn; + cdst->mc_dbi = csrc->mc_dbi; + cdst->mc_db = csrc->mc_db; + cdst->mc_dbx = csrc->mc_dbx; + cdst->mc_snum = csrc->mc_snum; + cdst->mc_top = csrc->mc_top; + cdst->mc_flags = csrc->mc_flags; + + for (i=0; imc_snum; i++) { + cdst->mc_pg[i] = csrc->mc_pg[i]; + cdst->mc_ki[i] = csrc->mc_ki[i]; + } +} + +/** Rebalance the tree after a delete operation. + * @param[in] mc Cursor pointing to the page where rebalancing + * should begin. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_rebalance(MDB_cursor *mc) +{ + MDB_node *node; + int rc; + unsigned int ptop, minkeys; + MDB_cursor mn; + + minkeys = 1 + (IS_BRANCH(mc->mc_pg[mc->mc_top])); +#if MDB_DEBUG + { + pgno_t pgno; + COPY_PGNO(pgno, mc->mc_pg[mc->mc_top]->mp_pgno); + DPRINTF("rebalancing %s page %zu (has %u keys, %.1f%% full)", + IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", + pgno, NUMKEYS(mc->mc_pg[mc->mc_top]), (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10); + } +#endif + + if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= FILL_THRESHOLD && + NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { +#if MDB_DEBUG + pgno_t pgno; + COPY_PGNO(pgno, mc->mc_pg[mc->mc_top]->mp_pgno); + DPRINTF("no need to rebalance page %zu, above fill threshold", + pgno); +#endif + return MDB_SUCCESS; + } + + if (mc->mc_snum < 2) { + MDB_page *mp = mc->mc_pg[0]; + if (IS_SUBP(mp)) { + DPUTS("Can't rebalance a subpage, ignoring"); + return MDB_SUCCESS; + } + if (NUMKEYS(mp) == 0) { + DPUTS("tree is completely empty"); + mc->mc_db->md_root = P_INVALID; + mc->mc_db->md_depth = 0; + mc->mc_db->md_leaf_pages = 0; + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + /* Adjust cursors pointing to mp */ + mc->mc_snum = 0; + mc->mc_top = 0; + { + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + if (mc->mc_flags & C_SUB) + dbi--; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum = 0; + m3->mc_top = 0; + } + } + } + } else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) { + DPUTS("collapsing root page!"); + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0)); + rc = mdb_page_get(mc->mc_txn,mc->mc_db->md_root,&mc->mc_pg[0],NULL); + if (rc) + return rc; + mc->mc_db->md_depth--; + mc->mc_db->md_branch_pages--; + mc->mc_ki[0] = mc->mc_ki[1]; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + if (mc->mc_flags & C_SUB) + dbi--; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[0] == mp) { + m3->mc_pg[0] = mc->mc_pg[0]; + m3->mc_snum = 1; + m3->mc_top = 0; + m3->mc_ki[0] = m3->mc_ki[1]; + } + } + } + } else + DPUTS("root page doesn't need rebalancing"); + return MDB_SUCCESS; + } + + /* The parent (branch page) must have at least 2 pointers, + * otherwise the tree is invalid. + */ + ptop = mc->mc_top-1; + assert(NUMKEYS(mc->mc_pg[ptop]) > 1); + + /* Leaf page fill factor is below the threshold. + * Try to move keys from left or right neighbor, or + * merge with a neighbor page. + */ + + /* Find neighbors. + */ + mdb_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + + if (mc->mc_ki[ptop] == 0) { + /* We're the leftmost leaf in our parent. + */ + DPUTS("reading right neighbor"); + mn.mc_ki[ptop]++; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = 0; + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + } else { + /* There is at least one neighbor to the left. + */ + DPUTS("reading left neighbor"); + mn.mc_ki[ptop]--; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1; + mc->mc_ki[mc->mc_top] = 0; + } + + DPRINTF("found neighbor page %zu (%u keys, %.1f%% full)", + mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10); + + /* If the neighbor page is above threshold and has enough keys, + * move one key from it. Otherwise we should try to merge them. + * (A branch page must never have less than 2 keys.) + */ + minkeys = 1 + (IS_BRANCH(mn.mc_pg[mn.mc_top])); + if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= FILL_THRESHOLD && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) + return mdb_node_move(&mn, mc); + else { + if (mc->mc_ki[ptop] == 0) + rc = mdb_page_merge(&mn, mc); + else + rc = mdb_page_merge(mc, &mn); + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + } + return rc; +} + +/** Complete a delete operation started by #mdb_cursor_del(). */ +static int +mdb_cursor_del0(MDB_cursor *mc, MDB_node *leaf) +{ + int rc; + MDB_page *mp; + indx_t ki; + + mp = mc->mc_pg[mc->mc_top]; + ki = mc->mc_ki[mc->mc_top]; + + /* add overflow pages to free list */ + if (!IS_LEAF2(mp) && F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + + memcpy(&pg, NODEDATA(leaf), sizeof(pg)); + if ((rc = mdb_page_get(mc->mc_txn, pg, &omp, NULL)) || + (rc = mdb_ovpage_free(mc, omp))) + return rc; + } + mdb_node_del(mp, ki, mc->mc_db->md_pad); + mc->mc_db->md_entries--; + rc = mdb_rebalance(mc); + if (rc != MDB_SUCCESS) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + /* if mc points past last node in page, invalidate */ + else if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2; + unsigned int nkeys; + MDB_dbi dbi = mc->mc_dbi; + + mp = mc->mc_pg[mc->mc_top]; + nkeys = NUMKEYS(mp); + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (m2 == mc) + continue; + if (!(m2->mc_flags & C_INITIALIZED)) + continue; + if (m2->mc_pg[mc->mc_top] == mp) { + if (m2->mc_ki[mc->mc_top] > ki) + m2->mc_ki[mc->mc_top]--; + if (m2->mc_ki[mc->mc_top] >= nkeys) + m2->mc_flags &= ~(C_INITIALIZED|C_EOF); + } + } + } + + return rc; +} + +int +mdb_del(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + MDB_cursor mc; + MDB_xcursor mx; + MDB_cursor_op op; + MDB_val rdata, *xdata; + int rc, exact; + DKBUF; + + assert(key != NULL); + + DPRINTF("====> delete db %u key [%s]", dbi, DKEY(key)); + + if (txn == NULL || !dbi || dbi >= txn->mt_numdbs || !(txn->mt_dbflags[dbi] & DB_VALID)) + return EINVAL; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + return EACCES; + } + + if (key->mv_size == 0 || key->mv_size > MDB_MAXKEYSIZE) { + return EINVAL; + } + + mdb_cursor_init(&mc, txn, dbi, &mx); + + exact = 0; + if (data) { + op = MDB_GET_BOTH; + rdata = *data; + xdata = &rdata; + } else { + op = MDB_SET; + xdata = NULL; + } + rc = mdb_cursor_set(&mc, key, xdata, op, &exact); + if (rc == 0) { + /* let mdb_page_split know about this cursor if needed: + * delete will trigger a rebalance; if it needs to move + * a node from one page to another, it will have to + * update the parent's separator key(s). If the new sepkey + * is larger than the current one, the parent page may + * run out of space, triggering a split. We need this + * cursor to be consistent until the end of the rebalance. + */ + mc.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &mc; + rc = mdb_cursor_del(&mc, data ? 0 : MDB_NODUPDATA); + txn->mt_cursors[dbi] = mc.mc_next; + } + return rc; +} + +/** Split a page and insert a new node. + * @param[in,out] mc Cursor pointing to the page and desired insertion index. + * The cursor will be updated to point to the actual page and index where + * the node got inserted after the split. + * @param[in] newkey The key for the newly inserted node. + * @param[in] newdata The data for the newly inserted node. + * @param[in] newpgno The page number, if the new node is a branch node. + * @param[in] nflags The #NODE_ADD_FLAGS for the new node. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, + unsigned int nflags) +{ + unsigned int flags; + int rc = MDB_SUCCESS, ins_new = 0, new_root = 0, newpos = 1, did_split = 0; + indx_t newindx; + pgno_t pgno = 0; + unsigned int i, j, split_indx, nkeys, pmax; + MDB_node *node; + MDB_val sepkey, rkey, xdata, *rdata = &xdata; + MDB_page *copy; + MDB_page *mp, *rp, *pp; + unsigned int ptop; + MDB_cursor mn; + DKBUF; + + mp = mc->mc_pg[mc->mc_top]; + newindx = mc->mc_ki[mc->mc_top]; + + DPRINTF("-----> splitting %s page %zu and adding [%s] at index %i", + IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, + DKEY(newkey), mc->mc_ki[mc->mc_top]); + + /* Create a right sibling. */ + if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) + return rc; + DPRINTF("new right sibling: page %zu", rp->mp_pgno); + + if (mc->mc_snum < 2) { + if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp))) + return rc; + /* shift current top to make room for new parent */ + mc->mc_pg[1] = mc->mc_pg[0]; + mc->mc_ki[1] = mc->mc_ki[0]; + mc->mc_pg[0] = pp; + mc->mc_ki[0] = 0; + mc->mc_db->md_root = pp->mp_pgno; + DPRINTF("root split! new root = %zu", pp->mp_pgno); + mc->mc_db->md_depth++; + new_root = 1; + + /* Add left (implicit) pointer. */ + if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { + /* undo the pre-push */ + mc->mc_pg[0] = mc->mc_pg[1]; + mc->mc_ki[0] = mc->mc_ki[1]; + mc->mc_db->md_root = mp->mp_pgno; + mc->mc_db->md_depth--; + return rc; + } + mc->mc_snum = 2; + mc->mc_top = 1; + ptop = 0; + } else { + ptop = mc->mc_top-1; + DPRINTF("parent branch page is %zu", mc->mc_pg[ptop]->mp_pgno); + } + + mc->mc_flags |= C_SPLITTING; + mdb_cursor_copy(mc, &mn); + mn.mc_pg[mn.mc_top] = rp; + mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; + + if (nflags & MDB_APPEND) { + mn.mc_ki[mn.mc_top] = 0; + sepkey = *newkey; + split_indx = newindx; + nkeys = 0; + goto newsep; + } + + nkeys = NUMKEYS(mp); + split_indx = nkeys / 2; + if (newindx < split_indx) + newpos = 0; + + if (IS_LEAF2(rp)) { + char *split, *ins; + int x; + unsigned int lsize, rsize, ksize; + /* Move half of the keys to the right sibling */ + copy = NULL; + x = mc->mc_ki[mc->mc_top] - split_indx; + ksize = mc->mc_db->md_pad; + split = LEAF2KEY(mp, split_indx, ksize); + rsize = (nkeys - split_indx) * ksize; + lsize = (nkeys - split_indx) * sizeof(indx_t); + mp->mp_lower -= lsize; + rp->mp_lower += lsize; + mp->mp_upper += rsize - lsize; + rp->mp_upper -= rsize - lsize; + sepkey.mv_size = ksize; + if (newindx == split_indx) { + sepkey.mv_data = newkey->mv_data; + } else { + sepkey.mv_data = split; + } + if (x<0) { + ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize); + memcpy(rp->mp_ptrs, split, rsize); + sepkey.mv_data = rp->mp_ptrs; + memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); + memcpy(ins, newkey->mv_data, ksize); + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + } else { + if (x) + memcpy(rp->mp_ptrs, split, x * ksize); + ins = LEAF2KEY(rp, x, ksize); + memcpy(ins, newkey->mv_data, ksize); + memcpy(ins+ksize, split + x * ksize, rsize - x * ksize); + rp->mp_lower += sizeof(indx_t); + rp->mp_upper -= ksize - sizeof(indx_t); + mc->mc_ki[mc->mc_top] = x; + mc->mc_pg[mc->mc_top] = rp; + } + goto newsep; + } + + /* For leaf pages, check the split point based on what + * fits where, since otherwise mdb_node_add can fail. + * + * This check is only needed when the data items are + * relatively large, such that being off by one will + * make the difference between success or failure. + * + * It's also relevant if a page happens to be laid out + * such that one half of its nodes are all "small" and + * the other half of its nodes are "large." If the new + * item is also "large" and falls on the half with + * "large" nodes, it also may not fit. + */ + if (IS_LEAF(mp)) { + unsigned int psize, nsize; + /* Maximum free space in an empty page */ + pmax = mc->mc_txn->mt_env->me_psize - PAGEHDRSZ; + nsize = mdb_leaf_size(mc->mc_txn->mt_env, newkey, newdata); + if ((nkeys < 20) || (nsize > pmax/16)) { + if (newindx <= split_indx) { + psize = nsize; + newpos = 0; + for (i=0; imn_flags, F_BIGDATA)) + psize += sizeof(pgno_t); + else + psize += NODEDSZ(node); + psize += psize & 1; + if (psize > pmax) { + if (i <= newindx) { + split_indx = newindx; + if (i < newindx) + newpos = 1; + } + else + split_indx = i; + break; + } + } + } else { + psize = nsize; + for (i=nkeys-1; i>=split_indx; i--) { + node = NODEPTR(mp, i); + psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t); + if (F_ISSET(node->mn_flags, F_BIGDATA)) + psize += sizeof(pgno_t); + else + psize += NODEDSZ(node); + psize += psize & 1; + if (psize > pmax) { + if (i >= newindx) { + split_indx = newindx; + newpos = 0; + } else + split_indx = i+1; + break; + } + } + } + } + } + + /* First find the separating key between the split pages. + * The case where newindx == split_indx is ambiguous; the + * new item could go to the new page or stay on the original + * page. If newpos == 1 it goes to the new page. + */ + if (newindx == split_indx && newpos) { + sepkey.mv_size = newkey->mv_size; + sepkey.mv_data = newkey->mv_data; + } else { + node = NODEPTR(mp, split_indx); + sepkey.mv_size = node->mn_ksize; + sepkey.mv_data = NODEKEY(node); + } + +newsep: + DPRINTF("separator is [%s]", DKEY(&sepkey)); + + /* Copy separator key to the parent. + */ + if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(mc->mc_txn->mt_env, &sepkey)) { + mn.mc_snum--; + mn.mc_top--; + did_split = 1; + rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0); + + /* root split? */ + if (mn.mc_snum == mc->mc_snum) { + mc->mc_pg[mc->mc_snum] = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_snum] = mc->mc_ki[mc->mc_top]; + mc->mc_pg[mc->mc_top] = mc->mc_pg[ptop]; + mc->mc_ki[mc->mc_top] = mc->mc_ki[ptop]; + mc->mc_snum++; + mc->mc_top++; + ptop++; + } + /* Right page might now have changed parent. + * Check if left page also changed parent. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; imc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + mc->mc_pg[ptop] = mn.mc_pg[ptop]; + mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; + } + } else { + mn.mc_top--; + rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0); + mn.mc_top++; + } + mc->mc_flags ^= C_SPLITTING; + if (rc != MDB_SUCCESS) { + return rc; + } + if (nflags & MDB_APPEND) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[mc->mc_top] = 0; + rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags); + if (rc) + return rc; + for (i=0; imc_top; i++) + mc->mc_ki[i] = mn.mc_ki[i]; + goto done; + } + if (IS_LEAF2(rp)) { + goto done; + } + + /* Move half of the keys to the right sibling. */ + + /* grab a page to hold a temporary copy */ + copy = mdb_page_malloc(mc->mc_txn, 1); + if (copy == NULL) + return ENOMEM; + + copy->mp_pgno = mp->mp_pgno; + copy->mp_flags = mp->mp_flags; + copy->mp_lower = PAGEHDRSZ; + copy->mp_upper = mc->mc_txn->mt_env->me_psize; + mc->mc_pg[mc->mc_top] = copy; + for (i = j = 0; i <= nkeys; j++) { + if (i == split_indx) { + /* Insert in right sibling. */ + /* Reset insert index for right sibling. */ + if (i != newindx || (newpos ^ ins_new)) { + j = 0; + mc->mc_pg[mc->mc_top] = rp; + } + } + + if (i == newindx && !ins_new) { + /* Insert the original entry that caused the split. */ + rkey.mv_data = newkey->mv_data; + rkey.mv_size = newkey->mv_size; + if (IS_LEAF(mp)) { + rdata = newdata; + } else + pgno = newpgno; + flags = nflags; + + ins_new = 1; + + /* Update index for the new key. */ + mc->mc_ki[mc->mc_top] = j; + } else if (i == nkeys) { + break; + } else { + node = NODEPTR(mp, i); + rkey.mv_data = NODEKEY(node); + rkey.mv_size = node->mn_ksize; + if (IS_LEAF(mp)) { + xdata.mv_data = NODEDATA(node); + xdata.mv_size = NODEDSZ(node); + rdata = &xdata; + } else + pgno = NODEPGNO(node); + flags = node->mn_flags; + + i++; + } + + if (!IS_LEAF(mp) && j == 0) { + /* First branch index doesn't need key data. */ + rkey.mv_size = 0; + } + + rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags); + if (rc) break; + } + + nkeys = NUMKEYS(copy); + for (i=0; imp_ptrs[i] = copy->mp_ptrs[i]; + mp->mp_lower = copy->mp_lower; + mp->mp_upper = copy->mp_upper; + memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1), + mc->mc_txn->mt_env->me_psize - copy->mp_upper); + + /* reset back to original page */ + if (newindx < split_indx || (!newpos && newindx == split_indx)) { + mc->mc_pg[mc->mc_top] = mp; + if (nflags & MDB_RESERVE) { + node = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (!(node->mn_flags & F_BIGDATA)) + newdata->mv_data = NODEDATA(node); + } + } else { + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; imc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + mc->mc_pg[ptop] = mn.mc_pg[ptop]; + mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; + } + } + + /* return tmp page to freelist */ + mdb_page_free(mc->mc_txn->mt_env, copy); +done: + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + int fixup = NUMKEYS(mp); + + if (mc->mc_flags & C_SUB) + dbi--; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc) + continue; + if (!(m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_flags & C_SPLITTING) + continue; + if (new_root) { + int k; + /* root split */ + for (k=m3->mc_top; k>=0; k--) { + m3->mc_ki[k+1] = m3->mc_ki[k]; + m3->mc_pg[k+1] = m3->mc_pg[k]; + } + if (m3->mc_ki[0] >= split_indx) { + m3->mc_ki[0] = 1; + } else { + m3->mc_ki[0] = 0; + } + m3->mc_pg[0] = mc->mc_pg[0]; + m3->mc_snum++; + m3->mc_top++; + } + if (m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE)) + m3->mc_ki[mc->mc_top]++; + if (m3->mc_ki[mc->mc_top] >= fixup) { + m3->mc_pg[mc->mc_top] = rp; + m3->mc_ki[mc->mc_top] -= fixup; + m3->mc_ki[ptop] = mn.mc_ki[ptop]; + } + } else if (!did_split && m3->mc_pg[ptop] == mc->mc_pg[ptop] && + m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { + m3->mc_ki[ptop]++; + } + } + } + return rc; +} + +int +mdb_put(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned int flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + + assert(key != NULL); + assert(data != NULL); + + if (txn == NULL || !dbi || dbi >= txn->mt_numdbs || !(txn->mt_dbflags[dbi] & DB_VALID)) + return EINVAL; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + return EACCES; + } + + if (key->mv_size == 0 || key->mv_size > MDB_MAXKEYSIZE) { + return EINVAL; + } + + if ((flags & (MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) != flags) + return EINVAL; + + mdb_cursor_init(&mc, txn, dbi, &mx); + return mdb_cursor_put(&mc, key, data, flags); +} + +int +mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) +{ + if ((flag & CHANGEABLE) != flag) + return EINVAL; + if (onoff) + env->me_flags |= flag; + else + env->me_flags &= ~flag; + return MDB_SUCCESS; +} + +int +mdb_env_get_flags(MDB_env *env, unsigned int *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_flags; + return MDB_SUCCESS; +} + +int +mdb_env_get_path(MDB_env *env, const char **arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_path; + return MDB_SUCCESS; +} + +/** Common code for #mdb_stat() and #mdb_env_stat(). + * @param[in] env the environment to operate in. + * @param[in] db the #MDB_db record containing the stats to return. + * @param[out] arg the address of an #MDB_stat structure to receive the stats. + * @return 0, this function always succeeds. + */ +static int +mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg) +{ + arg->ms_psize = env->me_psize; + arg->ms_depth = db->md_depth; + arg->ms_branch_pages = db->md_branch_pages; + arg->ms_leaf_pages = db->md_leaf_pages; + arg->ms_overflow_pages = db->md_overflow_pages; + arg->ms_entries = db->md_entries; + + return MDB_SUCCESS; +} +int +mdb_env_stat(MDB_env *env, MDB_stat *arg) +{ + int toggle; + + if (env == NULL || arg == NULL) + return EINVAL; + + toggle = mdb_env_pick_meta(env); + + return mdb_stat0(env, &env->me_metas[toggle]->mm_dbs[MAIN_DBI], arg); +} + +int +mdb_env_info(MDB_env *env, MDB_envinfo *arg) +{ + int toggle; + + if (env == NULL || arg == NULL) + return EINVAL; + + toggle = mdb_env_pick_meta(env); + arg->me_mapaddr = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : 0; + arg->me_mapsize = env->me_mapsize; + arg->me_maxreaders = env->me_maxreaders; + arg->me_numreaders = env->me_numreaders; + arg->me_last_pgno = env->me_metas[toggle]->mm_last_pg; + arg->me_last_txnid = env->me_metas[toggle]->mm_txnid; + return MDB_SUCCESS; +} + +/** Set the default comparison functions for a database. + * Called immediately after a database is opened to set the defaults. + * The user can then override them with #mdb_set_compare() or + * #mdb_set_dupsort(). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +static void +mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi) +{ + uint16_t f = txn->mt_dbs[dbi].md_flags; + + txn->mt_dbxs[dbi].md_cmp = + (f & MDB_REVERSEKEY) ? mdb_cmp_memnr : + (f & MDB_INTEGERKEY) ? mdb_cmp_cint : mdb_cmp_memn; + + txn->mt_dbxs[dbi].md_dcmp = + !(f & MDB_DUPSORT) ? 0 : + ((f & MDB_INTEGERDUP) + ? ((f & MDB_DUPFIXED) ? mdb_cmp_int : mdb_cmp_cint) + : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn)); +} + +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi) +{ + MDB_val key, data; + MDB_dbi i; + MDB_cursor mc; + int rc, dbflag, exact; + unsigned int unused = 0; + size_t len; + + if (txn->mt_dbxs[FREE_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, FREE_DBI); + } + + if ((flags & VALID_FLAGS) != flags) + return EINVAL; + + /* main DB? */ + if (!name) { + *dbi = MAIN_DBI; + if (flags & PERSISTENT_FLAGS) { + uint16_t f2 = flags & PERSISTENT_FLAGS; + /* make sure flag changes get committed */ + if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) { + txn->mt_dbs[MAIN_DBI].md_flags |= f2; + txn->mt_flags |= MDB_TXN_DIRTY; + } + } + mdb_default_cmp(txn, MAIN_DBI); + return MDB_SUCCESS; + } + + if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, MAIN_DBI); + } + + /* Is the DB already open? */ + len = strlen(name); + for (i=2; imt_numdbs; i++) { + if (!txn->mt_dbxs[i].md_name.mv_size) { + /* Remember this free slot */ + if (!unused) unused = i; + continue; + } + if (len == txn->mt_dbxs[i].md_name.mv_size && + !strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) { + *dbi = i; + return MDB_SUCCESS; + } + } + + /* If no free slot and max hit, fail */ + if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs) + return MDB_DBS_FULL; + + /* Cannot mix named databases with some mainDB flags */ + if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY)) + return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; + + /* Find the DB info */ + dbflag = DB_NEW|DB_VALID; + exact = 0; + key.mv_size = len; + key.mv_data = (void *)name; + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact); + if (rc == MDB_SUCCESS) { + /* make sure this is actually a DB */ + MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); + if (!(node->mn_flags & F_SUBDATA)) + return EINVAL; + } else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) { + /* Create if requested */ + MDB_db dummy; + data.mv_size = sizeof(MDB_db); + data.mv_data = &dummy; + memset(&dummy, 0, sizeof(dummy)); + dummy.md_root = P_INVALID; + dummy.md_flags = flags & PERSISTENT_FLAGS; + rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA); + dbflag |= DB_DIRTY; + } + + /* OK, got info, add to table */ + if (rc == MDB_SUCCESS) { + unsigned int slot = unused ? unused : txn->mt_numdbs; + txn->mt_dbxs[slot].md_name.mv_data = strdup(name); + txn->mt_dbxs[slot].md_name.mv_size = len; + txn->mt_dbxs[slot].md_rel = NULL; + txn->mt_dbflags[slot] = dbflag; + memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); + *dbi = slot; + txn->mt_env->me_dbflags[slot] = txn->mt_dbs[slot].md_flags; + mdb_default_cmp(txn, slot); + if (!unused) { + txn->mt_numdbs++; + } + } + + return rc; +} + +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) +{ + if (txn == NULL || arg == NULL || dbi >= txn->mt_numdbs) + return EINVAL; + + if (txn->mt_dbflags[dbi] & DB_STALE) { + MDB_cursor mc; + MDB_xcursor mx; + /* Stale, must read the DB's root. cursor_init does it for us. */ + mdb_cursor_init(&mc, txn, dbi, &mx); + } + return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg); +} + +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) +{ + char *ptr; + if (dbi <= MAIN_DBI || dbi >= env->me_maxdbs) + return; + ptr = env->me_dbxs[dbi].md_name.mv_data; + env->me_dbxs[dbi].md_name.mv_data = NULL; + env->me_dbxs[dbi].md_name.mv_size = 0; + env->me_dbflags[dbi] = 0; + free(ptr); +} + +/** Add all the DB's pages to the free list. + * @param[in] mc Cursor on the DB to free. + * @param[in] subs non-Zero to check for sub-DBs in this DB. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_drop0(MDB_cursor *mc, int subs) +{ + int rc; + + rc = mdb_page_search(mc, NULL, 0); + if (rc == MDB_SUCCESS) { + MDB_txn *txn = mc->mc_txn; + MDB_node *ni; + MDB_cursor mx; + unsigned int i; + + /* LEAF2 pages have no nodes, cannot have sub-DBs */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) + mdb_cursor_pop(mc); + + mdb_cursor_copy(mc, &mx); + while (mc->mc_snum > 0) { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + unsigned n = NUMKEYS(mp); + if (IS_LEAF(mp)) { + for (i=0; imn_flags & F_BIGDATA) { + MDB_page *omp; + pgno_t pg; + memcpy(&pg, NODEDATA(ni), sizeof(pg)); + rc = mdb_page_get(txn, pg, &omp, NULL); + if (rc != 0) + return rc; + assert(IS_OVERFLOW(omp)); + rc = mdb_midl_append_range(&txn->mt_free_pgs, + pg, omp->mp_pages); + if (rc) + return rc; + } else if (subs && (ni->mn_flags & F_SUBDATA)) { + mdb_xcursor_init1(mc, ni); + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + return rc; + } + } + } else { + if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) + return rc; + for (i=0; imt_free_pgs, pg); + } + } + if (!mc->mc_top) + break; + mc->mc_ki[mc->mc_top] = i; + rc = mdb_cursor_sibling(mc, 1); + if (rc) { + /* no more siblings, go back to beginning + * of previous level. + */ + mdb_cursor_pop(mc); + mc->mc_ki[0] = 0; + for (i=1; imc_snum; i++) { + mc->mc_ki[i] = 0; + mc->mc_pg[i] = mx.mc_pg[i]; + } + } + } + /* free it */ + rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root); + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + return rc; +} + +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) +{ + MDB_cursor *mc, *m2; + int rc; + + if (!txn || !dbi || dbi >= txn->mt_numdbs || (unsigned)del > 1 || !(txn->mt_dbflags[dbi] & DB_VALID)) + return EINVAL; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) + return rc; + + rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT); + /* Invalidate the dropped DB's cursors */ + for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) + m2->mc_flags &= ~(C_INITIALIZED|C_EOF); + if (rc) + goto leave; + + /* Can't delete the main DB */ + if (del && dbi > MAIN_DBI) { + rc = mdb_del(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL); + if (!rc) { + txn->mt_dbflags[dbi] = DB_STALE; + mdb_dbi_close(txn->mt_env, dbi); + } + } else { + /* reset the DB record, mark it dirty */ + txn->mt_dbflags[dbi] |= DB_DIRTY; + txn->mt_dbs[dbi].md_depth = 0; + txn->mt_dbs[dbi].md_branch_pages = 0; + txn->mt_dbs[dbi].md_leaf_pages = 0; + txn->mt_dbs[dbi].md_overflow_pages = 0; + txn->mt_dbs[dbi].md_entries = 0; + txn->mt_dbs[dbi].md_root = P_INVALID; + + txn->mt_flags |= MDB_TXN_DIRTY; + } +leave: + mdb_cursor_close(mc); + return rc; +} + +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (txn == NULL || !dbi || dbi >= txn->mt_numdbs || !(txn->mt_dbflags[dbi] & DB_VALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_cmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (txn == NULL || !dbi || dbi >= txn->mt_numdbs || !(txn->mt_dbflags[dbi] & DB_VALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_dcmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) +{ + if (txn == NULL || !dbi || dbi >= txn->mt_numdbs || !(txn->mt_dbflags[dbi] & DB_VALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_rel = rel; + return MDB_SUCCESS; +} + +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) +{ + if (txn == NULL || !dbi || dbi >= txn->mt_numdbs || !(txn->mt_dbflags[dbi] & DB_VALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_relctx = ctx; + return MDB_SUCCESS; +} + +/** @} */ diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1 new file mode 100644 index 0000000000..7837de5f6b --- /dev/null +++ b/libraries/liblmdb/mdb_copy.1 @@ -0,0 +1,28 @@ +.TH MDB_COPY 1 "2012/12/12" "LMDB 0.9.5" +.\" Copyright 2012 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_copy \- LMDB environment copy tool +.SH SYNOPSIS +.B mdb_copy +.I srcpath\ [dstpath] +.SH DESCRIPTION +The +.B mdb_copy +utility copies an LMDB environment. The environment can +be copied regardless of whether it is currently in use. + +If +.I dstpath +is specified it must be the path of an empty directory +for storing the backup. Otherwise, the backup will be +written to stdout. + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH "SEE ALSO" +.BR mdb_stat (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c new file mode 100644 index 0000000000..ca92009cff --- /dev/null +++ b/libraries/liblmdb/mdb_copy.c @@ -0,0 +1,66 @@ +/* mdb_copy.c - memory-mapped database backup tool */ +/* + * Copyright 2012 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#ifdef _WIN32 +#include +#define MDB_STDOUT GetStdHandle(STD_OUTPUT_HANDLE) +#else +#define MDB_STDOUT 1 +#endif +#include +#include +#include +#include "lmdb.h" + +static void +sighandle(int sig) +{ +} + +int main(int argc,char * argv[]) +{ + int rc; + MDB_env *env; + char *envname = argv[1]; + + if (argc<2 || argc>3) { + fprintf(stderr, "usage: %s srcpath [dstpath]\n", argv[0]); + exit(EXIT_FAILURE); + } + +#ifdef SIGPIPE + signal(SIGPIPE, sighandle); +#endif +#ifdef SIGHUP + signal(SIGHUP, sighandle); +#endif + signal(SIGINT, sighandle); + signal(SIGTERM, sighandle); + + rc = mdb_env_create(&env); + + rc = mdb_env_open(env, envname, MDB_RDONLY, 0); + if (rc) { + printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + } else { + if (argc == 2) + rc = mdb_env_copyfd(env, MDB_STDOUT); + else + rc = mdb_env_copy(env, argv[2]); + if (rc) + printf("mdb_env_copy failed, error %d %s\n", rc, mdb_strerror(rc)); + } + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1 new file mode 100644 index 0000000000..1307c39d09 --- /dev/null +++ b/libraries/liblmdb/mdb_stat.1 @@ -0,0 +1,47 @@ +.TH MDB_STAT 1 "2012/12/12" "LMDB 0.9.5" +.\" Copyright 2012 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_stat \- LMDB environment status tool +.SH SYNOPSIS +.B mdb_stat +.BR \ envpath +[\c +.BR \-e ] +[\c +.BR \-f [ f [ f ]]] +[\c +.BR \-n ] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.SH DESCRIPTION +The +.B mdb_stat +utility displays the status of an LMDB environment. +.SH OPTIONS +.TP +.BR \-e +Display information about the database environment. +.TP +.BR \-f +Display information about the environment freelist. +If \fB\-ff\fP is given, summarize each freelist entry. +If \fB\-fff\fP is given, display the full list of page IDs in the freelist. +.TP +.BR \-n +Display the status of an LMDB database which does not use subdirectories. +.TP +.BR \-a +Display the status of all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Display the status of a specific subdatabase. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH "SEE ALSO" +.BR mdb_copy (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c new file mode 100644 index 0000000000..3e6be21597 --- /dev/null +++ b/libraries/liblmdb/mdb_stat.c @@ -0,0 +1,230 @@ +/* mdb_stat.c - memory-mapped database status tool */ +/* + * Copyright 2011-2013 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include +#include +#include +#include "lmdb.h" + +static void prstat(MDB_stat *ms) +{ +#if 0 + printf(" Page size: %u\n", ms->ms_psize); +#endif + printf(" Tree depth: %u\n", ms->ms_depth); + printf(" Branch pages: %zu\n", ms->ms_branch_pages); + printf(" Leaf pages: %zu\n", ms->ms_leaf_pages); + printf(" Overflow pages: %zu\n", ms->ms_overflow_pages); + printf(" Entries: %zu\n", ms->ms_entries); +} + +static void usage(char *prog) +{ + fprintf(stderr, "usage: %s dbpath [-e] [-f[f[f]]] [-n] [-a|-s subdb]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + MDB_stat mst; + MDB_envinfo mei; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0; + + if (argc < 2) { + usage(prog); + } + + /* -a: print stat of main DB and all subDBs + * -s: print stat of only the named subDB + * -e: print env info + * -f: print freelist info + * -n: use NOSUBDIR flag on env_open + * (default) print stat of only the main DB + */ + while ((i = getopt(argc, argv, "aefns:")) != EOF) { + switch(i) { + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'e': + envinfo++; + break; + case 'f': + freinfo++; + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + + envname = argv[optind]; + rc = mdb_env_create(&env); + + if (alldbs || subname) { + mdb_env_set_maxdbs(env, 4); + } + + rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); + if (rc) { + printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + if (envinfo) { + rc = mdb_env_stat(env, &mst); + rc = mdb_env_info(env, &mei); + printf("Environment Info\n"); + printf(" Map address: %p\n", mei.me_mapaddr); + printf(" Map size: %zu\n", mei.me_mapsize); + printf(" Page size: %u\n", mst.ms_psize); + printf(" Max pages: %zu\n", mei.me_mapsize / mst.ms_psize); + printf(" Number of pages used: %zu\n", mei.me_last_pgno+1); + printf(" Last transaction ID: %zu\n", mei.me_last_txnid); + printf(" Max readers: %u\n", mei.me_maxreaders); + printf(" Number of readers used: %u\n", mei.me_numreaders); + } + + if (freinfo) { + MDB_cursor *cursor; + MDB_val key, data; + size_t pages = 0, *iptr; + + printf("Freelist Status\n"); + dbi = 0; + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + rc = mdb_stat(txn, dbi, &mst); + if (rc) { + printf("mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + iptr = data.mv_data; + pages += *iptr; + if (freinfo > 1) { + char *bad = ""; + size_t pg, prev; + ssize_t i, j, span = 0; + j = *iptr++; + for (i = j, prev = 1; --i >= 0; ) { + pg = iptr[i]; + if (pg <= prev) + bad = " [bad sequence]"; + prev = pg; + pg += span; + for (; i >= span && iptr[i-span] == pg; span++, pg++) ; + } + printf(" Transaction %zu, %zd pages, maxspan %zd%s\n", + *(size_t *)key.mv_data, j, span, bad); + if (freinfo > 2) { + for (--j; j >= 0; ) { + pg = iptr[j]; + for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ; + printf(span>1 ? " %9zu[%zd]\n" : " %9zu\n", + pg, span); + } + } + } + } + mdb_cursor_close(cursor); + printf(" Free pages: %zu\n", pages); + } + + rc = mdb_open(txn, subname, 0, &dbi); + if (rc) { + printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + rc = mdb_stat(txn, dbi, &mst); + if (rc) { + printf("mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + printf("Status of %s\n", subname ? subname : "Main DB"); + prstat(&mst); + + if (alldbs) { + MDB_cursor *cursor; + MDB_val key; + + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { + char *str; + MDB_dbi db2; + if (memchr(key.mv_data, '\0', key.mv_size)) + continue; + str = malloc(key.mv_size+1); + memcpy(str, key.mv_data, key.mv_size); + str[key.mv_size] = '\0'; + rc = mdb_open(txn, str, 0, &db2); + if (rc == MDB_SUCCESS) + printf("Status of %s\n", str); + free(str); + if (rc) continue; + rc = mdb_stat(txn, db2, &mst); + if (rc) { + printf("mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + mdb_close(env, db2); + } + mdb_cursor_close(cursor); + } + + if (rc == MDB_NOTFOUND) + rc = MDB_SUCCESS; + + mdb_close(env, dbi); +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c new file mode 100644 index 0000000000..e7bd680cb0 --- /dev/null +++ b/libraries/liblmdb/midl.c @@ -0,0 +1,348 @@ +/** @file midl.c + * @brief ldap bdb back-end ID List functions */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 2000-2013 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include +#include +#include +#include +#include +#include +#include "midl.h" + +/** @defgroup internal MDB Internals + * @{ + */ +/** @defgroup idls ID List Management + * @{ + */ +#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) + +#if 0 /* superseded by append/sort */ +static unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( ids[cursor], id ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) +{ + unsigned x, i; + + x = mdb_midl_search( ids, id ); + assert( x > 0 ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0] && ids[x] == id ) { + /* duplicate */ + assert(0); + return -1; + } + + if ( ++ids[0] >= MDB_IDL_DB_MAX ) { + /* no room */ + --ids[0]; + return -2; + + } else { + /* insert id */ + for (i=ids[0]; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = id; + } + + return 0; +} +#endif + +MDB_IDL mdb_midl_alloc(int num) +{ + MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); + if (ids) { + *ids++ = num; + *ids = 0; + } + return ids; +} + +void mdb_midl_free(MDB_IDL ids) +{ + if (ids) + free(ids-1); +} + +int mdb_midl_shrink( MDB_IDL *idp ) +{ + MDB_IDL ids = *idp; + if (*(--ids) > MDB_IDL_UM_MAX && + (ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID)))) + { + *ids++ = MDB_IDL_UM_MAX; + *idp = ids; + return 1; + } + return 0; +} + +static int mdb_midl_grow( MDB_IDL *idp, int num ) +{ + MDB_IDL idn = *idp-1; + /* grow it */ + idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); + if (!idn) + return ENOMEM; + *idn++ += num; + *idp = idn; + return 0; +} + +int mdb_midl_need( MDB_IDL *idp, unsigned num ) +{ + MDB_IDL ids = *idp; + num += ids[0]; + if (num > ids[-1]) { + num = (num + num/4 + (256 + 2)) & -256; + if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) + return ENOMEM; + *ids++ = num -= 2; + *idp = ids; + } + return 0; +} + +int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] >= ids[-1]) { + if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0]++; + ids[ids[0]] = id; + return 0; +} + +int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] + app[0] >= ids[-1]) { + if (mdb_midl_grow(idp, app[0])) + return ENOMEM; + ids = *idp; + } + memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); + ids[0] += app[0]; + return 0; +} + +int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) +{ + MDB_ID *ids = *idp, len = ids[0]; + /* Too big? */ + if (len + n > ids[-1]) { + if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0] = len + n; + ids += len; + while (n) + ids[n--] = id++; + return 0; +} + +/* Quicksort + Insertion sort for small arrays */ + +#define SMALL 8 +#define SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } + +void +mdb_midl_sort( MDB_IDL ids ) +{ + /* Max possible depth of int-indexed tree * 2 items/level */ + int istack[sizeof(int)*CHAR_BIT * 2]; + int i,j,k,l,ir,jstack; + MDB_ID a, itmp; + + ir = (int)ids[0]; + l = 1; + jstack = 0; + for(;;) { + if (ir - l < SMALL) { /* Insertion sort */ + for (j=l+1;j<=ir;j++) { + a = ids[j]; + for (i=j-1;i>=1;i--) { + if (ids[i] >= a) break; + ids[i+1] = ids[i]; + } + ids[i+1] = a; + } + if (jstack == 0) break; + ir = istack[jstack--]; + l = istack[jstack--]; + } else { + k = (l + ir) >> 1; /* Choose median of left, center, right */ + SWAP(ids[k], ids[l+1]); + if (ids[l] < ids[ir]) { + SWAP(ids[l], ids[ir]); + } + if (ids[l+1] < ids[ir]) { + SWAP(ids[l+1], ids[ir]); + } + if (ids[l] < ids[l+1]) { + SWAP(ids[l], ids[l+1]); + } + i = l+1; + j = ir; + a = ids[l+1]; + for(;;) { + do i++; while(ids[i] > a); + do j--; while(ids[j] < a); + if (j < i) break; + SWAP(ids[i],ids[j]); + } + ids[l+1] = ids[j]; + ids[j] = a; + jstack += 2; + if (ir-i+1 >= j-l) { + istack[jstack] = ir; + istack[jstack-1] = i; + ir = j-1; + } else { + istack[jstack] = j-1; + istack[jstack-1] = l; + l = i; + } + } + } +} + +unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = (unsigned)ids[0].mid; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( id, ids[cursor].mid ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) +{ + unsigned x, i; + + x = mdb_mid2l_search( ids, id->mid ); + assert( x > 0 ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0].mid && ids[x].mid == id->mid ) { + /* duplicate */ + return -1; + } + + if ( ids[0].mid >= MDB_IDL_UM_MAX ) { + /* too big */ + return -2; + + } else { + /* insert id */ + ids[0].mid++; + for (i=(unsigned)ids[0].mid; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = *id; + } + + return 0; +} + +int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) +{ + /* Too big? */ + if (ids[0].mid >= MDB_IDL_UM_MAX) { + return -2; + } + ids[0].mid++; + ids[ids[0].mid] = *id; + return 0; +} + +/** @} */ +/** @} */ diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h new file mode 100644 index 0000000000..9ce7133c6e --- /dev/null +++ b/libraries/liblmdb/midl.h @@ -0,0 +1,179 @@ +/** @file midl.h + * @brief mdb ID List header file. + * + * This file was originally part of back-bdb but has been + * modified for use in libmdb. Most of the macros defined + * in this file are unused, just left over from the original. + * + * This file is only used internally in libmdb and its definitions + * are not exposed publicly. + */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 2000-2013 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _MDB_MIDL_H_ +#define _MDB_MIDL_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @defgroup internal MDB Internals + * @{ + */ + +/** @defgroup idls ID List Management + * @{ + */ + /** A generic ID number. These were entryIDs in back-bdb. + * Preferably it should have the same size as a pointer. + */ +typedef size_t MDB_ID; + + /** An IDL is an ID List, a sorted array of IDs. The first + * element of the array is a counter for how many actual + * IDs are in the list. In the original back-bdb code, IDLs are + * sorted in ascending order. For libmdb IDLs are sorted in + * descending order. + */ +typedef MDB_ID *MDB_IDL; + +/* IDL sizes - likely should be even bigger + * limiting factors: sizeof(ID), thread stack size + */ +#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ +#define MDB_IDL_DB_SIZE (1<. + */ +#define _XOPEN_SOURCE 500 /* srandom(), random() */ +#include +#include +#include +#include "lmdb.h" + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor, *cur2; + int count; + int *values; + char sval[32]; + + srandom(time(NULL)); + + count = (random()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (random()%5)) { + j++; + txn=NULL; + rc = mdb_txn_begin(env, NULL, 0, &txn); + sprintf(sval, "%03x ", values[i]); + rc = mdb_del(txn, dbi, &key, NULL); + if (rc) { + j--; + mdb_txn_abort(txn); + } else { + rc = mdb_txn_commit(txn); + } + } + free(values); + printf("Deleted %d values\n", j); + + rc = mdb_env_stat(env, &mst); + rc = mdb_txn_begin(env, NULL, 1, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + printf("Cursor last\n"); + rc = mdb_cursor_get(cursor, &key, &data, MDB_LAST); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + printf("Cursor last/prev\n"); + rc = mdb_cursor_get(cursor, &key, &data, MDB_LAST); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + + mdb_txn_abort(txn); + + printf("Deleting with cursor\n"); + rc = mdb_txn_begin(env, NULL, 0, &txn); + rc = mdb_cursor_open(txn, dbi, &cur2); + for (i=0; i<50; i++) { + rc = mdb_cursor_get(cur2, &key, &data, MDB_NEXT); + if (rc) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + rc = mdb_del(txn, dbi, &key, NULL); + } + + printf("Restarting cursor in txn\n"); + rc = mdb_cursor_get(cur2, &key, &data, MDB_FIRST); + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + for (i=0; i<32; i++) { + rc = mdb_cursor_get(cur2, &key, &data, MDB_NEXT); + if (rc) break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cur2); + rc = mdb_txn_commit(txn); + + printf("Restarting cursor outside txn\n"); + rc = mdb_txn_begin(env, NULL, 0, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + rc = mdb_cursor_get(cursor, &key, &data, MDB_FIRST); + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + for (i=0; i<32; i++) { + rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT); + if (rc) break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c new file mode 100644 index 0000000000..44d1de7ccd --- /dev/null +++ b/libraries/liblmdb/mtest2.c @@ -0,0 +1,117 @@ +/* mtest2.c - memory-mapped database tester/toy */ +/* + * Copyright 2011 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Just like mtest.c, but using a subDB instead of the main DB */ + +#define _XOPEN_SOURCE 500 /* srandom(), random() */ +#include +#include +#include +#include "lmdb.h" + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32]; + + srandom(time(NULL)); + + count = (random()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (random()%5)) { + j++; + txn=NULL; + rc = mdb_txn_begin(env, NULL, 0, &txn); + sprintf(sval, "%03x ", values[i]); + rc = mdb_del(txn, dbi, &key, NULL); + if (rc) { + j--; + mdb_txn_abort(txn); + } else { + rc = mdb_txn_commit(txn); + } + } + free(values); + printf("Deleted %d values\n", j); + + rc = mdb_env_stat(env, &mst); + rc = mdb_txn_begin(env, NULL, 1, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/libraries/liblmdb/mtest3.c b/libraries/liblmdb/mtest3.c new file mode 100644 index 0000000000..c189eaa952 --- /dev/null +++ b/libraries/liblmdb/mtest3.c @@ -0,0 +1,127 @@ +/* mtest3.c - memory-mapped database tester/toy */ +/* + * Copyright 2011 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Tests for sorted duplicate DBs */ +#define _XOPEN_SOURCE 500 /* srandom(), random() */ +#include +#include +#include +#include +#include "lmdb.h" + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32]; + char kval[sizeof(int)]; + + srandom(time(NULL)); + + memset(sval, 0, sizeof(sval)); + + count = (random()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (random()%5)) { + j++; + txn=NULL; + rc = mdb_txn_begin(env, NULL, 0, &txn); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + rc = mdb_del(txn, dbi, &key, &data); + if (rc) { + j--; + mdb_txn_abort(txn); + } else { + rc = mdb_txn_commit(txn); + } + } + free(values); + printf("Deleted %d values\n", j); + + rc = mdb_env_stat(env, &mst); + rc = mdb_txn_begin(env, NULL, 1, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/libraries/liblmdb/mtest4.c b/libraries/liblmdb/mtest4.c new file mode 100644 index 0000000000..e0ba7e20b6 --- /dev/null +++ b/libraries/liblmdb/mtest4.c @@ -0,0 +1,161 @@ +/* mtest4.c - memory-mapped database tester/toy */ +/* + * Copyright 2011 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Tests for sorted duplicate DBs with fixed-size keys */ +#define _XOPEN_SOURCE 500 /* srandom(), random() */ +#include +#include +#include +#include +#include "lmdb.h" + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[8]; + char kval[sizeof(int)]; + + memset(sval, 0, sizeof(sval)); + + count = 510; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (random()%3)) { + j++; + txn=NULL; + rc = mdb_txn_begin(env, NULL, 0, &txn); + sprintf(sval, "%07x", values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + rc = mdb_del(txn, dbi, &key, &data); + if (rc) { + j--; + mdb_txn_abort(txn); + } else { + rc = mdb_txn_commit(txn); + } + } + free(values); + printf("Deleted %d values\n", j); + + rc = mdb_env_stat(env, &mst); + rc = mdb_txn_begin(env, NULL, 1, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/libraries/liblmdb/mtest5.c b/libraries/liblmdb/mtest5.c new file mode 100644 index 0000000000..bc472fa093 --- /dev/null +++ b/libraries/liblmdb/mtest5.c @@ -0,0 +1,129 @@ +/* mtest5.c - memory-mapped database tester/toy */ +/* + * Copyright 2011 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Tests for sorted duplicate DBs using cursor_put */ +#define _XOPEN_SOURCE 500 /* srandom(), random() */ +#include +#include +#include +#include +#include "lmdb.h" + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32]; + char kval[sizeof(int)]; + + srandom(time(NULL)); + + memset(sval, 0, sizeof(sval)); + + count = (random()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (random()%5)) { + j++; + txn=NULL; + rc = mdb_txn_begin(env, NULL, 0, &txn); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + rc = mdb_del(txn, dbi, &key, &data); + if (rc) { + j--; + mdb_txn_abort(txn); + } else { + rc = mdb_txn_commit(txn); + } + } + free(values); + printf("Deleted %d values\n", j); + + rc = mdb_env_stat(env, &mst); + rc = mdb_txn_begin(env, NULL, 1, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c new file mode 100644 index 0000000000..0bf26ccc45 --- /dev/null +++ b/libraries/liblmdb/mtest6.c @@ -0,0 +1,131 @@ +/* mtest6.c - memory-mapped database tester/toy */ +/* + * Copyright 2011 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Tests for DB splits and merges */ +#define _XOPEN_SOURCE 500 /* srandom(), random() */ +#include +#include +#include +#include +#include "lmdb.h" + +char dkbuf[1024]; + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + long kval; + char *sval; + + srandom(time(NULL)); + + rc = mdb_env_create(&env); + rc = mdb_env_set_mapsize(env, 10485760); + rc = mdb_env_set_maxdbs(env, 4); + rc = mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664); + rc = mdb_txn_begin(env, NULL, 0, &txn); + rc = mdb_open(txn, "id2", MDB_CREATE|MDB_INTEGERKEY, &dbi); + rc = mdb_cursor_open(txn, dbi, &cursor); + rc = mdb_stat(txn, dbi, &mst); + + sval = calloc(1, mst.ms_psize / 4); + key.mv_size = sizeof(long); + key.mv_data = &kval; + data.mv_size = mst.ms_psize / 4 - 30; + data.mv_data = sval; + + printf("Adding 12 values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5; + sprintf(sval, "%08x", kval); + rc = mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE); + } + printf("Adding 12 more values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5+4; + sprintf(sval, "%08x", kval); + rc = mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE); + } + printf("Adding 12 more values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5+1; + sprintf(sval, "%08x", kval); + rc = mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE); + } + rc = mdb_cursor_get(cursor, &key, &data, MDB_FIRST); + + do { + printf("key: %p %s, data: %p %.*s\n", + key.mv_data, mdb_dkey(&key, dkbuf), + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0); + mdb_cursor_close(cursor); + mdb_txn_commit(txn); + +#if 0 + j=0; + + for (i= count - 1; i > -1; i-= (random()%5)) { + j++; + txn=NULL; + rc = mdb_txn_begin(env, NULL, 0, &txn); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + rc = mdb_del(txn, dbi, &key, &data); + if (rc) { + j--; + mdb_txn_abort(txn); + } else { + rc = mdb_txn_commit(txn); + } + } + free(values); + printf("Deleted %d values\n", j); + + rc = mdb_env_stat(env, &mst); + rc = mdb_txn_begin(env, NULL, 1, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_close(txn, dbi); + + mdb_txn_abort(txn); +#endif + mdb_env_close(env); + + return 0; +} diff --git a/libraries/liblmdb/sample-bdb.c b/libraries/liblmdb/sample-bdb.c new file mode 100644 index 0000000000..2c11bb38a0 --- /dev/null +++ b/libraries/liblmdb/sample-bdb.c @@ -0,0 +1,71 @@ +/* sample-bdb.c - BerkeleyDB toy/sample + * + * Do a line-by-line comparison of this and sample-mdb.c + */ +/* + * Copyright 2012 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include +#include + +int main(int argc,char * argv[]) +{ + int rc; + DB_ENV *env; + DB *dbi; + DBT key, data; + DB_TXN *txn; + DBC *cursor; + char sval[32], kval[32]; + +#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD) + rc = db_env_create(&env, 0); + rc = env->open(env, "./testdb", FLAGS, 0664); + rc = db_create(&dbi, env, 0); + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664); + + memset(&key, 0, sizeof(DBT)); + memset(&data, 0, sizeof(DBT)); + key.size = sizeof(int); + key.data = sval; + data.size = sizeof(sval); + data.data = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = dbi->put(dbi, txn, &key, &data, 0); + rc = txn->commit(txn, 0); + if (rc) { + fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc)); + goto leave; + } + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->cursor(dbi, txn, &cursor, 0); + key.flags = DB_DBT_USERMEM; + key.data = kval; + key.ulen = sizeof(kval); + data.flags = DB_DBT_USERMEM; + data.data = sval; + data.ulen = sizeof(sval); + while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.data, (int) key.size, (char *) key.data, + data.data, (int) data.size, (char *) data.data); + } + rc = cursor->c_close(cursor); + rc = txn->abort(txn); +leave: + rc = dbi->close(dbi, 0); + rc = env->close(env, 0); + return rc; +} diff --git a/libraries/liblmdb/sample-mdb.c b/libraries/liblmdb/sample-mdb.c new file mode 100644 index 0000000000..0b10f47173 --- /dev/null +++ b/libraries/liblmdb/sample-mdb.c @@ -0,0 +1,60 @@ +/* sample-mdb.c - MDB toy/sample + * + * Do a line-by-line comparison of this and sample-bdb.c + */ +/* + * Copyright 2012 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include "lmdb.h" + +int main(int argc,char * argv[]) +{ + int rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_cursor *cursor; + char sval[32]; + + rc = mdb_env_create(&env); + rc = mdb_env_open(env, "./testdb", 0, 0664); + rc = mdb_txn_begin(env, NULL, 0, &txn); + rc = mdb_open(txn, NULL, 0, &dbi); + + key.mv_size = sizeof(int); + key.mv_data = sval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = mdb_put(txn, dbi, &key, &data, 0); + rc = mdb_txn_commit(txn); + if (rc) { + fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc)); + goto leave; + } + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_txn_abort(txn); +leave: + mdb_close(env, dbi); + mdb_env_close(env); + return 0; +} diff --git a/modules/ripple_app/basics/ripple_BuildVersion.h b/modules/ripple_app/basics/ripple_BuildVersion.h new file mode 100644 index 0000000000..6d5504c422 --- /dev/null +++ b/modules/ripple_app/basics/ripple_BuildVersion.h @@ -0,0 +1,45 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_BUILDVERSION_RIPPLEHEADER +#define RIPPLE_BUILDVERSION_RIPPLEHEADER + +/** Versioning information for the build. +*/ + +class BuildVersion +{ +public: + /** Retrieve the build version number. + + This is typically incremented when an official version is publshed + with a list of changes. + + Format is: + + .. + */ + static char const* getBuildVersion () + { + return "0.0.1"; + } + + /** Retrieve the client API version number. + + The client API version is incremented whenever a new feature + or breaking change is made to the websocket / RPC interface. + + Format is: + + + */ + static char const* getClientVersion () + { + return "1"; + } +}; + +#endif diff --git a/modules/ripple_app/basics/ripple_RPCServerHandler.cpp b/modules/ripple_app/basics/ripple_RPCServerHandler.cpp new file mode 100644 index 0000000000..faa90eb227 --- /dev/null +++ b/modules/ripple_app/basics/ripple_RPCServerHandler.cpp @@ -0,0 +1,98 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +RPCServerHandler::RPCServerHandler (NetworkOPs& networkOPs) + : m_networkOPs (networkOPs) +{ +} + +std::string RPCServerHandler::createResponse ( + int statusCode, + std::string const& description) +{ + return HTTPReply (statusCode, description); +} + +bool RPCServerHandler::isAuthorized ( + std::map const& headers) +{ + return HTTPAuthorized (headers); +} + +std::string RPCServerHandler::processRequest (std::string const& request, std::string const& remoteAddress) +{ + Json::Value jvRequest; + { + Json::Reader reader; + + if (! reader.parse (request, jvRequest) || + jvRequest.isNull () || + ! jvRequest.isObject ()) + { + return createResponse (400, "Unable to parse request"); + } + } + + int role = iAdminGet (jvRequest, remoteAddress); + + // Parse id now so errors from here on will have the id + // + // VFALCO NOTE Except that "id" isn't included in the following errors... + // + Json::Value const id = jvRequest ["id"]; + + Json::Value const method = jvRequest ["method"]; + + if (method.isNull ()) + { + return createResponse (400, "Null method"); + } + else if (! method.isString ()) + { + return createResponse (400, "method is not string"); + } + + std::string strMethod = method.asString (); + + // Parse params + Json::Value params = jvRequest ["params"]; + + if (params.isNull ()) + { + params = Json::Value (Json::arrayValue); + } + else if (!params.isArray ()) + { + return HTTPReply (400, "params unparseable"); + } + + // VFALCO TODO Shouldn't we handle this earlier? + // + if (role == RPCHandler::FORBID) + { + // VFALCO TODO Needs implementing + // FIXME Needs implementing + // XXX This needs rate limiting to prevent brute forcing password. + return HTTPReply (403, "Forbidden"); + } + + std::string response; + + WriteLog (lsINFO, RPCServer) << params; + + RPCHandler rpcHandler (&m_networkOPs); + + LoadType loadType = LT_RPCReference; + + Json::Value const result = rpcHandler.doRpcCommand (strMethod, params, role, &loadType); + // VFALCO NOTE We discard loadType since there is no endpoint to punish + + WriteLog (lsINFO, RPCServer) << result; + + response = JSONRPCReply (result, Json::Value (), id); + + return createResponse (200, response); +} diff --git a/modules/ripple_app/basics/ripple_RPCServerHandler.h b/modules/ripple_app/basics/ripple_RPCServerHandler.h new file mode 100644 index 0000000000..537ff265e4 --- /dev/null +++ b/modules/ripple_app/basics/ripple_RPCServerHandler.h @@ -0,0 +1,27 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_RPCSERVERHANDLER_H_INCLUDED +#define RIPPLE_RPCSERVERHANDLER_H_INCLUDED + +/** Handles RPC requests. +*/ +class RPCServerHandler : public RPCServer::Handler +{ +public: + explicit RPCServerHandler (NetworkOPs& networkOPs); + + std::string createResponse (int statusCode, std::string const& description); + + bool isAuthorized (std::map const& headers); + + std::string processRequest (std::string const& request, std::string const& remoteAddress); + +private: + NetworkOPs& m_networkOPs; +}; + +#endif diff --git a/src/cpp/ripple/ripple_Version.h b/modules/ripple_app/basics/ripple_Version.h similarity index 91% rename from src/cpp/ripple/ripple_Version.h rename to modules/ripple_app/basics/ripple_Version.h index dcd77fe60e..f5fb669e4a 100644 --- a/src/cpp/ripple/ripple_Version.h +++ b/modules/ripple_app/basics/ripple_Version.h @@ -6,13 +6,15 @@ #ifndef RIPPLE_VERSION_H #define RIPPLE_VERSION_H + // // Versions // +// VFALCO TODO Roll this together into ripple_BuildVersion #define SERVER_VERSION_MAJOR 0 #define SERVER_VERSION_MINOR 9 -#define SERVER_VERSION_SUB "-b" +#define SERVER_VERSION_SUB "-c" #define SERVER_NAME "Ripple" #define SV_STRINGIZE(x) SV_STRINGIZE2(x) diff --git a/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.cpp b/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.cpp new file mode 100644 index 0000000000..12a3892378 --- /dev/null +++ b/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.cpp @@ -0,0 +1,152 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#if RIPPLE_HYPERLEVELDB_AVAILABLE + +class HyperLevelDBBackendFactory::Backend : public NodeStore::Backend +{ +public: + Backend (StringPairArray const& keyValues) + : mName(keyValues ["path"].toStdString ()) + , mDB(NULL) + { + if (mName.empty()) + throw std::runtime_error ("Missing path in LevelDB backend"); + + hyperleveldb::Options options; + options.create_if_missing = true; + + if (keyValues["cache_mb"].isEmpty()) + options.block_cache = hyperleveldb::NewLRUCache (theConfig.getSize (siHashNodeDBCache) * 1024 * 1024); + else + options.block_cache = hyperleveldb::NewLRUCache (keyValues["cache_mb"].getIntValue() * 1024L * 1024L); + + if (keyValues["filter_bits"].isEmpty()) + { + if (theConfig.NODE_SIZE >= 2) + options.filter_policy = hyperleveldb::NewBloomFilterPolicy (10); + } + else if (keyValues["filter_bits"].getIntValue() != 0) + options.filter_policy = hyperleveldb::NewBloomFilterPolicy (keyValues["filter_bits"].getIntValue()); + + if (!keyValues["open_files"].isEmpty()) + options.max_open_files = keyValues["open_files"].getIntValue(); + + hyperleveldb::Status status = hyperleveldb::DB::Open (options, mName, &mDB); + if (!status.ok () || !mDB) + throw (std::runtime_error (std::string("Unable to open/create leveldb: ") + status.ToString())); + } + + ~Backend () + { + delete mDB; + } + + std::string getDataBaseName() + { + return mName; + } + + bool bulkStore (const std::vector< NodeObject::pointer >& objs) + { + hyperleveldb::WriteBatch batch; + + BOOST_FOREACH (NodeObject::ref obj, objs) + { + Blob blob (toBlob (obj)); + batch.Put ( + hyperleveldb::Slice (reinterpret_cast(obj->getHash ().begin ()), 256 / 8), + hyperleveldb::Slice (reinterpret_cast(&blob.front ()), blob.size ())); + } + return mDB->Write (hyperleveldb::WriteOptions (), &batch).ok (); + } + + NodeObject::pointer retrieve (uint256 const& hash) + { + std::string sData; + if (!mDB->Get (hyperleveldb::ReadOptions (), + hyperleveldb::Slice (reinterpret_cast(hash.begin ()), 256 / 8), &sData).ok ()) + { + return NodeObject::pointer(); + } + return fromBinary(hash, &sData[0], sData.size ()); + } + + void visitAll (FUNCTION_TYPE func) + { + hyperleveldb::Iterator* it = mDB->NewIterator (hyperleveldb::ReadOptions ()); + for (it->SeekToFirst (); it->Valid (); it->Next ()) + { + if (it->key ().size () == 256 / 8) + { + uint256 hash; + memcpy(hash.begin(), it->key ().data(), 256 / 8); + func (fromBinary (hash, it->value ().data (), it->value ().size ())); + } + } + } + + Blob toBlob(NodeObject::ref obj) + { + Blob rawData (9 + obj->getData ().size ()); + unsigned char* bufPtr = &rawData.front(); + + *reinterpret_cast (bufPtr + 0) = ntohl (obj->getIndex ()); + *reinterpret_cast (bufPtr + 4) = ntohl (obj->getIndex ()); + * (bufPtr + 8) = static_cast (obj->getType ()); + memcpy (bufPtr + 9, &obj->getData ().front (), obj->getData ().size ()); + + return rawData; + } + + NodeObject::pointer fromBinary(uint256 const& hash, + char const* data, int size) + { + if (size < 9) + throw std::runtime_error ("undersized object"); + + uint32 index = htonl (*reinterpret_cast (data)); + int htype = data[8]; + + return boost::make_shared (static_cast (htype), index, + data + 9, size - 9, hash); + } + +private: + std::string mName; + hyperleveldb::DB* mDB; +}; + +//------------------------------------------------------------------------------ + +HyperLevelDBBackendFactory::HyperLevelDBBackendFactory () +{ +} + +HyperLevelDBBackendFactory::~HyperLevelDBBackendFactory () +{ +} + +HyperLevelDBBackendFactory& HyperLevelDBBackendFactory::getInstance () +{ + static HyperLevelDBBackendFactory instance; + + return instance; +} + +String HyperLevelDBBackendFactory::getName () const +{ + return "HyperLevelDB"; +} + +NodeStore::Backend* HyperLevelDBBackendFactory::createInstance (StringPairArray const& keyValues) +{ + return new HyperLevelDBBackendFactory::Backend (keyValues); +} + +//------------------------------------------------------------------------------ + +#endif diff --git a/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.h b/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.h new file mode 100644 index 0000000000..1b44e4f9d1 --- /dev/null +++ b/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.h @@ -0,0 +1,31 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED +#define RIPPLE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED + +#if RIPPLE_HYPERLEVELDB_AVAILABLE + +/** Factory to produce HyperLevelDB backends for the NodeStore. +*/ +class HyperLevelDBBackendFactory : public NodeStore::BackendFactory +{ +private: + class Backend; + + HyperLevelDBBackendFactory (); + ~HyperLevelDBBackendFactory (); + +public: + static HyperLevelDBBackendFactory& getInstance (); + + String getName () const; + NodeStore::Backend* createInstance (StringPairArray const& keyValues); +}; + +#endif + +#endif diff --git a/modules/ripple_app/node/ripple_LevelDBBackendFactory.cpp b/modules/ripple_app/node/ripple_LevelDBBackendFactory.cpp new file mode 100644 index 0000000000..b00fd0f287 --- /dev/null +++ b/modules/ripple_app/node/ripple_LevelDBBackendFactory.cpp @@ -0,0 +1,149 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +class LevelDBBackendFactory::Backend : public NodeStore::Backend +{ +public: + Backend (StringPairArray const& keyValues) + : mName(keyValues ["path"].toStdString ()) + , mDB(NULL) + { + if (mName.empty()) + throw std::runtime_error ("Missing path in LevelDB backend"); + + leveldb::Options options; + options.create_if_missing = true; + + if (keyValues["cache_mb"].isEmpty()) + options.block_cache = leveldb::NewLRUCache (theConfig.getSize (siHashNodeDBCache) * 1024 * 1024); + else + options.block_cache = leveldb::NewLRUCache (keyValues["cache_mb"].getIntValue() * 1024L * 1024L); + + if (keyValues["filter_bits"].isEmpty()) + { + if (theConfig.NODE_SIZE >= 2) + options.filter_policy = leveldb::NewBloomFilterPolicy (10); + } + else if (keyValues["filter_bits"].getIntValue() != 0) + options.filter_policy = leveldb::NewBloomFilterPolicy (keyValues["filter_bits"].getIntValue()); + + if (!keyValues["open_files"].isEmpty()) + options.max_open_files = keyValues["open_files"].getIntValue(); + + leveldb::Status status = leveldb::DB::Open (options, mName, &mDB); + if (!status.ok () || !mDB) + throw (std::runtime_error (std::string("Unable to open/create leveldb: ") + status.ToString())); + } + + ~Backend () + { + delete mDB; + } + + std::string getDataBaseName() + { + return mName; + } + + bool bulkStore (const std::vector< NodeObject::pointer >& objs) + { + leveldb::WriteBatch batch; + + BOOST_FOREACH (NodeObject::ref obj, objs) + { + Blob blob (toBlob (obj)); + batch.Put ( + leveldb::Slice (reinterpret_cast(obj->getHash ().begin ()), 256 / 8), + leveldb::Slice (reinterpret_cast(&blob.front ()), blob.size ())); + } + return mDB->Write (leveldb::WriteOptions (), &batch).ok (); + } + + NodeObject::pointer retrieve (uint256 const& hash) + { + std::string sData; + if (!mDB->Get (leveldb::ReadOptions (), + leveldb::Slice (reinterpret_cast(hash.begin ()), 256 / 8), &sData).ok ()) + { + return NodeObject::pointer(); + } + return fromBinary(hash, &sData[0], sData.size ()); + } + + void visitAll (FUNCTION_TYPE func) + { + leveldb::Iterator* it = mDB->NewIterator (leveldb::ReadOptions ()); + for (it->SeekToFirst (); it->Valid (); it->Next ()) + { + if (it->key ().size () == 256 / 8) + { + uint256 hash; + memcpy(hash.begin(), it->key ().data(), 256 / 8); + func (fromBinary (hash, it->value ().data (), it->value ().size ())); + } + } + } + + Blob toBlob(NodeObject::ref obj) + { + Blob rawData (9 + obj->getData ().size ()); + unsigned char* bufPtr = &rawData.front(); + + *reinterpret_cast (bufPtr + 0) = ntohl (obj->getIndex ()); + *reinterpret_cast (bufPtr + 4) = ntohl (obj->getIndex ()); + * (bufPtr + 8) = static_cast (obj->getType ()); + memcpy (bufPtr + 9, &obj->getData ().front (), obj->getData ().size ()); + + return rawData; + } + + NodeObject::pointer fromBinary(uint256 const& hash, + char const* data, int size) + { + if (size < 9) + throw std::runtime_error ("undersized object"); + + uint32 index = htonl (*reinterpret_cast (data)); + int htype = data[8]; + + return boost::make_shared (static_cast (htype), index, + data + 9, size - 9, hash); + } + +private: + std::string mName; + leveldb::DB* mDB; +}; + +//------------------------------------------------------------------------------ + +LevelDBBackendFactory::LevelDBBackendFactory () +{ +} + +LevelDBBackendFactory::~LevelDBBackendFactory () +{ +} + +LevelDBBackendFactory& LevelDBBackendFactory::getInstance () +{ + static LevelDBBackendFactory instance; + + return instance; +} + +String LevelDBBackendFactory::getName () const +{ + return "LevelDB"; +} + +NodeStore::Backend* LevelDBBackendFactory::createInstance (StringPairArray const& keyValues) +{ + return new LevelDBBackendFactory::Backend (keyValues); +} + +//------------------------------------------------------------------------------ + diff --git a/modules/ripple_app/node/ripple_LevelDBBackendFactory.h b/modules/ripple_app/node/ripple_LevelDBBackendFactory.h new file mode 100644 index 0000000000..b2f324f927 --- /dev/null +++ b/modules/ripple_app/node/ripple_LevelDBBackendFactory.h @@ -0,0 +1,27 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_LEVELDBBACKENDFACTORY_H_INCLUDED +#define RIPPLE_LEVELDBBACKENDFACTORY_H_INCLUDED + +/** Factory to produce LevelDB backends for the NodeStore. +*/ +class LevelDBBackendFactory : public NodeStore::BackendFactory +{ +private: + class Backend; + + LevelDBBackendFactory (); + ~LevelDBBackendFactory (); + +public: + static LevelDBBackendFactory& getInstance (); + + String getName () const; + NodeStore::Backend* createInstance (StringPairArray const& keyValues); +}; + +#endif diff --git a/modules/ripple_app/node/ripple_MdbBackendFactory.cpp b/modules/ripple_app/node/ripple_MdbBackendFactory.cpp new file mode 100644 index 0000000000..0b74349ab3 --- /dev/null +++ b/modules/ripple_app/node/ripple_MdbBackendFactory.cpp @@ -0,0 +1,205 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#if RIPPLE_MDB_AVAILABLE + +class MdbBackendFactory::Backend : public NodeStore::Backend +{ +public: + explicit Backend (StringPairArray const& keyValues) + : m_env (nullptr) + { + if (keyValues ["path"].isEmpty ()) + throw std::runtime_error ("Missing path in MDB backend"); + + int error = 0; + + error = mdb_env_create (&m_env); + + if (error == 0) // Should use the size of the file plus the free space on the disk + error = mdb_env_set_mapsize(m_env, 512L * 1024L * 1024L * 1024L); + + if (error == 0) + error = mdb_env_open ( + m_env, + keyValues ["path"].toStdString().c_str (), + MDB_NOTLS, + 0664); + + MDB_txn * txn; + if (error == 0) + error = mdb_txn_begin(m_env, NULL, 0, &txn); + if (error == 0) + error = mdb_dbi_open(txn, NULL, 0, &m_dbi); + if (error == 0) + error = mdb_txn_commit(txn); + + + if (error != 0) + { + String s; + s << "Error #" << error << " creating mdb environment"; + throw std::runtime_error (s.toStdString ()); + } + m_name = keyValues ["path"].toStdString(); + } + + ~Backend () + { + if (m_env != nullptr) + { + mdb_dbi_close(m_env, m_dbi); + mdb_env_close (m_env); + } + } + + std::string getDataBaseName() + { + return m_name; + } + + bool bulkStore (std::vector const& objs) + { + MDB_txn *txn = nullptr; + int rc = 0; + + rc = mdb_txn_begin(m_env, NULL, 0, &txn); + + if (rc == 0) + { + BOOST_FOREACH (NodeObject::ref obj, objs) + { + MDB_val key, data; + Blob blob (toBlob (obj)); + + key.mv_size = (256 / 8); + key.mv_data = const_cast(obj->getHash().begin()); + + data.mv_size = blob.size(); + data.mv_data = &blob.front(); + + rc = mdb_put(txn, m_dbi, &key, &data, 0); + if (rc != 0) + { + assert(false); + break; + } + } + } + else + assert(false); + + if (rc == 0) + rc = mdb_txn_commit(txn); + else if (txn) + mdb_txn_abort(txn); + + assert(rc == 0); + return rc == 0; + } + + NodeObject::pointer retrieve (uint256 const& hash) + { + NodeObject::pointer ret; + + MDB_txn *txn = nullptr; + int rc = 0; + + rc = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &txn); + + if (rc == 0) + { + MDB_val key, data; + + key.mv_size = (256 / 8); + key.mv_data = const_cast(hash.begin()); + + rc = mdb_get(txn, m_dbi, &key, &data); + if (rc == 0) + ret = fromBinary(hash, static_cast(data.mv_data), data.mv_size); + else + assert(rc == MDB_NOTFOUND); + } + else + assert(false); + + mdb_txn_abort(txn); + + return ret; + } + + void visitAll (FUNCTION_TYPE func) + { // WRITEME + assert(false); + } + + Blob toBlob (NodeObject::ref obj) const + { + Blob rawData (9 + obj->getData ().size ()); + unsigned char* bufPtr = &rawData.front(); + + *reinterpret_cast (bufPtr + 0) = ntohl (obj->getIndex ()); + + *reinterpret_cast (bufPtr + 4) = ntohl (obj->getIndex ()); + + *(bufPtr + 8) = static_cast (obj->getType ()); + + memcpy (bufPtr + 9, &obj->getData ().front (), obj->getData ().size ()); + + return rawData; + } + + NodeObject::pointer fromBinary (uint256 const& hash, char const* data, int size) const + { + if (size < 9) + throw std::runtime_error ("undersized object"); + + uint32 const index = htonl (*reinterpret_cast (data)); + + int const htype = data [8]; + + return boost::make_shared ( + static_cast (htype), + index, + data + 9, + size - 9, + hash); + } + +private: + std::string m_name; + MDB_env* m_env; + MDB_dbi m_dbi; +}; + +//------------------------------------------------------------------------------ + +MdbBackendFactory::MdbBackendFactory () +{ +} + +MdbBackendFactory::~MdbBackendFactory () +{ +} + +MdbBackendFactory& MdbBackendFactory::getInstance () +{ + static MdbBackendFactory instance; + + return instance; +} + +String MdbBackendFactory::getName () const +{ + return "mdb"; +} + +NodeStore::Backend* MdbBackendFactory::createInstance (StringPairArray const& keyValues) +{ + return new MdbBackendFactory::Backend (keyValues); +} + +#endif diff --git a/modules/ripple_app/node/ripple_MdbBackendFactory.h b/modules/ripple_app/node/ripple_MdbBackendFactory.h new file mode 100644 index 0000000000..702ca3a14a --- /dev/null +++ b/modules/ripple_app/node/ripple_MdbBackendFactory.h @@ -0,0 +1,33 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_MDBBACKENDFACTORY_H_INCLUDED +#define RIPPLE_MDBBACKENDFACTORY_H_INCLUDED + +#if RIPPLE_MDB_AVAILABLE + +/** Factory to produce a backend using MDB. + + @note MDB is not currently available for Win32 +*/ +class MdbBackendFactory : public NodeStore::BackendFactory +{ +private: + class Backend; + + MdbBackendFactory (); + ~MdbBackendFactory (); + +public: + static MdbBackendFactory& getInstance (); + + String getName () const; + NodeStore::Backend* createInstance (StringPairArray const& keyValues); +}; + +#endif + +#endif diff --git a/modules/ripple_app/node/ripple_MemoryBackendFactory.cpp b/modules/ripple_app/node/ripple_MemoryBackendFactory.cpp new file mode 100644 index 0000000000..d0536b3505 --- /dev/null +++ b/modules/ripple_app/node/ripple_MemoryBackendFactory.cpp @@ -0,0 +1,101 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +class MemoryBackendFactory::Backend : public NodeStore::Backend +{ +private: + typedef std::map Map; + +public: + Backend (size_t keyBytes, StringPairArray const& keyValues) + : m_keyBytes (keyBytes) + { + } + + ~Backend () + { + } + + std::string getDataBaseName () + { + return "memory"; + } + + //-------------------------------------------------------------------------- + + NodeObject::pointer retrieve (uint256 const &hash) + { + Map::iterator iter = m_map.find (hash); + + if (iter != m_map.end ()) + return iter->second; + + return NodeObject::pointer (); + } + + bool store (NodeObject::ref object) + { + Map::iterator iter = m_map.find (object->getHash ()); + + if (iter == m_map.end ()) + { + m_map.insert (std::make_pair (object->getHash (), object)); + } + + return true; + } + + bool bulkStore (const std::vector< NodeObject::pointer >& batch) + { + for (int i = 0; i < batch.size (); ++i) + store (batch [i]); + + return true; + } + + void visitAll (FUNCTION_TYPE f) + { + for (Map::const_iterator iter = m_map.begin (); iter != m_map.end (); ++iter) + f (iter->second); + } + + //-------------------------------------------------------------------------- + +private: + size_t const m_keyBytes; + + Map m_map; +}; + +//------------------------------------------------------------------------------ + +MemoryBackendFactory::MemoryBackendFactory () +{ +} + +MemoryBackendFactory::~MemoryBackendFactory () +{ +} + +MemoryBackendFactory& MemoryBackendFactory::getInstance () +{ + static MemoryBackendFactory instance; + + return instance; +} + +String MemoryBackendFactory::getName () const +{ + return "Memory"; +} + +NodeStore::Backend* MemoryBackendFactory::createInstance (StringPairArray const& keyValues) +{ + return new MemoryBackendFactory::Backend (32, keyValues); +} + +//------------------------------------------------------------------------------ + diff --git a/modules/ripple_app/node/ripple_MemoryBackendFactory.h b/modules/ripple_app/node/ripple_MemoryBackendFactory.h new file mode 100644 index 0000000000..5bb4cf2700 --- /dev/null +++ b/modules/ripple_app/node/ripple_MemoryBackendFactory.h @@ -0,0 +1,30 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_MEMORYBACKENDFACTORY_H_INCLUDED +#define RIPPLE_MEMORYBACKENDFACTORY_H_INCLUDED + +/** Factory to produce a RAM based backend for the NodeStore. + + @see NodeStore +*/ +class MemoryBackendFactory : public NodeStore::BackendFactory +{ +private: + class Backend; + + MemoryBackendFactory (); + ~MemoryBackendFactory (); + +public: + static MemoryBackendFactory& getInstance (); + + String getName () const; + + NodeStore::Backend* createInstance (StringPairArray const& keyValues); +}; + +#endif diff --git a/src/cpp/ripple/ripple_HashedObject.cpp b/modules/ripple_app/node/ripple_NodeObject.cpp similarity index 72% rename from src/cpp/ripple/ripple_HashedObject.cpp rename to modules/ripple_app/node/ripple_NodeObject.cpp index 1efc6b7390..ac8bce22ee 100644 --- a/src/cpp/ripple/ripple_HashedObject.cpp +++ b/modules/ripple_app/node/ripple_NodeObject.cpp @@ -4,10 +4,10 @@ */ //============================================================================== -SETUP_LOG (HashedObject) +SETUP_LOG (NodeObject) -HashedObject::HashedObject ( - HashedObjectType type, +NodeObject::NodeObject ( + NodeObjectType type, LedgerIndex ledgerIndex, Blob const& binaryDataToCopy, uint256 const& hash) @@ -18,8 +18,8 @@ HashedObject::HashedObject ( { } -HashedObject::HashedObject ( - HashedObjectType type, +NodeObject::NodeObject ( + NodeObjectType type, LedgerIndex ledgerIndex, void const* bufferToCopy, int bytesInBuffer, @@ -32,22 +32,22 @@ HashedObject::HashedObject ( { } -HashedObjectType HashedObject::getType () const +NodeObjectType NodeObject::getType () const { return mType; } -uint256 const& HashedObject::getHash () const +uint256 const& NodeObject::getHash () const { return mHash; } -LedgerIndex HashedObject::getIndex () const +LedgerIndex NodeObject::getIndex () const { return mLedgerIndex; } -Blob const& HashedObject::getData () const +Blob const& NodeObject::getData () const { return mData; } diff --git a/src/cpp/ripple/ripple_HashedObject.h b/modules/ripple_app/node/ripple_NodeObject.h similarity index 76% rename from src/cpp/ripple/ripple_HashedObject.h rename to modules/ripple_app/node/ripple_NodeObject.h index 5760f27599..b889666f48 100644 --- a/src/cpp/ripple/ripple_HashedObject.h +++ b/modules/ripple_app/node/ripple_NodeObject.h @@ -4,12 +4,12 @@ */ //============================================================================== -#ifndef RIPPLE_HASHEDOBJECT_H -#define RIPPLE_HASHEDOBJECT_H +#ifndef RIPPLE_NODEOBJECT_H_INCLUDED +#define RIPPLE_NODEOBJECT_H_INCLUDED -/** The types of hashed objects. +/** The types of node objects. */ -enum HashedObjectType +enum NodeObjectType { hotUNKNOWN = 0, hotLEDGER = 1, @@ -29,21 +29,19 @@ enum HashedObjectType @note No checking is performed to make sure the hash matches the data. @see SHAMap */ -// VFALCO TODO consider making the instance a private member of SHAMap -// since its the primary user. -// -class HashedObject - : public CountedObject +class NodeObject : public CountedObject { public: - typedef boost::shared_ptr pointer; + static char const* getCountedObjectName () { return "NodeObject"; } + + typedef boost::shared_ptr pointer; typedef pointer const& ref; /** Create from a vector of data. @note A copy of the data is created. */ - HashedObject (HashedObjectType type, + NodeObject (NodeObjectType type, LedgerIndex ledgerIndex, Blob const & binaryDataToCopy, uint256 const & hash); @@ -52,7 +50,7 @@ public: @note A copy of the data is created. */ - HashedObject (HashedObjectType type, + NodeObject (NodeObjectType type, LedgerIndex ledgerIndex, void const * bufferToCopy, int bytesInBuffer, @@ -60,7 +58,7 @@ public: /** Retrieve the type of this object. */ - HashedObjectType getType () const; + NodeObjectType getType () const; /** Retrieve the hash metadata. */ @@ -76,11 +74,10 @@ public: Blob const& getData () const; private: - HashedObjectType const mType; + NodeObjectType const mType; uint256 const mHash; LedgerIndex const mLedgerIndex; Blob const mData; }; #endif -// vim:ts=4 diff --git a/modules/ripple_app/node/ripple_NodeStore.cpp b/modules/ripple_app/node/ripple_NodeStore.cpp new file mode 100644 index 0000000000..960e0d805f --- /dev/null +++ b/modules/ripple_app/node/ripple_NodeStore.cpp @@ -0,0 +1,247 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +Array NodeStore::s_factories; + +NodeStore::NodeStore (String backendParameters, String fastBackendParameters, int cacheSize, int cacheAge) + : m_backend (createBackend (backendParameters)) + , mCache ("NodeStore", cacheSize, cacheAge) + , mNegativeCache ("HashedObjectNegativeCache", 0, 120) +{ + if (fastBackendParameters.isNotEmpty ()) + m_fastBackend = createBackend (fastBackendParameters); +} + +void NodeStore::addBackendFactory (BackendFactory& factory) +{ + s_factories.add (&factory); +} + +float NodeStore::getCacheHitRate () +{ + return mCache.getHitRate (); +} + +void NodeStore::tune (int size, int age) +{ + mCache.setTargetSize (size); + mCache.setTargetAge (age); +} + +void NodeStore::sweep () +{ + mCache.sweep (); + mNegativeCache.sweep (); +} + +void NodeStore::waitWrite () +{ + m_backend->waitWrite (); + if (m_fastBackend) + m_fastBackend->waitWrite (); +} + +int NodeStore::getWriteLoad () +{ + return m_backend->getWriteLoad (); +} + +bool NodeStore::store (NodeObjectType type, uint32 index, + Blob const& data, uint256 const& hash) +{ + // return: false = already in cache, true = added to cache + if (mCache.touch (hash)) + return false; + +#ifdef PARANOID + assert (hash == Serializer::getSHA512Half (data)); +#endif + + NodeObject::pointer object = boost::make_shared (type, index, data, hash); + + if (!mCache.canonicalize (hash, object)) + { + m_backend->store (object); + if (m_fastBackend) + m_fastBackend->store (object); + } + + mNegativeCache.del (hash); + return true; +} + +NodeObject::pointer NodeStore::retrieve (uint256 const& hash) +{ + NodeObject::pointer obj = mCache.fetch (hash); + + if (obj || mNegativeCache.isPresent (hash)) + return obj; + + if (m_fastBackend) + { + obj = m_fastBackend->retrieve (hash); + + if (obj) + { + mCache.canonicalize (hash, obj); + return obj; + } + } + + { + LoadEvent::autoptr event (getApp().getJobQueue ().getLoadEventAP (jtHO_READ, "HOS::retrieve")); + obj = m_backend->retrieve(hash); + + if (!obj) + { + mNegativeCache.add (hash); + return obj; + } + } + + mCache.canonicalize (hash, obj); + + if (m_fastBackend) + m_fastBackend->store(obj); + + WriteLog (lsTRACE, NodeObject) << "HOS: " << hash << " fetch: in db"; + return obj; +} + +void NodeStore::importVisitor ( + std::vector & objects, + NodeObject::pointer object) +{ + if (objects.size() >= 128) + { + m_backend->bulkStore (objects); + + objects.clear (); + objects.reserve (128); + } + + objects.push_back (object); +} + +int NodeStore::import (String sourceBackendParameters) +{ + ScopedPointer srcBackend (createBackend (sourceBackendParameters)); + + WriteLog (lsWARNING, NodeObject) << + "Node import from '" << srcBackend->getDataBaseName() << "' to '" + << m_backend->getDataBaseName() << "'."; + + std::vector objects; + + objects.reserve (128); + + srcBackend->visitAll (BIND_TYPE (&NodeStore::importVisitor, this, boost::ref (objects), P_1)); + + if (!objects.empty ()) + m_backend->bulkStore (objects); + + return 0; +} + +NodeStore::Backend* NodeStore::createBackend (String const& parameters) +{ + Backend* backend = nullptr; + + StringPairArray keyValues = parseKeyValueParameters (parameters, '|'); + + String const& type = keyValues ["type"]; + + if (type.isNotEmpty ()) + { + BackendFactory* factory = nullptr; + + for (int i = 0; i < s_factories.size (); ++i) + { + if (s_factories [i]->getName () == type) + { + factory = s_factories [i]; + break; + } + } + + if (factory != nullptr) + { + backend = factory->createInstance (keyValues); + } + else + { + throw std::runtime_error ("unkown backend type"); + } + } + else + { + throw std::runtime_error ("missing backend type"); + } + + return backend; +} + +bool NodeStore::Backend::store (NodeObject::ref object) +{ + boost::mutex::scoped_lock sl (mWriteMutex); + mWriteSet.push_back (object); + + if (!mWritePending) + { + mWritePending = true; + getApp().getJobQueue ().addJob (jtWRITE, "NodeObject::store", + BIND_TYPE (&NodeStore::Backend::bulkWrite, this, P_1)); + } + return true; +} + +void NodeStore::Backend::bulkWrite (Job &) +{ + int setSize = 0; + + while (1) + { + std::vector< boost::shared_ptr > set; + set.reserve (128); + + { + boost::mutex::scoped_lock sl (mWriteMutex); + + mWriteSet.swap (set); + assert (mWriteSet.empty ()); + ++mWriteGeneration; + mWriteCondition.notify_all (); + + if (set.empty ()) + { + mWritePending = false; + mWriteLoad = 0; + return; + } + + mWriteLoad = std::max (setSize, static_cast (mWriteSet.size ())); + setSize = set.size (); + } + + bulkStore (set); + } +} + +void NodeStore::Backend::waitWrite () +{ + boost::mutex::scoped_lock sl (mWriteMutex); + int gen = mWriteGeneration; + + while (mWritePending && (mWriteGeneration == gen)) + mWriteCondition.wait (sl); +} + +int NodeStore::Backend::getWriteLoad () +{ + boost::mutex::scoped_lock sl (mWriteMutex); + + return std::max (mWriteLoad, static_cast (mWriteSet.size ())); +} diff --git a/modules/ripple_app/node/ripple_NodeStore.h b/modules/ripple_app/node/ripple_NodeStore.h new file mode 100644 index 0000000000..8e6de63fcc --- /dev/null +++ b/modules/ripple_app/node/ripple_NodeStore.h @@ -0,0 +1,132 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_H_INCLUDED +#define RIPPLE_NODESTORE_H_INCLUDED + +/** Persistency layer for NodeObject +*/ +class NodeStore : LeakChecked +{ +public: + /** Back end used for the store. + */ + class Backend + { + public: + // VFALCO TODO Move the function definition to the .cpp + Backend () + : mWriteGeneration(0) + , mWriteLoad(0) + , mWritePending(false) + { + mWriteSet.reserve(128); + } + + virtual ~Backend () { } + + virtual std::string getDataBaseName() = 0; + + // Store/retrieve a single object + // These functions must be thread safe + virtual bool store (NodeObject::ref); + virtual NodeObject::pointer retrieve (uint256 const &hash) = 0; + + // Store a group of objects + // This function will only be called from a single thread + virtual bool bulkStore (const std::vector< NodeObject::pointer >&) = 0; + + // Visit every object in the database + // This function will only be called during an import operation + virtual void visitAll (FUNCTION_TYPE ) = 0; + + // VFALCO TODO Put this bulk writing logic into a separate class. + virtual void bulkWrite (Job &); + virtual void waitWrite (); + virtual int getWriteLoad (); + + protected: + // VFALCO TODO Put this bulk writing logic into a separate class. + boost::mutex mWriteMutex; + boost::condition_variable mWriteCondition; + int mWriteGeneration; + int mWriteLoad; + bool mWritePending; + std::vector > mWriteSet; + }; + +public: + /** Factory to produce backends. + */ + class BackendFactory + { + public: + virtual ~BackendFactory () { } + + /** Retrieve the name of this factory. + */ + virtual String getName () const = 0; + + /** Create an instance of this factory's backend. + */ + virtual Backend* createInstance (StringPairArray const& keyValues) = 0; + }; + +public: + /** Construct a node store. + + parameters has the format: + + =['|'=] + + The key "type" must exist, it defines the backend. For example + "type=LevelDB|path=/mnt/ephemeral" + */ + // VFALCO NOTE Is cacheSize in bytes? objects? KB? + // Is cacheAge in minutes? seconds? + // + NodeStore (String backendParameters, + String fastBackendParameters, + int cacheSize, + int cacheAge); + + /** Add the specified backend factory to the list of available factories. + + The names of available factories are compared against the "type" + value in the parameter list on construction. + */ + static void addBackendFactory (BackendFactory& factory); + + float getCacheHitRate (); + + bool store (NodeObjectType type, uint32 index, Blob const& data, + uint256 const& hash); + + NodeObject::pointer retrieve (uint256 const& hash); + + void waitWrite (); + void tune (int size, int age); + void sweep (); + int getWriteLoad (); + + int import (String sourceBackendParameters); + +private: + void importVisitor (std::vector & objects, NodeObject::pointer object); + + static Backend* createBackend (String const& parameters); + + static Array s_factories; + +private: + ScopedPointer m_backend; + ScopedPointer m_fastBackend; + + TaggedCache mCache; + KeyCache mNegativeCache; +}; + +#endif diff --git a/modules/ripple_app/node/ripple_NullBackendFactory.cpp b/modules/ripple_app/node/ripple_NullBackendFactory.cpp new file mode 100644 index 0000000000..6ffb0d8299 --- /dev/null +++ b/modules/ripple_app/node/ripple_NullBackendFactory.cpp @@ -0,0 +1,71 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +class NullBackendFactory::Backend : public NodeStore::Backend +{ +public: + Backend () + { + } + + ~Backend () + { + } + + std::string getDataBaseName() + { + return std::string (); + } + + bool store (NodeObject::ref obj) + { + return false; + } + + bool bulkStore (const std::vector< NodeObject::pointer >& objs) + { + return false; + } + + NodeObject::pointer retrieve (uint256 const& hash) + { + return NodeObject::pointer (); + } + + void visitAll (FUNCTION_TYPE func) + { + } +}; + +//------------------------------------------------------------------------------ + +NullBackendFactory::NullBackendFactory () +{ +} + +NullBackendFactory::~NullBackendFactory () +{ +} + +NullBackendFactory& NullBackendFactory::getInstance () +{ + static NullBackendFactory instance; + + return instance; +} + +String NullBackendFactory::getName () const +{ + return "none"; +} + +NodeStore::Backend* NullBackendFactory::createInstance (StringPairArray const& keyValues) +{ + return new NullBackendFactory::Backend; +} + +//------------------------------------------------------------------------------ + diff --git a/modules/ripple_app/node/ripple_NullBackendFactory.h b/modules/ripple_app/node/ripple_NullBackendFactory.h new file mode 100644 index 0000000000..7112473384 --- /dev/null +++ b/modules/ripple_app/node/ripple_NullBackendFactory.h @@ -0,0 +1,29 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NULLBACKENDFACTORY_H_INCLUDED +#define RIPPLE_NULLBACKENDFACTORY_H_INCLUDED + +/** Factory to produce a null backend. + + This is for standalone / testing mode. +*/ +class NullBackendFactory : public NodeStore::BackendFactory +{ +private: + class Backend; + + NullBackendFactory (); + ~NullBackendFactory (); + +public: + static NullBackendFactory& getInstance (); + + String getName () const; + NodeStore::Backend* createInstance (StringPairArray const& keyValues); +}; + +#endif diff --git a/modules/ripple_app/node/ripple_SqliteBackendFactory.cpp b/modules/ripple_app/node/ripple_SqliteBackendFactory.cpp new file mode 100644 index 0000000000..0b421ac5be --- /dev/null +++ b/modules/ripple_app/node/ripple_SqliteBackendFactory.cpp @@ -0,0 +1,153 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +class SqliteBackendFactory::Backend : public NodeStore::Backend +{ +public: + Backend(std::string const& path) : mName(path) + { + mDb = new DatabaseCon(path, HashNodeDBInit, HashNodeDBCount); + mDb->getDB()->executeSQL(boost::str(boost::format("PRAGMA cache_size=-%d;") % + (theConfig.getSize(siHashNodeDBCache) * 1024))); + } + + Backend() + { + delete mDb; + } + + std::string getDataBaseName() + { + return mName; + } + + bool bulkStore(const std::vector< NodeObject::pointer >& objects) + { + ScopedLock sl(mDb->getDBLock()); + static SqliteStatement pStB(mDb->getDB()->getSqliteDB(), "BEGIN TRANSACTION;"); + static SqliteStatement pStE(mDb->getDB()->getSqliteDB(), "END TRANSACTION;"); + static SqliteStatement pSt(mDb->getDB()->getSqliteDB(), + "INSERT OR IGNORE INTO CommittedObjects " + "(Hash,ObjType,LedgerIndex,Object) VALUES (?, ?, ?, ?);"); + + pStB.step(); + pStB.reset(); + + BOOST_FOREACH(NodeObject::ref object, objects) + { + bind(pSt, object); + pSt.step(); + pSt.reset(); + } + + pStE.step(); + pStE.reset(); + + return true; + + } + + NodeObject::pointer retrieve(uint256 const& hash) + { + NodeObject::pointer ret; + + { + ScopedLock sl(mDb->getDBLock()); + static SqliteStatement pSt(mDb->getDB()->getSqliteDB(), + "SELECT ObjType,LedgerIndex,Object FROM CommittedObjects WHERE Hash = ?;"); + + pSt.bind(1, hash.GetHex()); + + if (pSt.isRow(pSt.step())) + ret = boost::make_shared(getType(pSt.peekString(0)), pSt.getUInt32(1), pSt.getBlob(2), hash); + + pSt.reset(); + } + + return ret; + } + + void visitAll(FUNCTION_TYPE func) + { + uint256 hash; + + static SqliteStatement pSt(mDb->getDB()->getSqliteDB(), + "SELECT ObjType,LedgerIndex,Object,Hash FROM CommittedObjects;"); + + while (pSt.isRow(pSt.step())) + { + hash.SetHexExact(pSt.getString(3)); + func(boost::make_shared(getType(pSt.peekString(0)), pSt.getUInt32(1), pSt.getBlob(2), hash)); + } + + pSt.reset(); + } + + void bind(SqliteStatement& statement, NodeObject::ref object) + { + char const* type; + switch (object->getType()) + { + case hotLEDGER: type = "L"; break; + case hotTRANSACTION: type = "T"; break; + case hotACCOUNT_NODE: type = "A"; break; + case hotTRANSACTION_NODE: type = "N"; break; + default: type = "U"; + } + + statement.bind(1, object->getHash().GetHex()); + statement.bind(2, type); + statement.bind(3, object->getIndex()); + statement.bindStatic(4, object->getData()); + } + + NodeObjectType getType(std::string const& type) + { + NodeObjectType htype = hotUNKNOWN; + if (!type.empty()) + { + switch (type[0]) + { + case 'L': htype = hotLEDGER; break; + case 'T': htype = hotTRANSACTION; break; + case 'A': htype = hotACCOUNT_NODE; break; + case 'N': htype = hotTRANSACTION_NODE; break; + } + } + return htype; + } + +private: + std::string mName; + DatabaseCon* mDb; +}; + +//------------------------------------------------------------------------------ + +SqliteBackendFactory::SqliteBackendFactory () +{ +} + +SqliteBackendFactory::~SqliteBackendFactory () +{ +} + +SqliteBackendFactory& SqliteBackendFactory::getInstance () +{ + static SqliteBackendFactory instance; + + return instance; +} + +String SqliteBackendFactory::getName () const +{ + return "Sqlite"; +} + +NodeStore::Backend* SqliteBackendFactory::createInstance (StringPairArray const& keyValues) +{ + return new Backend (keyValues ["path"].toStdString ()); +} diff --git a/modules/ripple_app/node/ripple_SqliteBackendFactory.h b/modules/ripple_app/node/ripple_SqliteBackendFactory.h new file mode 100644 index 0000000000..e6420cbde2 --- /dev/null +++ b/modules/ripple_app/node/ripple_SqliteBackendFactory.h @@ -0,0 +1,27 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_SQLITEBACKENDFACTORY_H_INCLUDED +#define RIPPLE_SQLITEBACKENDFACTORY_H_INCLUDED + +/** Factory to produce SQLite backends for the NodeStore. +*/ +class SqliteBackendFactory : public NodeStore::BackendFactory +{ +private: + class Backend; + + SqliteBackendFactory (); + ~SqliteBackendFactory (); + +public: + static SqliteBackendFactory& getInstance (); + + String getName () const; + NodeStore::Backend* createInstance (StringPairArray const& keyValues); +}; + +#endif diff --git a/modules/ripple_app/ripple_app.cpp b/modules/ripple_app/ripple_app.cpp index 6cf4dfd666..b263b583dc 100644 --- a/modules/ripple_app/ripple_app.cpp +++ b/modules/ripple_app/ripple_app.cpp @@ -12,21 +12,14 @@ //------------------------------------------------------------------------------ -//------------------------------------------------------------------------------ +#include "BeastConfig.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +// This must come first to work around the boost placeholders issues +#include "beast/modules/beast_basics/beast_basics.h" + +#if BEAST_LINUX || BEAST_MAC || BEAST_BSD +#include +#endif // VFALCO NOTE Holy smokes...that's a lot of boost!!! #include @@ -40,11 +33,9 @@ #include #include #include -#include #include #include #include -#include #include #include #include @@ -70,24 +61,9 @@ #include #include -#if ! defined (RIPPLE_MAIN_PART) || RIPPLE_MAIN_PART == 1 -#include -#endif +#include "../ripple_sqlite/ripple_sqlite.h" // for SqliteDatabase.cpp -#include - -#include -#include -#include -#include -#include -#include - - - -#include "../modules/ripple_sqlite/ripple_sqlite.h" // for SqliteDatabase.cpp - -#include "../modules/ripple_core/ripple_core.h" +#include "../ripple_core/ripple_core.h" // VFALCO TODO fix these warnings! #ifdef _MSC_VER @@ -102,21 +78,36 @@ #include "ripple_app.h" #include "../ripple_data/ripple_data.h" - - +#include "../ripple_mdb/ripple_mdb.h" +#include "../ripple_leveldb/ripple_leveldb.h" +#include "../ripple_hyperleveldb/ripple_hyperleveldb.h" +#include "../ripple_net/ripple_net.h" #include "../modules/ripple_websocket/ripple_websocket.h" //------------------------------------------------------------------------------ +namespace ripple +{ + // VFALCO NOTE The order of these includes is critical, since they do not // include their own dependencies. This is what allows us to // linearize the include sequence and view it in one place. // -// VFALCO BEGIN CLEAN AREA These are all include-stripped +#include "src/cpp/ripple/ripple_Database.h" +#include "src/cpp/ripple/ripple_DatabaseCon.h" +#include "src/cpp/ripple/ripple_SqliteDatabase.h" +#include "src/cpp/ripple/ripple_DBInit.h" -#include "src/cpp/ripple/ripple_HashedObject.h" +#include "node/ripple_NodeObject.h" +#include "node/ripple_NodeStore.h" +#include "node/ripple_LevelDBBackendFactory.h" +#include "node/ripple_MemoryBackendFactory.h" +#include "node/ripple_HyperLevelDBBackendFactory.h" +#include "node/ripple_MdbBackendFactory.h" +#include "node/ripple_NullBackendFactory.h" +#include "node/ripple_SqliteBackendFactory.h" #include "src/cpp/ripple/ripple_SHAMapItem.h" #include "src/cpp/ripple/ripple_SHAMapNode.h" @@ -125,7 +116,6 @@ #include "src/cpp/ripple/ripple_SHAMapSyncFilter.h" #include "src/cpp/ripple/ripple_SHAMapAddNode.h" #include "src/cpp/ripple/ripple_SHAMap.h" - #include "src/cpp/ripple/ripple_SerializedTransaction.h" #include "src/cpp/ripple/ripple_SerializedLedger.h" #include "src/cpp/ripple/TransactionMeta.h" @@ -133,56 +123,28 @@ #include "src/cpp/ripple/ripple_AccountState.h" #include "src/cpp/ripple/ripple_NicknameState.h" #include "src/cpp/ripple/Ledger.h" - #include "src/cpp/ripple/SerializedValidation.h" #include "src/cpp/ripple/ripple_ILoadManager.h" - -// These have few dependencies -#include "src/cpp/ripple/ripple_DatabaseCon.h" #include "src/cpp/ripple/ripple_ProofOfWork.h" #include "src/cpp/ripple/ripple_InfoSub.h" -#include "src/cpp/ripple/ripple_HashedObject.h" -#include "src/cpp/ripple/ripple_HashedObjectStore.h" #include "src/cpp/ripple/ripple_OrderBook.h" #include "src/cpp/ripple/ripple_SHAMapSyncFilters.h" - -// Abstract interfaces #include "src/cpp/ripple/ripple_IFeatures.h" #include "src/cpp/ripple/ripple_IFeeVote.h" #include "src/cpp/ripple/ripple_IHashRouter.h" #include "src/cpp/ripple/ripple_Peer.h" // VFALCO TODO Rename to IPeer #include "src/cpp/ripple/ripple_IPeers.h" #include "src/cpp/ripple/ripple_IProofOfWorkFactory.h" -#include "src/cpp/ripple/ripple_IUniqueNodeList.h" +#include "src/cpp/ripple/ripple_ClusterNodeStatus.h" +#include "src/cpp/ripple/ripple_UniqueNodeList.h" #include "src/cpp/ripple/ripple_IValidations.h" - #include "src/cpp/ripple/ripple_PeerSet.h" #include "src/cpp/ripple/ripple_InboundLedger.h" #include "src/cpp/ripple/ripple_InboundLedgers.h" - -#include "src/cpp/ripple/ripple_Database.h" -#include "src/cpp/ripple/ripple_SqliteDatabase.h" - -// VFALCO END CLEAN AREA - -//------------------------------------------------------------------------------ - -// VFALCO NOTE Order matters! If you get compile errors, move just 1 -// include upwards as little as possible to fix it. -// #include "src/cpp/ripple/ScriptData.h" #include "src/cpp/ripple/Contract.h" #include "src/cpp/ripple/Interpreter.h" #include "src/cpp/ripple/Operation.h" -// VFALCO NOTE Order matters - -// ----------- -// VFALCO NOTE These have all been include-stripped -// ORDER MATTERS A LOT! - - - - #include "src/cpp/ripple/ripple_AccountItem.h" #include "src/cpp/ripple/ripple_AccountItems.h" #include "src/cpp/ripple/ripple_AcceptedLedgerTx.h" @@ -190,31 +152,21 @@ #include "src/cpp/ripple/ripple_LedgerEntrySet.h" #include "src/cpp/ripple/TransactionEngine.h" #include "src/cpp/ripple/ripple_CanonicalTXSet.h" - #include "src/cpp/ripple/ripple_LedgerHistory.h" #include "src/cpp/ripple/LedgerMaster.h" - #include "src/cpp/ripple/LedgerProposal.h" #include "src/cpp/ripple/NetworkOPs.h" - -// -// ----------- - #include "src/cpp/ripple/TransactionMaster.h" #include "src/cpp/ripple/ripple_LocalCredentials.h" #include "src/cpp/ripple/WSDoor.h" -#include "src/cpp/ripple/SNTPClient.h" #include "src/cpp/ripple/RPCHandler.h" #include "src/cpp/ripple/TransactionQueue.h" #include "src/cpp/ripple/OrderBookDB.h" #include "src/cpp/ripple/ripple_DatabaseCon.h" - #include "src/cpp/ripple/ripple_IApplication.h" #include "src/cpp/ripple/CallRPC.h" #include "src/cpp/ripple/Transactor.h" #include "src/cpp/ripple/ChangeTransactor.h" -#include "src/cpp/ripple/HTTPRequest.h" -#include "src/cpp/ripple/HttpsClient.h" #include "src/cpp/ripple/ripple_TransactionAcquire.h" #include "src/cpp/ripple/ripple_DisputedTx.h" #include "src/cpp/ripple/ripple_LedgerConsensus.h" @@ -224,17 +176,13 @@ #include "src/cpp/ripple/OfferCreateTransactor.h" #include "src/cpp/ripple/ripple_PathRequest.h" #include "src/cpp/ripple/ParameterTable.h" - #include "src/cpp/ripple/ripple_RippleLineCache.h" #include "src/cpp/ripple/ripple_PathState.h" #include "src/cpp/ripple/ripple_RippleCalc.h" #include "src/cpp/ripple/ripple_Pathfinder.h" - #include "src/cpp/ripple/PaymentTransactor.h" #include "src/cpp/ripple/PeerDoor.h" #include "src/cpp/ripple/RPC.h" -#include "src/cpp/ripple/RPCServer.h" -#include "src/cpp/ripple/RPCDoor.h" #include "src/cpp/ripple/RPCErr.h" #include "src/cpp/ripple/RPCSub.h" #include "src/cpp/ripple/RegularKeySetTransactor.h" @@ -242,11 +190,41 @@ #include "src/cpp/ripple/SerializedValidation.h" #include "src/cpp/ripple/AccountSetTransactor.h" #include "src/cpp/ripple/TrustSetTransactor.h" -#include "src/cpp/ripple/ripple_Version.h" #include "src/cpp/ripple/WSConnection.h" -#include "src/cpp/ripple/WSHandler.h" +#include "src/cpp/ripple/ripple_WSHandler.h" #include "src/cpp/ripple/WalletAddTransactor.h" +#include "basics/ripple_Version.h" // VFALCO TODO Should this be private? +#include "basics/ripple_BuildVersion.h" // private +#include "basics/ripple_RPCServerHandler.h" + +#include "src/cpp/ripple/RPCDoor.h" // needs RPCServer + +} + +//------------------------------------------------------------------------------ + +// VFALCO TODO Move this to an appropriate header +namespace boost +{ + template <> + struct range_mutable_iterator + { + typedef ripple::LedgerEntrySet::iterator type; + }; + + template <> + struct range_const_iterator + { + typedef ripple::LedgerEntrySet::const_iterator type; + }; +} + +//------------------------------------------------------------------------------ + +namespace ripple +{ + //------------------------------------------------------------------------------ // VFALCO TODO figure out who needs these and move to a sensible private header. @@ -265,6 +243,16 @@ static const uint64 tenTo17m1 = tenTo17 - 1; #if ! defined (RIPPLE_MAIN_PART) || RIPPLE_MAIN_PART == 1 +#include "basics/ripple_RPCServerHandler.cpp" +#include "node/ripple_NodeObject.cpp" +#include "node/ripple_NodeStore.cpp" +#include "node/ripple_LevelDBBackendFactory.cpp" +#include "node/ripple_MemoryBackendFactory.cpp" +#include "node/ripple_HyperLevelDBBackendFactory.cpp" +#include "node/ripple_MdbBackendFactory.cpp" +#include "node/ripple_NullBackendFactory.cpp" +#include "node/ripple_SqliteBackendFactory.cpp" + #include "src/cpp/ripple/Ledger.cpp" #include "src/cpp/ripple/ripple_SHAMapDelta.cpp" #include "src/cpp/ripple/ripple_SHAMapNode.cpp" @@ -274,12 +262,6 @@ static const uint64 tenTo17m1 = tenTo17 - 1; #include "src/cpp/ripple/ripple_AccountItems.cpp" #include "src/cpp/ripple/ripple_AccountState.cpp" #include "src/cpp/ripple/ChangeTransactor.cpp" -#include "src/cpp/ripple/ripple_DBInit.cpp" -#include "src/cpp/ripple/Interpreter.cpp" -#include "src/cpp/ripple/LedgerTiming.cpp" -#include "src/cpp/ripple/main.cpp" -#include "src/cpp/ripple/ripple_Offer.cpp" -#include "src/cpp/ripple/Operation.cpp" #endif @@ -297,7 +279,6 @@ static const uint64 tenTo17m1 = tenTo17 - 1; #include "src/cpp/ripple/AccountSetTransactor.cpp" #include "src/cpp/ripple/ripple_CanonicalTXSet.cpp" #include "src/cpp/ripple/Contract.cpp" -#include "src/cpp/ripple/HTTPRequest.cpp" #include "src/cpp/ripple/LedgerProposal.cpp" #include "src/cpp/ripple/ripple_LoadManager.cpp" #include "src/cpp/ripple/ripple_NicknameState.cpp" @@ -315,7 +296,7 @@ static const uint64 tenTo17m1 = tenTo17 - 1; static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength) { // VFALCO TODO eliminate this horrendous dependency on theApp and LocalCredentials - return 512 == iKeyLength ? theApp->getLocalCredentials ().getDh512 () : theApp->getLocalCredentials ().getDh1024 (); + return 512 == iKeyLength ? getApp().getLocalCredentials ().getDh512 () : getApp().getLocalCredentials ().getDh1024 (); } #include "src/cpp/ripple/ripple_RippleCalc.cpp" @@ -342,14 +323,6 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength) #if ! defined (RIPPLE_MAIN_PART) || RIPPLE_MAIN_PART == 4 -// This is for PeerDoor and WSDoor -// Generate DH for SSL connection. -static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength) -{ - // VFALCO TODO eliminate this horrendous dependency on theApp and LocalCredentials - return 512 == iKeyLength ? theApp->getLocalCredentials ().getDh512 () : theApp->getLocalCredentials ().getDh1024 (); -} - #include "src/cpp/ripple/ripple_UniqueNodeList.cpp" #include "src/cpp/ripple/ripple_InboundLedger.cpp" #include "src/cpp/ripple/ripple_SqliteDatabase.cpp" @@ -358,14 +331,12 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength) #include "src/cpp/ripple/RegularKeySetTransactor.cpp" #include "src/cpp/ripple/ripple_RippleState.cpp" #include "src/cpp/ripple/RPCDoor.cpp" -#include "src/cpp/ripple/RPCServer.cpp" #include "src/cpp/ripple/ScriptData.cpp" -#include "src/cpp/ripple/SNTPClient.cpp" #include "src/cpp/ripple/TransactionCheck.cpp" #include "src/cpp/ripple/TransactionMaster.cpp" #include "src/cpp/ripple/TransactionQueue.cpp" #include "src/cpp/ripple/TrustSetTransactor.cpp" -#include "src/cpp/ripple/WSHandler.cpp" +#include "src/cpp/ripple/ripple_WSHandler.cpp" #endif @@ -382,6 +353,9 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength) #include "src/cpp/ripple/ripple_AcceptedLedgerTx.cpp" #include "src/cpp/ripple/ripple_DatabaseCon.cpp" #include "src/cpp/ripple/ripple_FeeVote.cpp" +#include "src/cpp/ripple/ripple_DBInit.cpp" +#include "src/cpp/ripple/Interpreter.cpp" +#include "src/cpp/ripple/LedgerTiming.cpp" #endif @@ -394,10 +368,11 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength) #include "src/cpp/ripple/ripple_Features.cpp" #include "src/cpp/ripple/ripple_LocalCredentials.cpp" -#include "src/cpp/ripple/ripple_HashedObject.cpp" #include "src/cpp/ripple/ripple_AcceptedLedger.cpp" #include "src/cpp/ripple/ripple_DisputedTx.cpp" #include "src/cpp/ripple/ripple_HashRouter.cpp" +#include "src/cpp/ripple/ripple_Main.cpp" +#include "src/cpp/ripple/ripple_Offer.cpp" #endif @@ -407,13 +382,13 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength) #include "src/cpp/ripple/NetworkOPs.cpp" #include "src/cpp/ripple/ripple_Peers.cpp" -#include "src/cpp/ripple/ripple_HashedObjectStore.cpp" #include "src/cpp/ripple/ripple_InboundLedgers.cpp" #include "src/cpp/ripple/ripple_LedgerHistory.cpp" #include "src/cpp/ripple/ripple_PathRequest.cpp" #include "src/cpp/ripple/ripple_SerializedLedger.cpp" #include "src/cpp/ripple/ripple_TransactionAcquire.cpp" +#include "src/cpp/ripple/Operation.cpp" #endif @@ -423,12 +398,12 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength) #include "src/cpp/ripple/ripple_LedgerConsensus.cpp" #include "src/cpp/ripple/LedgerMaster.cpp" -#include "src/cpp/ripple/HttpsClient.cpp" #include "src/cpp/ripple/ripple_InfoSub.cpp" #include "src/cpp/ripple/ripple_OrderBook.cpp" #include "src/cpp/ripple/ripple_PeerSet.cpp" #include "src/cpp/ripple/ripple_ProofOfWork.cpp" +#include "src/cpp/ripple/ripple_ProofOfWorkFactory.h" // private #include "src/cpp/ripple/ripple_ProofOfWorkFactory.cpp" // requires ProofOfWork.cpp for ProofOfWork::sMaxDifficulty #include "src/cpp/ripple/ripple_SerializedTransaction.cpp" @@ -438,6 +413,39 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength) //------------------------------------------------------------------------------ +} + +//------------------------------------------------------------------------------ + +#if ! defined (RIPPLE_MAIN_PART) || RIPPLE_MAIN_PART == 8 + +// Unit Tests +// +// These must be outside the namespace +// +// VFALCO TODO Eliminate the need for boost for unit tests. +// +#include "src/cpp/ripple/LedgerUnitTests.cpp" +#include "src/cpp/ripple/ripple_SHAMapUnitTests.cpp" +#include "src/cpp/ripple/ripple_SHAMapSyncUnitTests.cpp" +#include "src/cpp/ripple/ripple_ProofOfWorkFactoryUnitTests.cpp" // Requires ProofOfWorkFactory.h +#include "src/cpp/ripple/ripple_SerializedTransactionUnitTests.cpp" + +//------------------------------------------------------------------------------ + +namespace ripple +{ + extern int rippleMain (int argc, char** argv); +} + +// Must be outside the namespace for obvious reasons +int main (int argc, char** argv) +{ + return ripple::rippleMain (argc, argv); +} + +#endif + //------------------------------------------------------------------------------ #ifdef _MSC_VER diff --git a/modules/ripple_app/ripple_app.h b/modules/ripple_app/ripple_app.h index 15e362a289..95f4794ace 100644 --- a/modules/ripple_app/ripple_app.h +++ b/modules/ripple_app/ripple_app.h @@ -17,9 +17,11 @@ @defgroup ripple_app */ -#ifndef RIPPLE_MAIN_H -#define RIPPLE_MAIN_H +#ifndef RIPPLE_APP_H_INCLUDED +#define RIPPLE_APP_H_INCLUDED #include "modules/ripple_basics/ripple_basics.h" +#include "modules/ripple_basio/ripple_basio.h" + #endif diff --git a/modules/ripple_basics/containers/ripple_KeyCache.h b/modules/ripple_basics/containers/ripple_KeyCache.h index aff5cf2248..ee0678f0e0 100644 --- a/modules/ripple_basics/containers/ripple_KeyCache.h +++ b/modules/ripple_basics/containers/ripple_KeyCache.h @@ -4,8 +4,8 @@ */ //============================================================================== -#ifndef RIPPLE_KEYCACHE_H -#define RIPPLE_KEYCACHE_H +#ifndef RIPPLE_KEYCACHE_H_INCLUDED +#define RIPPLE_KEYCACHE_H_INCLUDED /** Maintains a cache of keys with no associated data. @@ -27,7 +27,7 @@ class KeyCache public: /** Provides a type for the key. */ - typedef Key key_type; + typedef Key key_type; /** Construct with the specified name. @@ -36,7 +36,10 @@ public: */ KeyCache (const std::string& name, int size = 0, - int age = 120) : mName (name), mTargetSize (size), mTargetAge (age) + int age = 120) + : mName (name) + , mTargetSize (size) + , mTargetAge (age) { assert ((size >= 0) && (age > 2)); } @@ -80,7 +83,7 @@ public: /** Retrieve the name of this object. */ - const std::string& getName () + std::string const& getName () { return mName; } diff --git a/modules/ripple_basics/containers/ripple_RangeSet.cpp b/modules/ripple_basics/containers/ripple_RangeSet.cpp index be3d29733b..0509029e83 100644 --- a/modules/ripple_basics/containers/ripple_RangeSet.cpp +++ b/modules/ripple_basics/containers/ripple_RangeSet.cpp @@ -6,6 +6,10 @@ SETUP_LOG (RangeSet) +// VFALCO NOTE std::min and std::max not good enough? +// NOTE Why isn't this written as a template? +// TODO Replace this with std calls. +// inline uint32 min (uint32 x, uint32 y) { return (x < y) ? x : y; @@ -30,7 +34,7 @@ uint32 RangeSet::getFirst () const const_iterator it = mRanges.begin (); if (it == mRanges.end ()) - return RangeSetAbsent; + return absent; return it->first; } @@ -45,7 +49,7 @@ uint32 RangeSet::getNext (uint32 v) const if (contains (it, v + 1)) return v + 1; } - return RangeSetAbsent; + return absent; } uint32 RangeSet::getLast () const @@ -53,7 +57,7 @@ uint32 RangeSet::getLast () const const_reverse_iterator it = mRanges.rbegin (); if (it == mRanges.rend ()) - return RangeSetAbsent; + return absent; return it->second; } @@ -66,30 +70,39 @@ uint32 RangeSet::getPrev (uint32 v) const return it.second; if (contains (it, v + 1)) + return v - 1; } - return RangeSetAbsent; + return absent; } +// Return the largest number not in the set that is less than the given number +// uint32 RangeSet::prevMissing (uint32 v) const { - // largest number not in the set that is less than the given number + uint32 result = absent; - // Nothing before zero - if (v == 0) - return RangeSetAbsent; - - BOOST_FOREACH (const value_type & it, mRanges) + if (v != 0) { - if (contains (it, v - 1)) - { // We have (v-1) in the set - if (it.first == 0) - return RangeSetAbent; - return it.first - 1; + checkInternalConsistency (); + + // Handle the case where the loop reaches the terminating condition + // + result = v - 1; + + for (const_reverse_iterator cur = mRanges.rbegin (); cur != mRanges.rend (); ++cur) + { + // See if v-1 is in the range + if (contains (*cur, result)) + { + result = cur->first - 1; + break; + } } } - // We don't have (v-1), so v-1 is it - return v - 1; + bassert (result == absent || !hasValue (result)); + + return result; } void RangeSet::setValue (uint32 v) @@ -97,6 +110,7 @@ void RangeSet::setValue (uint32 v) if (!hasValue (v)) { mRanges[v] = v; + simplify (); } } @@ -112,6 +126,7 @@ void RangeSet::setRange (uint32 minV, uint32 maxV) } mRanges[minV] = maxV; + simplify (); } @@ -124,7 +139,9 @@ void RangeSet::clearValue (uint32 v) if (it->first == v) { if (it->second == v) + { mRanges.erase (it); + } else { uint32 oldEnd = it->second; @@ -133,7 +150,9 @@ void RangeSet::clearValue (uint32 v) } } else if (it->second == v) + { -- (it->second); + } else { uint32 oldEnd = it->second; @@ -141,6 +160,7 @@ void RangeSet::clearValue (uint32 v) mRanges[v + 1] = oldEnd; } + checkInternalConsistency(); return; } } @@ -176,7 +196,10 @@ void RangeSet::simplify () iterator nit = it; if (++nit == mRanges.end ()) + { + checkInternalConsistency(); return; + } if (it->second >= (nit->first - 1)) { @@ -185,35 +208,37 @@ void RangeSet::simplify () mRanges.erase (nit); } else + { it = nit; + } } } -BOOST_AUTO_TEST_SUITE (RangeSet_suite) - -BOOST_AUTO_TEST_CASE (RangeSet_test) +void RangeSet::checkInternalConsistency () const noexcept { - WriteLog (lsTRACE, RangeSet) << "RangeSet test begins"; +#if BEAST_DEBUG + if (mRanges.size () > 1) + { + const_iterator const last = std::prev (mRanges.end ()); - RangeSet r1, r2; + for (const_iterator cur = mRanges.begin (); cur != last; ++cur) + { + const_iterator const next = std::next (cur); - r1.setRange (1, 10); - r1.clearValue (5); - r1.setRange (11, 20); + bassert (cur->first <= cur->second); - r2.setRange (1, 4); - r2.setRange (6, 10); - r2.setRange (10, 20); + bassert (next->first <= next->second); - if (r1.hasValue (5)) BOOST_FAIL ("RangeSet fail"); + bassert (cur->second + 1 < next->first); + } + } + else if (mRanges.size () == 1) + { + const_iterator const iter = mRanges.begin (); - if (!r2.hasValue (9)) BOOST_FAIL ("RangeSet fail"); + bassert (iter->first <= iter->second); + } - // TODO: Traverse functions must be tested - - WriteLog (lsTRACE, RangeSet) << "RangeSet test complete"; +#endif } -BOOST_AUTO_TEST_SUITE_END () - -// vim:ts=4 diff --git a/modules/ripple_basics/containers/ripple_RangeSet.h b/modules/ripple_basics/containers/ripple_RangeSet.h index fb1fc8412a..bc41095d66 100644 --- a/modules/ripple_basics/containers/ripple_RangeSet.h +++ b/modules/ripple_basics/containers/ripple_RangeSet.h @@ -4,8 +4,8 @@ */ //============================================================================== -#ifndef RIPPLE_RANGESET_H -#define RIPPLE_RANGESET_H +#ifndef RIPPLE_RANGESET_H_INCLUDED +#define RIPPLE_RANGESET_H_INCLUDED /** A sparse set of integers. */ @@ -13,39 +13,51 @@ class RangeSet { public: - static const uint32 RangeSetAbsent = static_cast (-1); - -protected: - std::map mRanges; // First is lowest value in range, last is highest value in range - - typedef std::map::const_iterator const_iterator; - typedef std::map::const_reverse_iterator const_reverse_iterator; - typedef std::map::value_type value_type; - typedef std::map::iterator iterator; - - static bool contains (value_type const& it, uint32 v) - { - return (it.first <= v) && (it.second >= v); - } - - void simplify (); + static const uint32 absent = static_cast (-1); public: RangeSet () { } bool hasValue (uint32) const; + uint32 getFirst () const; uint32 getNext (uint32) const; uint32 getLast () const; uint32 getPrev (uint32) const; - uint32 prevMissing (uint32) const; // largest number not in the set that is less than the given number + // largest number not in the set that is less than the given number + uint32 prevMissing (uint32) const; void setValue (uint32); void setRange (uint32, uint32); void clearValue (uint32); std::string toString () const; + + /** Check invariants of the data. + + This is for diagnostics, and does nothing in release builds. + */ + void checkInternalConsistency () const noexcept; + +private: + void simplify (); + +private: + typedef std::map Map; + + typedef Map::const_iterator const_iterator; + typedef Map::const_reverse_iterator const_reverse_iterator; + typedef Map::value_type value_type; + typedef Map::iterator iterator; + + static bool contains (value_type const& it, uint32 v) + { + return (it.first <= v) && (it.second >= v); + } + + // First is lowest value in range, last is highest value in range + Map mRanges; }; #endif diff --git a/modules/ripple_basics/containers/ripple_RangeSetUnitTests.cpp b/modules/ripple_basics/containers/ripple_RangeSetUnitTests.cpp new file mode 100644 index 0000000000..87a50f2d03 --- /dev/null +++ b/modules/ripple_basics/containers/ripple_RangeSetUnitTests.cpp @@ -0,0 +1,34 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +BOOST_AUTO_TEST_SUITE (RangeSet_suite) + +BOOST_AUTO_TEST_CASE (RangeSet_test) +{ + using namespace ripple; + + WriteLog (lsTRACE, RangeSet) << "RangeSet test begins"; + + RangeSet r1, r2; + + r1.setRange (1, 10); + r1.clearValue (5); + r1.setRange (11, 20); + + r2.setRange (1, 4); + r2.setRange (6, 10); + r2.setRange (10, 20); + + if (r1.hasValue (5)) BOOST_FAIL ("RangeSet fail"); + + if (!r2.hasValue (9)) BOOST_FAIL ("RangeSet fail"); + + // TODO: Traverse functions must be tested + + WriteLog (lsTRACE, RangeSet) << "RangeSet test complete"; +} + +BOOST_AUTO_TEST_SUITE_END () diff --git a/modules/ripple_basics/containers/ripple_SecureAllocator.h b/modules/ripple_basics/containers/ripple_SecureAllocator.h index f956de7964..0c8e0d7666 100644 --- a/modules/ripple_basics/containers/ripple_SecureAllocator.h +++ b/modules/ripple_basics/containers/ripple_SecureAllocator.h @@ -4,8 +4,8 @@ */ //============================================================================== -#ifndef RIPPLE_SECUREALLOCATOR_H -#define RIPPLE_SECUREALLOCATOR_H +#ifndef RIPPLE_SECUREALLOCATOR_H_INCLUDED +#define RIPPLE_SECUREALLOCATOR_H_INCLUDED // // Allocator that locks its contents from being paged diff --git a/modules/ripple_basics/containers/ripple_TaggedCache.cpp b/modules/ripple_basics/containers/ripple_TaggedCache.cpp index 989c0d7675..bd5bd4012a 100644 --- a/modules/ripple_basics/containers/ripple_TaggedCache.cpp +++ b/modules/ripple_basics/containers/ripple_TaggedCache.cpp @@ -4,4 +4,4 @@ */ //============================================================================== -SETUP_LOG (TaggedCacheLog) +SETUP_LOGN (TaggedCacheLog,"TaggedCache") diff --git a/modules/ripple_basics/ripple_basics.cpp b/modules/ripple_basics/ripple_basics.cpp index d94bb2ce2c..67495b7d51 100644 --- a/modules/ripple_basics/ripple_basics.cpp +++ b/modules/ripple_basics/ripple_basics.cpp @@ -10,10 +10,9 @@ @ingroup ripple_basics */ -#include "ripple_basics.h" +#include "BeastConfig.h" -#include -#include +#include "ripple_basics.h" // VFALCO TODO Rewrite Sustain to use beast::Process // @@ -28,11 +27,7 @@ #include #endif -#include -#include -#include -#include // VFALCO NOTE just for parseIpPort (!) -#include +//#include // VFALCO TODO Replace OpenSSL randomness with a dependency-free implementation // Perhaps Schneier's Fortuna or a variant. Abstract the collection of @@ -50,15 +45,18 @@ #include // for ripple_ByteOrder.cpp #endif -#if RIPPLE_USE_NAMESPACE +// This brings in the definitions for the Unit Test Framework. +// +#include + namespace ripple { -#endif #include "containers/ripple_RangeSet.cpp" #include "containers/ripple_TaggedCache.cpp" #include "utility/ripple_Log.cpp" +#include "utility/ripple_LogFile.cpp" #include "utility/ripple_ByteOrder.cpp" #include "utility/ripple_CountedObject.cpp" @@ -74,6 +72,8 @@ namespace ripple #include "types/ripple_UInt256.cpp" -#if RIPPLE_USE_NAMESPACE } -#endif + +// These must be outside the namespace (because of boost) +#include "containers/ripple_RangeSetUnitTests.cpp" +#include "utility/ripple_StringUtilitiesUnitTests.cpp" diff --git a/modules/ripple_basics/ripple_basics.h b/modules/ripple_basics/ripple_basics.h index 39a11237a4..49eabc2124 100644 --- a/modules/ripple_basics/ripple_basics.h +++ b/modules/ripple_basics/ripple_basics.h @@ -20,43 +20,41 @@ #ifndef RIPPLE_BASICS_RIPPLEHEADER #define RIPPLE_BASICS_RIPPLEHEADER -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "system/ripple_StandardIncludes.h" -#include -#if BOOST_VERSION < 104700 -#error Ripple requires Boost version 1.47 or later +// This must come before Boost, to fix the boost placeholders problem +#include "beast/modules/beast_basics/beast_basics.h" + +#include "system/ripple_BoostIncludes.h" + +#include "system/ripple_OpenSSLIncludes.h" + +//------------------------------------------------------------------------------ + +// From +// http://stackoverflow.com/questions/4682343/how-to-resolve-conflict-between-boostshared-ptr-and-using-stdshared-ptr +// +#if __cplusplus > 201100L +namespace boost +{ + template + const T* get_pointer (std::shared_ptr const& ptr) + { + return ptr.get(); + } + + template + T* get_pointer (std::shared_ptr& ptr) + { + return ptr.get(); + } +} #endif -// VFALCO TODO Move all boost includes into ripple_BoostHeaders.h -// -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +//------------------------------------------------------------------------------ // ByteOrder -#ifdef WIN32 +#if BEAST_WIN32 // (nothing) #elif __APPLE__ # include @@ -66,23 +64,16 @@ # include #endif -#include // for DiffieHellmanUtil -#include // For HashUtilities -#include // For HashUtilities - -#include "BeastConfig.h" // Must come before any Beast includes - -#include "modules/beast_core/beast_core.h" -#include "modules/beast_basics/beast_basics.h" +#include "beast/modules/beast_core/beast_core.h" #include "../ripple_json/ripple_json.h" -#if RIPPLE_USE_NAMESPACE namespace ripple { -#endif -#include "utility/ripple_IntegerTypes.h" // must come first +using namespace beast; + +#include "utility/ripple_LogFile.h" #include "utility/ripple_Log.h" // Needed by others #include "types/ripple_BasicTypes.h" @@ -108,8 +99,6 @@ namespace ripple #include "containers/ripple_SecureAllocator.h" #include "containers/ripple_TaggedCache.h" -#if RIPPLE_USE_NAMESPACE } -#endif #endif diff --git a/modules/ripple_basics/system/ripple_BoostIncludes.h b/modules/ripple_basics/system/ripple_BoostIncludes.h new file mode 100644 index 0000000000..4cb4b7476d --- /dev/null +++ b/modules/ripple_basics/system/ripple_BoostIncludes.h @@ -0,0 +1,56 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_BOOSTINCLUDES_RIPPLEHEADER +#define RIPPLE_BOOSTINCLUDES_RIPPLEHEADER + +// All Boost includes used throughout Ripple. +// +// This shows all the dependencies in one place. Please do not add +// boost includes anywhere else in the source code. If possible, do +// not add any more includes. +// +// A long term goal is to reduce and hopefully eliminate the usage of boost. +// + +#include + +#if BOOST_VERSION < 104700 +# error Ripple requires Boost version 1.47 or later +#endif + +// This is better than setting it in some Makefile or IDE Project file. +// +#define BOOST_FILESYSTEM_NO_DEPRECATED +#define BOOST_TEST_NO_LIB +#define BOOST_TEST_ALTERNATIVE_INIT_API +#define BOOST_TEST_NO_MAIN + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // VFALCO NOTE this looks like junk +#include +#include +#include +#include +#include +#include +#include +#include + +#endif diff --git a/modules/ripple_basics/system/ripple_OpenSSLIncludes.h b/modules/ripple_basics/system/ripple_OpenSSLIncludes.h new file mode 100644 index 0000000000..eddb8c8daa --- /dev/null +++ b/modules/ripple_basics/system/ripple_OpenSSLIncludes.h @@ -0,0 +1,23 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_OPENSSLINCLUDES_RIPPLEHEADER +#define RIPPLE_OPENSSLINCLUDES_RIPPLEHEADER + +// All OpenSSL includes we need + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif diff --git a/modules/ripple_basics/system/ripple_StandardIncludes.h b/modules/ripple_basics/system/ripple_StandardIncludes.h new file mode 100644 index 0000000000..b20927e779 --- /dev/null +++ b/modules/ripple_basics/system/ripple_StandardIncludes.h @@ -0,0 +1,35 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_STANDARDINCLUDES_RIPPLEHEADER +#define RIPPLE_STANDARDINCLUDES_RIPPLEHEADER + +// All required Standard C++ Library includes + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif diff --git a/modules/ripple_basics/types/ripple_BasicTypes.h b/modules/ripple_basics/types/ripple_BasicTypes.h index eceae71dea..44748aa30c 100644 --- a/modules/ripple_basics/types/ripple_BasicTypes.h +++ b/modules/ripple_basics/types/ripple_BasicTypes.h @@ -4,8 +4,8 @@ */ //============================================================================== -#ifndef RIPPLE_TYPES_H -#define RIPPLE_TYPES_H +#ifndef RIPPLE_BASICTYPES_H +#define RIPPLE_BASICTYPES_H /** Storage for linear binary data. diff --git a/modules/ripple_basics/types/ripple_HashMaps.h b/modules/ripple_basics/types/ripple_HashMaps.h index 8c4da244a5..bb1d22913e 100644 --- a/modules/ripple_basics/types/ripple_HashMaps.h +++ b/modules/ripple_basics/types/ripple_HashMaps.h @@ -14,8 +14,7 @@ done by seeding the hashing function with a random number generated at program startup. */ -// VFALCO TODO derive from Uncopyable -class HashMaps // : beast::Uncopayble +class HashMaps : Uncopyable { public: /** Golden ratio constant used in hashing functions. diff --git a/modules/ripple_basics/utility/ripple_ByteOrder.cpp b/modules/ripple_basics/utility/ripple_ByteOrder.cpp index 72e0300287..9882a10483 100644 --- a/modules/ripple_basics/utility/ripple_ByteOrder.cpp +++ b/modules/ripple_basics/utility/ripple_ByteOrder.cpp @@ -4,7 +4,7 @@ */ //============================================================================== -#ifdef WIN32 +#if BEAST_WIN32 // from: http://stackoverflow.com/questions/3022552/is-there-any-standard-htonl-like-function-for-64-bits-integers-in-c // but we don't need to check the endianness diff --git a/modules/ripple_basics/utility/ripple_ByteOrder.h b/modules/ripple_basics/utility/ripple_ByteOrder.h index 9a60b8b37a..8556d018aa 100644 --- a/modules/ripple_basics/utility/ripple_ByteOrder.h +++ b/modules/ripple_basics/utility/ripple_ByteOrder.h @@ -11,8 +11,7 @@ // Reference: http://www.mail-archive.com/licq-commits@googlegroups.com/msg02334.html -// VFALCO TODO use BEAST_* platform macros instead of hard-coded compiler specific ones -#ifdef WIN32 +#if BEAST_WIN32 extern uint64_t htobe64 (uint64_t value); extern uint64_t be64toh (uint64_t value); extern uint32_t htobe32 (uint32_t value); diff --git a/modules/ripple_basics/utility/ripple_CountedObject.h b/modules/ripple_basics/utility/ripple_CountedObject.h index 6cde850597..14a69dc4da 100644 --- a/modules/ripple_basics/utility/ripple_CountedObject.h +++ b/modules/ripple_basics/utility/ripple_CountedObject.h @@ -83,7 +83,7 @@ private: @ingroup ripple_basics */ template -class CountedObject +class CountedObject : LeakChecked > { public: CountedObject () @@ -109,23 +109,13 @@ private: char const* getName () const noexcept { - return getClassName (); + return Object::getCountedObjectName (); } void checkPureVirtual () const { } }; private: - /* Due to a bug in Visual Studio 10 and earlier, the string returned by - typeid().name() will appear to leak on exit. Therefore, we should - only call this function when there's an actual leak, or else there - will be spurious leak notices at exit. - */ - static char const* getClassName () noexcept - { - return typeid (Object).name (); - } - static Counter& getCounter () noexcept { // VFALCO TODO Research the thread safety of static initializers diff --git a/modules/ripple_basics/utility/ripple_HashUtilities.h b/modules/ripple_basics/utility/ripple_HashUtilities.h index bc2f76cb0d..f2ead4da77 100644 --- a/modules/ripple_basics/utility/ripple_HashUtilities.h +++ b/modules/ripple_basics/utility/ripple_HashUtilities.h @@ -67,7 +67,7 @@ inline uint160 Hash160 (Blob const& vch) } /* -#ifdef WIN32 +#if BEAST_WIN32 // This is used to attempt to keep keying material out of swap // Note that VirtualLock does not provide this as a guarantee on Windows, // but, in practice, memory that has been VirtualLock'd almost never gets written to diff --git a/modules/ripple_basics/utility/ripple_IniFile.cpp b/modules/ripple_basics/utility/ripple_IniFile.cpp index 6972fe096c..8f60104d83 100644 --- a/modules/ripple_basics/utility/ripple_IniFile.cpp +++ b/modules/ripple_basics/utility/ripple_IniFile.cpp @@ -6,10 +6,9 @@ #define SECTION_DEFAULT_NAME "" -// for logging -struct ParseSectionLog { }; +struct ParseSectionLog; // for Log -SETUP_LOG (ParseSectionLog) +SETUP_LOGN (ParseSectionLog,"ParseSection") Section ParseSection (const std::string& strInput, const bool bTrim) { @@ -63,13 +62,13 @@ Section ParseSection (const std::string& strInput, const bool bTrim) void SectionEntriesPrint (std::vector* vspEntries, const std::string& strSection) { - std::cerr << "[" << strSection << "]" << std::endl; + Log::out() << "[" << strSection << "]"; if (vspEntries) { BOOST_FOREACH (std::string & strValue, *vspEntries) { - std::cerr << strValue << std::endl; + Log::out() << strValue; } } } diff --git a/modules/ripple_basics/utility/ripple_IntegerTypes.h b/modules/ripple_basics/utility/ripple_IntegerTypes.h deleted file mode 100644 index d3f5455d9d..0000000000 --- a/modules/ripple_basics/utility/ripple_IntegerTypes.h +++ /dev/null @@ -1,36 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_INTEGERTYPES_H -#define RIPPLE_INTEGERTYPES_H - -// VFALCO TODO Get rid of these and use the types from beast instead. -// VFALCO TODO determine if Borland C is supported -#if defined (_MSC_VER) /*|| defined(__BORLANDC__)*/ -typedef __int64 int64; -typedef unsigned __int64 uint64; -typedef unsigned int uint32; -typedef unsigned short int uint16; -typedef int int32; - -#else -typedef long long int64; -typedef unsigned long long uint64; -typedef unsigned int uint32; -typedef unsigned short int uint16; -typedef int int32; - -#endif - -// VFALCO TODO make sure minimum VS version is 9, 10, or 11 -// If commenting this out creates a problem, contact me! -/* -#if defined(_MSC_VER) && _MSC_VER < 1300 -#define for if (false) ; else for -#endif -*/ - -#endif diff --git a/modules/ripple_basics/utility/ripple_Log.cpp b/modules/ripple_basics/utility/ripple_Log.cpp index b3c7a4df17..348d252329 100644 --- a/modules/ripple_basics/utility/ripple_Log.cpp +++ b/modules/ripple_basics/utility/ripple_Log.cpp @@ -4,24 +4,20 @@ */ //============================================================================== -boost::recursive_mutex Log::sLock; - +LogFile Log::s_logFile; +boost::recursive_mutex Log::s_lock; LogSeverity Log::sMinSeverity = lsINFO; -std::ofstream* Log::outStream = NULL; -boost::filesystem::path* Log::pathToLog = NULL; -uint32 Log::logRotateCounter = 0; - -#ifndef LOG_MAX_MESSAGE -#define LOG_MAX_MESSAGE (12 * 1024) -#endif +//------------------------------------------------------------------------------ LogPartition* LogPartition::headLog = NULL; -LogPartition::LogPartition (const char* name) : mNextLog (headLog), mMinSeverity (lsWARNING) +LogPartition::LogPartition (char const* partitionName) + : mNextLog (headLog) + , mMinSeverity (lsWARNING) { - const char* ptr = strrchr (name, '/'); - mName = (ptr == NULL) ? name : (ptr + 1); + const char* ptr = strrchr (partitionName, '/'); + mName = (ptr == NULL) ? partitionName : (ptr + 1); size_t p = mName.find (".cpp"); @@ -43,25 +39,6 @@ std::vector< std::pair > LogPartition::getSeverities ( //------------------------------------------------------------------------------ -// VFALCO TODO remove original code once we know the replacement is correct. -// Original code -/* -std::string ls = oss.str(); -size_t s = ls.find("\"secret\""); -if (s != std::string::npos) -{ - s += 8; - size_t sEnd = ls.size() - 1; - if (sEnd > (s + 35)) - sEnd = s + 35; - for (int i = s; i < sEnd; ++i) - ls[i] = '*'; -} -logMsg += ls; -*/ - -//------------------------------------------------------------------------------ - std::string Log::replaceFirstSecretWithAsterisks (std::string s) { using namespace std; @@ -133,76 +110,56 @@ Log::~Log () logMsg += replaceFirstSecretWithAsterisks (oss.str ()); - if (logMsg.size () > LOG_MAX_MESSAGE) + if (logMsg.size () > maximumMessageCharacters) { - logMsg.resize (LOG_MAX_MESSAGE); + logMsg.resize (maximumMessageCharacters); logMsg += "..."; } - boost::recursive_mutex::scoped_lock sl (sLock); - - if (mSeverity >= sMinSeverity) - std::cerr << logMsg << std::endl; - - if (outStream != NULL) - (*outStream) << logMsg << std::endl; + print (logMsg, mSeverity >= sMinSeverity); } -std::string Log::rotateLog (void) +void Log::print (std::string const& text, bool toStdErr) { - boost::recursive_mutex::scoped_lock sl (sLock); - boost::filesystem::path abs_path; - std::string abs_path_str; + boost::recursive_mutex::scoped_lock sl (s_lock); - uint32 failsafe = 0; + // Does nothing if not open. + s_logFile.writeln (text); - std::string abs_new_path_str; - - do + if (toStdErr) { - std::string s; - std::stringstream out; - - failsafe++; - - if (failsafe == std::numeric_limits::max ()) +#if BEAST_MSVC + if (beast_isRunningUnderDebugger ()) { - return "unable to create new log file; too many log files!"; + // Send it to the attached debugger's Output window + // + Logger::outputDebugString (text); + } + else +#endif + { + std::cerr << text << std::endl; } - - abs_path = boost::filesystem::absolute (""); - abs_path /= *pathToLog; - abs_path_str = abs_path.parent_path ().string (); - - out << logRotateCounter; - s = out.str (); - - abs_new_path_str = abs_path_str + "/" + s + "_" + pathToLog->filename ().string (); - - logRotateCounter++; - } - while (boost::filesystem::exists (boost::filesystem::path (abs_new_path_str))); +} - outStream->close (); +std::string Log::rotateLog () +{ + bool const wasOpened = s_logFile.closeAndReopen (); - try + if (wasOpened) { - boost::filesystem::rename (abs_path, boost::filesystem::path (abs_new_path_str)); + return "The log file was closed and reopened."; } - catch (...) + else { - // unable to rename existing log file + return "The log file could not be closed and reopened."; } - - setLogFile (*pathToLog); - - return abs_new_path_str; } void Log::setMinSeverity (LogSeverity s, bool all) { - boost::recursive_mutex::scoped_lock sl (sLock); + boost::recursive_mutex::scoped_lock sl (s_lock); sMinSeverity = s; @@ -212,7 +169,7 @@ void Log::setMinSeverity (LogSeverity s, bool all) LogSeverity Log::getMinSeverity () { - boost::recursive_mutex::scoped_lock sl (sLock); + boost::recursive_mutex::scoped_lock sl (s_lock); return sMinSeverity; } @@ -270,28 +227,11 @@ LogSeverity Log::stringToSeverity (const std::string& s) void Log::setLogFile (boost::filesystem::path const& path) { - std::ofstream* newStream = new std::ofstream (path.c_str (), std::fstream::app); + bool const wasOpened = s_logFile.open (path.c_str ()); - if (!newStream->good ()) + if (! wasOpened) { Log (lsFATAL) << "Unable to open logfile " << path; - delete newStream; - newStream = NULL; - } - - boost::recursive_mutex::scoped_lock sl (sLock); - - if (outStream != NULL) - delete outStream; - - outStream = newStream; - - if (pathToLog != &path) - { - if (pathToLog != NULL) - delete pathToLog; - - pathToLog = new boost::filesystem::path (path); } } diff --git a/modules/ripple_basics/utility/ripple_Log.h b/modules/ripple_basics/utility/ripple_Log.h index 6493a987b0..13c44ed155 100644 --- a/modules/ripple_basics/utility/ripple_Log.h +++ b/modules/ripple_basics/utility/ripple_Log.h @@ -4,8 +4,8 @@ */ //============================================================================== -#ifndef RIPPLE_LOG_H -#define RIPPLE_LOG_H +#ifndef RIPPLE_LOG_H_INCLUDED +#define RIPPLE_LOG_H_INCLUDED enum LogSeverity { @@ -20,23 +20,28 @@ enum LogSeverity //------------------------------------------------------------------------------ -// VFALCO TODO make this a nested class in Log -class LogPartition +// VFALCO TODO make this a nested class in Log? +class LogPartition // : public List ::Node { -protected: - static LogPartition* headLog; - - LogPartition* mNextLog; - LogSeverity mMinSeverity; - std::string mName; - public: - LogPartition (const char* name); + LogPartition (const char* partitionName); + + /** Retrieve the LogPartition associated with an object. + + Each LogPartition is a singleton. + */ + template + static LogPartition const& get () + { + static LogPartition logPartition (getPartitionName ()); + return logPartition; + } bool doLog (LogSeverity s) const { return s >= mMinSeverity; } + const std::string& getName () const { return mName; @@ -47,33 +52,34 @@ public: static std::vector< std::pair > getSeverities (); private: - /** Retrieve file name from a log partition. + /** Retrieve the name for a log partition. */ template - static char const* getFileName (); - /* - { - static_vfassert (false); - } - */ + static char const* getPartitionName (); -public: - template - static LogPartition const& get () - { - static LogPartition logPartition (getFileName ()); - return logPartition; - } +private: + // VFALCO TODO Use an intrusive linked list + // + static LogPartition* headLog; + + LogPartition* mNextLog; + LogSeverity mMinSeverity; + std::string mName; }; -#define SETUP_LOG(k) \ - template <> char const* LogPartition::getFileName () { return __FILE__; } \ - struct k##Instantiator { k##Instantiator () { LogPartition::get (); } }; \ - static k##Instantiator k##Instantiator_instance; +#define SETUP_LOG(Class) \ + template <> char const* LogPartition::getPartitionName () { return #Class; } \ + struct Class##Instantiator { Class##Instantiator () { LogPartition::get (); } }; \ + static Class##Instantiator Class##Instantiator_instance; + +#define SETUP_LOGN(Class,Name) \ + template <> char const* LogPartition::getPartitionName () { return Name; } \ + struct Class##Instantiator { Class##Instantiator () { LogPartition::get (); } }; \ + static Class##Instantiator Class##Instantiator_instance; //------------------------------------------------------------------------------ -class Log +class Log : public Uncopyable { public: explicit Log (LogSeverity s) : mSeverity (s) @@ -109,25 +115,91 @@ public: static void setLogFile (boost::filesystem::path const& pathToLogFile); + /** Rotate the log file. + + The log file is closed and reopened. This is for compatibility + with log management tools. + + @return A human readable string describing the result of the operation. + */ static std::string rotateLog (); -private: - // VFALCO TODO derive from beast::Uncopyable - Log (const Log&); // no implementation - Log& operator= (const Log&); // no implementation +public: + /** Write to log output. - // VFALCO TODO looks like there are really TWO classes in here. - // One is a stream target for '<<' operator and the other - // is a singleton. Split the singleton out to a new class. - // - static boost::recursive_mutex sLock; - static LogSeverity sMinSeverity; - static std::ofstream* outStream; - static boost::filesystem::path* pathToLog; - static uint32 logRotateCounter; + All logging eventually goes through this function. If a + debugger is attached, the string goes to the debugging console, + else it goes to the standard error output. If a log file is + open, then the message is additionally written to the open log + file. + + The text should not contain a newline, it will be automatically + added as needed. + + @note This acquires a global mutex. + + @param text The text to write. + @param toStdErr `true` to also write to std::cerr + */ + static void print (std::string const& text, + bool toStdErr = true); + + /** Output stream for logging + + This is a convenient replacement for writing to `std::cerr`. + + Usage: + + @code + + Log::out () << "item1" << 2; + + @endcode + + It is not necessary to append a newline. + */ + class out + { + public: + out () + { + } + + ~out () + { + Log::print (m_ss.str ()); + } + + template + out& operator<< (T t) + { + m_ss << t; + return *this; + } + + private: + std::stringstream m_ss; + }; + +private: + enum + { + /** Maximum line length for log messages. + + If the message exceeds this length it will be truncated + with elipses. + */ + maximumMessageCharacters = 12 * 1024 + }; static std::string replaceFirstSecretWithAsterisks (std::string s); + // Singleton variables + // + static LogFile s_logFile; + static boost::recursive_mutex s_lock; + static LogSeverity sMinSeverity; + mutable std::ostringstream oss; LogSeverity mSeverity; std::string mPartitionName; diff --git a/modules/ripple_basics/utility/ripple_LogFile.cpp b/modules/ripple_basics/utility/ripple_LogFile.cpp new file mode 100644 index 0000000000..8c6d0900f8 --- /dev/null +++ b/modules/ripple_basics/utility/ripple_LogFile.cpp @@ -0,0 +1,69 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +LogFile::LogFile () + : m_stream (nullptr) +{ +} + +LogFile::~LogFile () +{ +} + +bool LogFile::isOpen () const noexcept +{ + return m_stream != nullptr; +} + +bool LogFile::open (boost::filesystem::path const& path) +{ + close (); + + bool wasOpened = false; + + // VFALCO TODO Make this work with Unicode file paths + ScopedPointer stream ( + new std::ofstream (path.c_str (), std::fstream::app)); + + if (stream->good ()) + { + m_path = path; + + m_stream = stream.release (); + + wasOpened = true; + } + + return wasOpened; +} + +bool LogFile::closeAndReopen () +{ + close (); + + return open (m_path); +} + +void LogFile::close () +{ + m_stream = nullptr; +} + +void LogFile::write (char const* text) +{ + if (m_stream != nullptr) + (*m_stream) << text; +} + +void LogFile::writeln (char const* text) +{ + if (m_stream != nullptr) + { + (*m_stream) << text; + (*m_stream) << std::endl; + } +} + diff --git a/modules/ripple_basics/utility/ripple_LogFile.h b/modules/ripple_basics/utility/ripple_LogFile.h new file mode 100644 index 0000000000..1d457a9ae6 --- /dev/null +++ b/modules/ripple_basics/utility/ripple_LogFile.h @@ -0,0 +1,93 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_LOGFILE_H_INCLUDED +#define RIPPLE_LOGFILE_H_INCLUDED + +/** Manages a system file containing logged output. + + The system file remains open during program execution. Interfaces + are provided for interoperating with standard log management + tools like logrotate(8): + + http://linuxcommand.org/man_pages/logrotate8.html + + @note None of the listed interfaces are thread-safe. +*/ +class LogFile : Uncopyable +{ +public: + /** Construct with no associated system file. + + A system file may be associated later with @ref open. + + @see open + */ + LogFile (); + + /** Destroy the object. + + If a system file is associated, it will be flushed and closed. + */ + ~LogFile (); + + /** Determine if a system file is associated with the log. + + @return `true` if a system file is associated and opened for writing. + */ + bool isOpen () const noexcept; + + /** Associate a system file with the log. + + If the file does not exist an attempt is made to create it + and open it for writing. If the file already exists an attempt is + made to open it for appending. + + If a system file is already associated with the log, it is closed first. + + @return `true` if the file was opened. + */ + // VFALCO NOTE The parameter is unfortunately a boost type because it + // can be either wchar or char based depending on platform. + // TODO Replace with beast::File + // + bool open (boost::filesystem::path const& path); + + /** Close and re-open the system file associated with the log + + This assists in interoperating with external log management tools. + + @return `true` if the file was opened. + */ + bool closeAndReopen (); + + /** Close the system file if it is open. + */ + void close (); + + /** write to the log file. + + Does nothing if there is no associated system file. + */ + void write (char const* text); + + /** write to the log file and append an end of line marker. + + Does nothing if there is no associated system file. + */ + void writeln (char const* text); + + /** Write to the log file using std::string. + */ + inline void write (std::string const& str) { write (str.c_str ()); } + inline void writeln (std::string const& str) { writeln (str.c_str ()); } + +private: + ScopedPointer m_stream; + boost::filesystem::path m_path; +}; + +#endif diff --git a/modules/ripple_basics/utility/ripple_PlatformMacros.h b/modules/ripple_basics/utility/ripple_PlatformMacros.h index 86e081918e..75beb8aa9c 100644 --- a/modules/ripple_basics/utility/ripple_PlatformMacros.h +++ b/modules/ripple_basics/utility/ripple_PlatformMacros.h @@ -7,36 +7,30 @@ #ifndef RIPPLE_PLATFORMMACROS_H #define RIPPLE_PLATFORMMACROS_H +#define FUNCTION_TYPE beast::function +#define BIND_TYPE beast::bind +#define P_1 beast::_1 +#define P_2 beast::_2 +#define P_3 beast::_3 +#define P_4 beast::_4 + // VFALCO TODO Clean this up #if (!defined(FORCE_NO_C11X) && (__cplusplus > 201100L)) || defined(FORCE_C11X) -// VFALCO TODO replace BIND_TYPE with a namespace lift - +// VFALCO TODO Get rid of the C11X macro #define C11X #define UPTR_T std::unique_ptr #define MOVE_P(p) std::move(p) -#define BIND_TYPE std::bind -#define FUNCTION_TYPE std::function -#define P_1 std::placeholders::_1 -#define P_2 std::placeholders::_2 -#define P_3 std::placeholders::_3 -#define P_4 std::placeholders::_4 #else #define UPTR_T std::auto_ptr #define MOVE_P(p) (p) -#define BIND_TYPE boost::bind -#define FUNCTION_TYPE boost::function -#define P_1 _1 -#define P_2 _2 -#define P_3 _3 -#define P_4 _4 #endif -// VFALCO TODO Clean this junk up +// VFALCO TODO Clean this stuff up. Remove as much as possible #define nothing() do {} while (0) #define fallthru() do {} while (0) #define NUMBER(x) (sizeof(x)/sizeof((x)[0])) diff --git a/modules/ripple_basics/utility/ripple_RandomNumbers.cpp b/modules/ripple_basics/utility/ripple_RandomNumbers.cpp index 2323850c36..a3bd8108b4 100644 --- a/modules/ripple_basics/utility/ripple_RandomNumbers.cpp +++ b/modules/ripple_basics/utility/ripple_RandomNumbers.cpp @@ -37,7 +37,7 @@ void RandomNumbers::fillBytes (void* destinationBuffer, int numberOfBytes) if (! initialize ()) { char const* message = "Unable to add system entropy"; - std::cerr << message << std::endl; + Log::out() << message; throw std::runtime_error (message); } } @@ -63,8 +63,7 @@ RandomNumbers& RandomNumbers::getInstance () //------------------------------------------------------------------------------ -// VFALCO TODO replace WIN32 macro with BEAST_WIN32 -#ifdef WIN32 +#if BEAST_WIN32 // Get entropy from the Windows crypto provider bool RandomNumbers::platformAddEntropy () @@ -75,24 +74,24 @@ bool RandomNumbers::platformAddEntropy () if (!CryptGetDefaultProviderA (PROV_RSA_FULL, NULL, CRYPT_MACHINE_DEFAULT, name, &count)) { -#ifdef DEBUG - std::cerr << "Unable to get default crypto provider" << std::endl; +#ifdef BEAST_DEBUG + Log::out() << "Unable to get default crypto provider"; #endif return false; } if (!CryptAcquireContextA (&cryptoHandle, NULL, name, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) { -#ifdef DEBUG - std::cerr << "Unable to acquire crypto provider" << std::endl; +#ifdef BEAST_DEBUG + Log::out() << "Unable to acquire crypto provider"; #endif return false; } if (!CryptGenRandom (cryptoHandle, 128, reinterpret_cast (rand))) { -#ifdef DEBUG - std::cerr << "Unable to get entropy from crypto provider" << std::endl; +#ifdef BEAST_DEBUG + Log::out() << "Unable to get entropy from crypto provider"; #endif CryptReleaseContext (cryptoHandle, 0); return false; @@ -115,8 +114,8 @@ bool RandomNumbers::platformAddEntropy () if (!reader.is_open ()) { -#ifdef DEBUG - std::cerr << "Unable to open random source" << std::endl; +#ifdef BEAST_DEBUG + Log::out() << "Unable to open random source"; #endif return false; } @@ -127,8 +126,8 @@ bool RandomNumbers::platformAddEntropy () if (bytesRead == 0) { -#ifdef DEBUG - std::cerr << "Unable to read from random source" << std::endl; +#ifdef BEAST_DEBUG + Log::out() << "Unable to read from random source"; #endif return false; } @@ -170,7 +169,7 @@ void RandomNumbers::platformAddPerformanceMonitorEntropy () int64 operator () () const { int64 nCounter = 0; -#if defined(WIN32) || defined(WIN64) +#if BEAST_WIN32 QueryPerformanceCounter ((LARGE_INTEGER*)&nCounter); #else timeval t; @@ -198,7 +197,7 @@ void RandomNumbers::platformAddPerformanceMonitorEntropy () nLastPerfmon = GetTime (); -#ifdef WIN32 +#if BEAST_WIN32 // Don't need this on Linux, OpenSSL automatically uses /dev/urandom // Seed with the entire set of perfmon data unsigned char pdata[250000]; diff --git a/modules/ripple_basics/utility/ripple_StringUtilities.cpp b/modules/ripple_basics/utility/ripple_StringUtilities.cpp index 090fad3c9a..cdc438a6ef 100644 --- a/modules/ripple_basics/utility/ripple_StringUtilities.cpp +++ b/modules/ripple_basics/utility/ripple_StringUtilities.cpp @@ -4,7 +4,9 @@ */ //============================================================================== -#if !defined(WIN32) && !defined(WIN64) +// VFALCO TODO Replace these with something more robust and without macros. +// +#if ! BEAST_MSVC #define _vsnprintf(a,b,c,d) vsnprintf(a,b,c,d) #endif @@ -186,6 +188,7 @@ extern std::string urlEncode (const std::string& strSrc) // IP Port parsing // // <-- iPort: "" = -1 +// VFALCO TODO Make this not require boost... and especially boost::asio bool parseIpPort (const std::string& strSource, std::string& strIP, int& iPort) { boost::smatch smMatch; @@ -233,10 +236,10 @@ bool parseUrl (const std::string& strUrl, std::string& strScheme, std::string& s boost::algorithm::to_lower (strScheme); iPort = strPort.empty () ? -1 : lexical_cast_s (strPort); - // std::cerr << strUrl << " : " << bMatch << " : '" << strDomain << "' : '" << strPort << "' : " << iPort << " : '" << strPath << "'" << std::endl; + // Log::out() << strUrl << " : " << bMatch << " : '" << strDomain << "' : '" << strPort << "' : " << iPort << " : '" << strPath << "'"; } - // std::cerr << strUrl << " : " << bMatch << " : '" << strDomain << "' : '" << strPath << "'" << std::endl; + // Log::out() << strUrl << " : " << bMatch << " : '" << strDomain << "' : '" << strPath << "'"; return bMatch; } @@ -260,52 +263,50 @@ bool parseQuality (const std::string& strSource, uint32& uQuality) return !!uQuality; } -BOOST_AUTO_TEST_SUITE ( Utils) - -BOOST_AUTO_TEST_CASE ( ParseUrl ) +std::string addressToString (void const* address) { - std::string strScheme; - std::string strDomain; - int iPort; - std::string strPath; - - if (!parseUrl ("lower://domain", strScheme, strDomain, iPort, strPath)) - BOOST_FAIL ("parseUrl: lower://domain failed"); - - if (strScheme != "lower") - BOOST_FAIL ("parseUrl: lower://domain : scheme failed"); - - if (strDomain != "domain") - BOOST_FAIL ("parseUrl: lower://domain : domain failed"); - - if (iPort != -1) - BOOST_FAIL ("parseUrl: lower://domain : port failed"); - - if (strPath != "") - BOOST_FAIL ("parseUrl: lower://domain : path failed"); - - if (!parseUrl ("UPPER://domain:234/", strScheme, strDomain, iPort, strPath)) - BOOST_FAIL ("parseUrl: UPPER://domain:234/ failed"); - - if (strScheme != "upper") - BOOST_FAIL ("parseUrl: UPPER://domain:234/ : scheme failed"); - - if (iPort != 234) - BOOST_FAIL (boost::str (boost::format ("parseUrl: UPPER://domain:234/ : port failed: %d") % iPort)); - - if (strPath != "/") - BOOST_FAIL ("parseUrl: UPPER://domain:234/ : path failed"); - - if (!parseUrl ("Mixed://domain/path", strScheme, strDomain, iPort, strPath)) - BOOST_FAIL ("parseUrl: Mixed://domain/path failed"); - - if (strScheme != "mixed") - BOOST_FAIL ("parseUrl: Mixed://domain/path tolower failed"); - - if (strPath != "/path") - BOOST_FAIL ("parseUrl: Mixed://domain/path path failed"); + // VFALCO TODO Clean this up, use uintptr_t and only produce a 32 bit + // output on 32 bit platforms + // + return strHex (static_cast (address) - static_cast (0)); } -BOOST_AUTO_TEST_SUITE_END () +StringPairArray parseKeyValueParameters (String parameters, beast_wchar delimiter) +{ + StringPairArray keyValues; + + while (parameters.isNotEmpty ()) + { + String pair; + + { + int const delimiterPos = parameters.indexOfChar (delimiter); + + if (delimiterPos != -1) + { + pair = parameters.substring (0, delimiterPos); + + parameters = parameters.substring (delimiterPos + 1); + } + else + { + pair = parameters; + + parameters = String::empty; + } + } + + int const equalPos = pair.indexOfChar ('='); + + if (equalPos != -1) + { + String const key = pair.substring (0, equalPos); + String const value = pair.substring (equalPos + 1, pair.length ()); + + keyValues.set (key, value); + } + } + + return keyValues; +} -// vim:ts=4 diff --git a/modules/ripple_basics/utility/ripple_StringUtilities.h b/modules/ripple_basics/utility/ripple_StringUtilities.h index ffa766a818..dd002a2a23 100644 --- a/modules/ripple_basics/utility/ripple_StringUtilities.h +++ b/modules/ripple_basics/utility/ripple_StringUtilities.h @@ -208,6 +208,14 @@ template std::string lexical_cast_it (const T& t) bool parseUrl (const std::string& strUrl, std::string& strScheme, std::string& strDomain, int& iPort, std::string& strPath); -#endif +#define ADDRESS(p) strHex(uint64( ((char*) p) - ((char*) 0))) -// vim:ts=4 +/** Convert a pointer address to a string for display purposes. +*/ +extern std::string addressToString (void const* address); + +/** Parse a pipe delimited key/value parameter string. +*/ +StringPairArray parseKeyValueParameters (String parameters, beast_wchar delimiter); + +#endif diff --git a/modules/ripple_basics/utility/ripple_StringUtilitiesUnitTests.cpp b/modules/ripple_basics/utility/ripple_StringUtilitiesUnitTests.cpp new file mode 100644 index 0000000000..f16fc80c8a --- /dev/null +++ b/modules/ripple_basics/utility/ripple_StringUtilitiesUnitTests.cpp @@ -0,0 +1,55 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +BOOST_AUTO_TEST_SUITE ( Utils) + +BOOST_AUTO_TEST_CASE ( ParseUrl ) +{ + using namespace ripple; + + std::string strScheme; + std::string strDomain; + int iPort; + std::string strPath; + + if (!parseUrl ("lower://domain", strScheme, strDomain, iPort, strPath)) + BOOST_FAIL ("parseUrl: lower://domain failed"); + + if (strScheme != "lower") + BOOST_FAIL ("parseUrl: lower://domain : scheme failed"); + + if (strDomain != "domain") + BOOST_FAIL ("parseUrl: lower://domain : domain failed"); + + if (iPort != -1) + BOOST_FAIL ("parseUrl: lower://domain : port failed"); + + if (strPath != "") + BOOST_FAIL ("parseUrl: lower://domain : path failed"); + + if (!parseUrl ("UPPER://domain:234/", strScheme, strDomain, iPort, strPath)) + BOOST_FAIL ("parseUrl: UPPER://domain:234/ failed"); + + if (strScheme != "upper") + BOOST_FAIL ("parseUrl: UPPER://domain:234/ : scheme failed"); + + if (iPort != 234) + BOOST_FAIL (boost::str (boost::format ("parseUrl: UPPER://domain:234/ : port failed: %d") % iPort)); + + if (strPath != "/") + BOOST_FAIL ("parseUrl: UPPER://domain:234/ : path failed"); + + if (!parseUrl ("Mixed://domain/path", strScheme, strDomain, iPort, strPath)) + BOOST_FAIL ("parseUrl: Mixed://domain/path failed"); + + if (strScheme != "mixed") + BOOST_FAIL ("parseUrl: Mixed://domain/path tolower failed"); + + if (strPath != "/path") + BOOST_FAIL ("parseUrl: Mixed://domain/path path failed"); +} + +BOOST_AUTO_TEST_SUITE_END () diff --git a/modules/ripple_basics/utility/ripple_Sustain.cpp b/modules/ripple_basics/utility/ripple_Sustain.cpp index a5b0b88e43..ac6514490d 100644 --- a/modules/ripple_basics/utility/ripple_Sustain.cpp +++ b/modules/ripple_basics/utility/ripple_Sustain.cpp @@ -34,7 +34,7 @@ std::string StopSustain () return "Terminating monitor"; } -std::string DoSustain () +std::string DoSustain (std::string logFile) { int childCount = 0; pManager = getpid (); @@ -72,7 +72,9 @@ std::string DoSustain () while (kill (pChild, 0) == 0); rename ("core", boost::str (boost::format ("core.%d") % static_cast (pChild)).c_str ()); - rename ("debug.log", boost::str (boost::format ("debug.log.%d") % static_cast (pChild)).c_str ()); + if (!logFile.empty()) + rename (logFile.c_str(), + boost::str (boost::format ("%s.%d") % logFile % static_cast (pChild)).c_str ()); } } @@ -82,7 +84,7 @@ bool HaveSustain () { return false; } -std::string DoSustain () +std::string DoSustain (std::string) { return std::string (); } diff --git a/modules/ripple_basics/utility/ripple_Sustain.h b/modules/ripple_basics/utility/ripple_Sustain.h index 2c2a2ce46b..cbd542266c 100644 --- a/modules/ripple_basics/utility/ripple_Sustain.h +++ b/modules/ripple_basics/utility/ripple_Sustain.h @@ -15,6 +15,6 @@ // extern bool HaveSustain (); extern std::string StopSustain (); -extern std::string DoSustain (); +extern std::string DoSustain (std::string logFile); #endif diff --git a/modules/ripple_basics/utility/ripple_Time.cpp b/modules/ripple_basics/utility/ripple_Time.cpp index ea7dc321a0..1a91d8f2fa 100644 --- a/modules/ripple_basics/utility/ripple_Time.cpp +++ b/modules/ripple_basics/utility/ripple_Time.cpp @@ -4,6 +4,8 @@ */ //============================================================================== +// VFALCO TODO Tidy this up into a RippleTime object + // // Time support // We have our own epoch. diff --git a/modules/ripple_basio/boost/ripple_IoService.cpp b/modules/ripple_basio/boost/ripple_IoService.cpp new file mode 100644 index 0000000000..ff71280882 --- /dev/null +++ b/modules/ripple_basio/boost/ripple_IoService.cpp @@ -0,0 +1,44 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace basio +{ + +IoService* IoService::New (std::size_t concurrency_hint) +{ + return new IoService (concurrency_hint); +} + +IoService::~IoService () +{ +} + +IoService::operator boost::asio::io_service& () +{ + return *m_impl; +} + +IoService::IoService (std::size_t concurrency_hint) + : m_impl (new boost::asio::io_service (concurrency_hint)) +{ +} + +void IoService::stop () +{ + m_impl->stop (); +} + +bool IoService::stopped () +{ + return m_impl->stopped (); +} + +void IoService::run () +{ + m_impl->run (); +} + +} diff --git a/modules/ripple_basio/boost/ripple_IoService.h b/modules/ripple_basio/boost/ripple_IoService.h new file mode 100644 index 0000000000..6afe49384a --- /dev/null +++ b/modules/ripple_basio/boost/ripple_IoService.h @@ -0,0 +1,37 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_IOSERVICE_H_INCLUDED +#define RIPPLE_IOSERVICE_H_INCLUDED + +namespace basio +{ + +/** Hides a boost::asio::ioservice implementation. +*/ +class IoService +{ +public: + static IoService* New (std::size_t concurrency_hint); + + virtual ~IoService (); + + operator boost::asio::io_service& (); + + void stop (); + bool stopped (); + void run (); + +private: + explicit IoService (std::size_t concurrency_hint); + +private: + beast::ScopedPointer m_impl; +}; + +} + +#endif diff --git a/modules/ripple_basio/boost/ripple_SslContext.cpp b/modules/ripple_basio/boost/ripple_SslContext.cpp new file mode 100644 index 0000000000..da6db5e3dd --- /dev/null +++ b/modules/ripple_basio/boost/ripple_SslContext.cpp @@ -0,0 +1,110 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace basio +{ + +SslContext* SslContext::New () +{ + return new SslContext; +} + +SslContext::~SslContext () +{ +} + +SslContext::operator boost::asio::ssl::context& () +{ + return *m_impl; +} + +SslContext::SslContext () + : m_impl (new boost::asio::ssl::context (boost::asio::ssl::context::sslv23)) +{ +} + +// VFALCO TODO Can we call this function from the ctor of PeerDoor as well? +// Or can we move the common code to a new function? +// +void SslContext::initializeFromFile ( + boost::asio::ssl::context& context, + std::string key_file, + std::string cert_file, + std::string chain_file) +{ + SSL_CTX* sslContext = context.native_handle (); + + context.set_options (boost::asio::ssl::context::default_workarounds | + boost::asio::ssl::context::no_sslv2 | + boost::asio::ssl::context::single_dh_use); + + bool cert_set = false; + + if (!cert_file.empty ()) + { + boost::system::error_code error; + context.use_certificate_file (cert_file, boost::asio::ssl::context::pem, error); + + if (error) + throw std::runtime_error ("Unable to use certificate file"); + + cert_set = true; + } + + if (!chain_file.empty ()) + { + // VFALCO Replace fopen() with RAII + FILE* f = fopen (chain_file.c_str (), "r"); + + if (!f) + throw std::runtime_error ("Unable to open chain file"); + + try + { + while (true) + { + X509* x = PEM_read_X509 (f, NULL, NULL, NULL); + + if (x == NULL) + break; + + if (!cert_set) + { + if (SSL_CTX_use_certificate (sslContext, x) != 1) + throw std::runtime_error ("Unable to get certificate from chain file"); + + cert_set = true; + } + else if (SSL_CTX_add_extra_chain_cert (sslContext, x) != 1) + { + X509_free (x); + throw std::runtime_error ("Unable to add chain certificate"); + } + } + + fclose (f); + } + catch (...) + { + fclose (f); + throw; + } + } + + if (!key_file.empty ()) + { + boost::system::error_code error; + context.use_private_key_file (key_file, boost::asio::ssl::context::pem, error); + + if (error) + throw std::runtime_error ("Unable to use private key file"); + } + + if (SSL_CTX_check_private_key (sslContext) != 1) + throw std::runtime_error ("Private key not valid"); +} + +} diff --git a/modules/ripple_basio/boost/ripple_SslContext.h b/modules/ripple_basio/boost/ripple_SslContext.h new file mode 100644 index 0000000000..f19c01f707 --- /dev/null +++ b/modules/ripple_basio/boost/ripple_SslContext.h @@ -0,0 +1,39 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_SSLCONTEXT_H_INCLUDED +#define RIPPLE_SSLCONTEXT_H_INCLUDED + +namespace basio +{ + +/** Hides a boost::asio::ssl::context implementation. +*/ +class SslContext +{ +public: + static SslContext* New (); + + virtual ~SslContext (); + + operator boost::asio::ssl::context& (); + + static void initializeFromFile ( + boost::asio::ssl::context& context, + std::string key_file, + std::string cert_file, + std::string chain_file); + +private: + SslContext (); + +private: + beast::ScopedPointer m_impl; +}; + +} + +#endif diff --git a/modules/ripple_basio/ripple_basio.cpp b/modules/ripple_basio/ripple_basio.cpp new file mode 100644 index 0000000000..9658501654 --- /dev/null +++ b/modules/ripple_basio/ripple_basio.cpp @@ -0,0 +1,27 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +/** Add this to get the @ref ripple_basio module. + + @file ripple_basio.cpp + @ingroup ripple_basio +*/ + +//------------------------------------------------------------------------------ + +#include "BeastConfig.h" + +#include "ripple_basio.h" + +#include "ripple_basio_impl.h" + +namespace ripple +{ + +#include "boost/ripple_IoService.cpp" +#include "boost/ripple_SslContext.cpp" + +} diff --git a/modules/ripple_basio/ripple_basio.h b/modules/ripple_basio/ripple_basio.h new file mode 100644 index 0000000000..7e71f339af --- /dev/null +++ b/modules/ripple_basio/ripple_basio.h @@ -0,0 +1,35 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_BASIO_H_INCLUDED +#define RIPPLE_BASIO_H_INCLUDED + +#include "beast/modules/beast_core/beast_core.h" + +// Must be outside the namespace + +#include "ripple_basio_fwdecl.h" + +/** Abstractions for boost::asio + + This is the first step to removing the dependency on boost::asio. + These classes are designed to move boost::asio header material out of + the majority of include paths. + + @ingroup ripple_basio + @file ripple_basio.h +*/ +namespace ripple +{ + +using namespace beast; + +#include "boost/ripple_IoService.h" +#include "boost/ripple_SslContext.h" + +} + +#endif diff --git a/modules/ripple_basio/ripple_basio_fwdecl.h b/modules/ripple_basio/ripple_basio_fwdecl.h new file mode 100644 index 0000000000..9d11bf637c --- /dev/null +++ b/modules/ripple_basio/ripple_basio_fwdecl.h @@ -0,0 +1,32 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_BASIO_FWDECL_H_INCLUDED +#define RIPPLE_BASIO_FWDECL_H_INCLUDED + +/** Forward declarations for boost::asio. + + These allow the header material for boost::asio to be omitted for + most translation units. +*/ + +namespace boost { + +namespace asio { + +class io_service; + +namespace ssl { + +class context; + +} + +} + +} + +#endif diff --git a/modules/ripple_basio/ripple_basio_impl.h b/modules/ripple_basio/ripple_basio_impl.h new file mode 100644 index 0000000000..9d5be45b1b --- /dev/null +++ b/modules/ripple_basio/ripple_basio_impl.h @@ -0,0 +1,24 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_BASIO_IMPL_H_INCLUDED +#define RIPPLE_BASIO_IMPL_H_INCLUDED + +// Once everything is converted, these can be moved to ripple_basio.cpp +#include +#include + +/** Implementations for boost::asio abstractions. + + These require boost::asio header material to be included. +*/ + +namespace ripple +{ + +} + +#endif diff --git a/modules/ripple_core/functional/ripple_Config.cpp b/modules/ripple_core/functional/ripple_Config.cpp index 1bedd00949..896d298280 100644 --- a/modules/ripple_core/functional/ripple_Config.cpp +++ b/modules/ripple_core/functional/ripple_Config.cpp @@ -21,7 +21,7 @@ #define SECTION_FEE_ACCOUNT_RESERVE "fee_account_reserve" #define SECTION_FEE_OWNER_RESERVE "fee_owner_reserve" #define SECTION_NODE_DB "node_db" -#define SECTION_LDB_EPHEMERAL "ephemeral_db" +#define SECTION_FASTNODE_DB "temp_db" #define SECTION_LEDGER_HISTORY "ledger_history" #define SECTION_IPS "ips" #define SECTION_NETWORK_QUORUM "network_quorum" @@ -197,9 +197,9 @@ void Config::setup (const std::string& strConf, bool bTestNet, bool bQuiet) // Update default values load (); - // std::cerr << "CONFIG FILE: " << CONFIG_FILE << std::endl; - // std::cerr << "CONFIG DIR: " << CONFIG_DIR << std::endl; - // std::cerr << "DATA DIR: " << DATA_DIR << std::endl; + // Log::out() << "CONFIG FILE: " << CONFIG_FILE; + // Log::out() << "CONFIG DIR: " << CONFIG_DIR; + // Log::out() << "DATA DIR: " << DATA_DIR; boost::filesystem::create_directories (DATA_DIR, ec); @@ -208,7 +208,8 @@ void Config::setup (const std::string& strConf, bool bTestNet, bool bQuiet) } Config::Config () - : SSL_CONTEXT (boost::asio::ssl::context::sslv23) + : m_rpcPort (5001) + , SSL_CONTEXT (boost::asio::ssl::context::sslv23) { // // Defaults @@ -218,7 +219,6 @@ Config::Config () NETWORK_START_TIME = 1319844908; PEER_PORT = SYSTEM_PEER_PORT; - RPC_PORT = 5001; RPC_SECURE = 0; WEBSOCKET_PORT = SYSTEM_WEBSOCKET_PORT; WEBSOCKET_PUBLIC_PORT = SYSTEM_WEBSOCKET_PUBLIC_PORT; @@ -263,9 +263,6 @@ Config::Config () SSL_VERIFY = true; - NODE_DB = "sqlite"; - - LDB_IMPORT = false; ELB_SUPPORT = false; RUN_STANDALONE = false; START_UP = NORMAL; @@ -274,13 +271,13 @@ Config::Config () void Config::load () { if (!QUIET) - std::cerr << "Loading: " << CONFIG_FILE << std::endl; + Log::out() << "Loading: " << CONFIG_FILE; std::ifstream ifsConfig (CONFIG_FILE.c_str (), std::ios::in); if (!ifsConfig) { - std::cerr << "Failed to open '" << CONFIG_FILE << "'." << std::endl; + Log::out() << "Failed to open '" << CONFIG_FILE << "'."; } else { @@ -291,7 +288,7 @@ void Config::load () if (ifsConfig.bad ()) { - std::cerr << "Failed to read '" << CONFIG_FILE << "'." << std::endl; + Log::out() << "Failed to read '" << CONFIG_FILE << "'."; } else { @@ -373,14 +370,14 @@ void Config::load () (void) SectionSingleB (secConfig, SECTION_RPC_ADMIN_PASSWORD, RPC_ADMIN_PASSWORD); (void) SectionSingleB (secConfig, SECTION_RPC_ADMIN_USER, RPC_ADMIN_USER); - (void) SectionSingleB (secConfig, SECTION_RPC_IP, RPC_IP); + (void) SectionSingleB (secConfig, SECTION_RPC_IP, m_rpcIP); (void) SectionSingleB (secConfig, SECTION_RPC_PASSWORD, RPC_PASSWORD); (void) SectionSingleB (secConfig, SECTION_RPC_USER, RPC_USER); (void) SectionSingleB (secConfig, SECTION_NODE_DB, NODE_DB); - (void) SectionSingleB (secConfig, SECTION_LDB_EPHEMERAL, LDB_EPHEMERAL); + (void) SectionSingleB (secConfig, SECTION_FASTNODE_DB, FASTNODE_DB); if (SectionSingleB (secConfig, SECTION_RPC_PORT, strTemp)) - RPC_PORT = boost::lexical_cast (strTemp); + m_rpcPort = boost::lexical_cast (strTemp); if (SectionSingleB (secConfig, "ledger_creator" , strTemp)) LEDGER_CREATOR = boost::lexical_cast (strTemp); @@ -578,5 +575,23 @@ int Config::getSize (SizedItemName item) return -1; } +void Config::setRpcIpAndOptionalPort (std::string const& newAddress) +{ + String const s (newAddress.c_str ()); + + int const colonPosition = s.lastIndexOfChar (':'); + + if (colonPosition != -1) + { + String const ipPart = s.substring (0, colonPosition); + String const portPart = s.substring (colonPosition + 1, s.length ()); + + setRpcIP (ipPart.toRawUTF8 ()); + setRpcPort (portPart.getIntValue ()); + } + else + { + setRpcIP (newAddress); + } +} -// vim:ts=4 diff --git a/modules/ripple_core/functional/ripple_Config.h b/modules/ripple_core/functional/ripple_Config.h index 865cc32c32..a4ea376650 100644 --- a/modules/ripple_core/functional/ripple_Config.h +++ b/modules/ripple_core/functional/ripple_Config.h @@ -85,8 +85,8 @@ public: boost::filesystem::path DEBUG_LOGFILE; boost::filesystem::path VALIDATORS_FILE; // As specifed in rippled.cfg. std::string NODE_DB; // Database to use for nodes - std::string LDB_EPHEMERAL; // Database for temporary storage - bool LDB_IMPORT; // Import into LevelDB + std::string FASTNODE_DB; // Database for temporary storage + std::string DB_IMPORT; // Import from old DB bool ELB_SUPPORT; // Support Amazon ELB std::string VALIDATORS_SITE; // Where to find validators.txt on the Internet. @@ -128,7 +128,7 @@ public: - The ledger is not advanced automatically. - If no ledger is loaded, the default ledger with the root account is created. - */ + */ bool RUN_STANDALONE; // Note: The following parameters do not relate to the UNL or trust at all @@ -160,9 +160,71 @@ public: std::string WEBSOCKET_SSL_CHAIN; std::string WEBSOCKET_SSL_KEY; + //---------------------------------------------------------------------------- + // + // VFALCO NOTE Please follow this style for modifying or adding code in the file. + // +public: + /** Get the client or server RPC IP address. + + @note The string may not always be in a valid parsable state. + + @return A string representing the address. + */ + std::string getRpcIP () const { return m_rpcIP; } + + /** Get the client or server RPC port number. + + @note The port number may be invalid (out of range or zero) + + @return The RPC port number. + */ + int getRpcPort () const { return m_rpcPort; } + + /** Set the client or server RPC IP and optional port. + + @note The string is not syntax checked. + + @param newAddress A string in the format [':'] + */ + void setRpcIpAndOptionalPort (std::string const& newAddress); + + /** Set the client or server RPC IP. + + @note The string is not syntax-checked. + + @param newIP A string representing the IP address to use. + */ + void setRpcIP (std::string const& newIP) { m_rpcIP = newIP; } + + /** Set the client or server RPC port number. + + @note The port number is not range checked. + + @param newPort The RPC port number to use. + */ + void setRpcPort (int newPort) { m_rpcPort = newPort; } + + /** Convert the RPC/port combination to a readable string. + */ + String const getRpcAddress () + { + String s; + + s << m_rpcIP.c_str () << ":" << m_rpcPort; + + return s; + } + +private: + std::string m_rpcIP; + // VFALCO TODO This should be a short. + int m_rpcPort; + // + //---------------------------------------------------------------------------- + +public: // RPC parameters - std::string RPC_IP; - int RPC_PORT; std::vector RPC_ADMIN_ALLOW; std::string RPC_ADMIN_PASSWORD; std::string RPC_ADMIN_USER; @@ -175,6 +237,7 @@ public: std::string RPC_SSL_CERT; std::string RPC_SSL_CHAIN; std::string RPC_SSL_KEY; + //---------------------------------------------------------------------------- // Path searching int PATH_SEARCH_SIZE; diff --git a/modules/ripple_core/functional/ripple_ILoadFeeTrack.h b/modules/ripple_core/functional/ripple_ILoadFeeTrack.h index 95a9bcfe20..f7fa1b2ab6 100644 --- a/modules/ripple_core/functional/ripple_ILoadFeeTrack.h +++ b/modules/ripple_core/functional/ripple_ILoadFeeTrack.h @@ -32,20 +32,22 @@ public: // Scale using load as well as base rate virtual uint64 scaleFeeLoad (uint64 fee, uint64 baseFee, uint32 referenceFeeUnits, bool bAdmin) = 0; - // VFALCO NOTE These appear to be unused, so I'm hiding the declarations. - // - //virtual uint32 getRemoteFee () = 0; - //virtual uint32 getLocalFee () = 0; - //virtual void setRemoteFee (uint32) = 0; + virtual void setRemoteFee (uint32) = 0; + + virtual uint32 getRemoteFee () = 0; + virtual uint32 getLocalFee () = 0; + virtual uint32 getClusterFee () = 0; virtual uint32 getLoadBase () = 0; virtual uint32 getLoadFactor () = 0; virtual Json::Value getJson (uint64 baseFee, uint32 referenceFeeUnits) = 0; + virtual void setClusterFee (uint32) = 0; virtual bool raiseLocalFee () = 0; virtual bool lowerLocalFee () = 0; - virtual bool isLoaded () = 0; + virtual bool isLoadedLocal () = 0; + virtual bool isLoadedCluster () = 0; }; #endif diff --git a/modules/ripple_core/functional/ripple_Job.cpp b/modules/ripple_core/functional/ripple_Job.cpp index fe0075a271..f9eb191849 100644 --- a/modules/ripple_core/functional/ripple_Job.cpp +++ b/modules/ripple_core/functional/ripple_Job.cpp @@ -7,17 +7,20 @@ Job::Job () : mType (jtINVALID) , mJobIndex (0) + , m_limit (0) { } Job::Job (JobType type, uint64 index) : mType (type) , mJobIndex (index) + , m_limit (0) { } Job::Job (JobType type, std::string const& name, + int limit, uint64 index, LoadMonitor& lm, FUNCTION_TYPE const& job) @@ -25,6 +28,7 @@ Job::Job (JobType type, , mJobIndex (index) , mJob (job) , mName (name) + , m_limit(limit) { m_loadEvent = boost::make_shared (boost::ref (lm), name, false); } @@ -36,15 +40,9 @@ JobType Job::getType () const void Job::doJob () { - m_loadEvent->start (); + m_loadEvent->reName (mName); mJob (*this); - - // VFALCO TODO Isn't there a way to construct the load event with - // the proper name? This way the load event object - // can have the invariant "name is always set" - // - m_loadEvent->reName (mName); } void Job::rename (std::string const& newName) @@ -52,6 +50,16 @@ void Job::rename (std::string const& newName) mName = newName; } +int Job::getLimit () const +{ + return m_limit; +} + +LoadEvent& Job::peekEvent() const +{ + return *m_loadEvent; +} + const char* Job::toString (JobType t) { switch (t) diff --git a/modules/ripple_core/functional/ripple_Job.h b/modules/ripple_core/functional/ripple_Job.h index a49589bf3f..67db68e5f4 100644 --- a/modules/ripple_core/functional/ripple_Job.h +++ b/modules/ripple_core/functional/ripple_Job.h @@ -66,6 +66,7 @@ public: // VFALCO TODO try to remove the dependency on LoadMonitor. Job (JobType type, std::string const& name, + int limit, uint64 index, LoadMonitor& lm, FUNCTION_TYPE const& job); @@ -76,6 +77,10 @@ public: void rename (const std::string& n); + int getLimit () const; + + LoadEvent& peekEvent() const; + // These comparison operators make the jobs sort in priority order in the job set bool operator< (const Job& j) const; bool operator> (const Job& j) const; @@ -90,6 +95,7 @@ private: FUNCTION_TYPE mJob; LoadEvent::pointer m_loadEvent; std::string mName; + int m_limit; }; #endif diff --git a/modules/ripple_core/functional/ripple_JobQueue.cpp b/modules/ripple_core/functional/ripple_JobQueue.cpp index e6741cd79f..f617f228e5 100644 --- a/modules/ripple_core/functional/ripple_JobQueue.cpp +++ b/modules/ripple_core/functional/ripple_JobQueue.cpp @@ -25,12 +25,17 @@ JobQueue::JobQueue (boost::asio::io_service& svc) mJobLoads [ jtPROPOSAL_t ].setTargetLatency (100, 500); mJobLoads [ jtCLIENT ].setTargetLatency (2000, 5000); - mJobLoads [ jtPEER ].setTargetLatency (200, 1250); + mJobLoads [ jtPEER ].setTargetLatency (200, 2500); mJobLoads [ jtDISK ].setTargetLatency (500, 1000); mJobLoads [ jtACCEPTLEDGER ].setTargetLatency (1000, 2500); } void JobQueue::addJob (JobType type, const std::string& name, const FUNCTION_TYPE& jobFunc) +{ + addLimitJob(type, name, 0, jobFunc); +} + +void JobQueue::addLimitJob (JobType type, const std::string& name, int limit, const FUNCTION_TYPE& jobFunc) { assert (type != jtINVALID); @@ -39,7 +44,9 @@ void JobQueue::addJob (JobType type, const std::string& name, const FUNCTION_TYP if (type != jtCLIENT) // FIXME: Workaround incorrect client shutdown ordering assert (mThreadCount != 0); // do not add jobs to a queue with no threads - mJobSet.insert (Job (type, name, ++mLastJob, mJobLoads[type], jobFunc)); + std::pair< std::set ::iterator, bool > it = + mJobSet.insert (Job (type, name, limit, ++mLastJob, mJobLoads[type], jobFunc)); + it.first->peekEvent().start(); // start timing how long it stays in the queue ++mJobCounts[type].first; mJobCond.notify_one (); } @@ -232,6 +239,37 @@ void JobQueue::setThreadCount (int c, bool const standaloneMode) mJobCond.notify_one (); // in case we sucked up someone else's signal } +bool JobQueue::getJob(Job& job) +{ + if (mJobSet.empty() || mShuttingDown) + return false; + + std::set::iterator it = mJobSet.begin (); + + while (1) + { + // Are we out of jobs? + if (it == mJobSet.end()) + return false; + + // Does this job have no limit? + if (it->getLimit() == 0) + break; + + // Is this job category below the limit? + if (mJobCounts[it->getType()].second < it->getLimit()) + break; + + // Try the next job, if any + ++it; + } + + job = *it; + mJobSet.erase (it); + + return true; +} + // do jobs until asked to stop void JobQueue::threadEntry () { @@ -239,27 +277,32 @@ void JobQueue::threadEntry () while (1) { + JobType type; + setCallingThreadName ("waiting"); - while (mJobSet.empty () && !mShuttingDown) { - mJobCond.wait (sl); - } - - if (mJobSet.empty ()) - break; - - JobType type; - std::set::iterator it = mJobSet.begin (); - { - Job job (*it); - mJobSet.erase (it); + Job job; + while (!getJob(job)) + { + if (mShuttingDown) + { + --mThreadCount; + mJobCond.notify_all(); + return; + } + mJobCond.wait (sl); + } type = job.getType (); -- (mJobCounts[type].first); if (type == jtDEATH) - break; + { + --mThreadCount; + mJobCond.notify_all(); + return; + } ++ (mJobCounts[type].second); sl.unlock (); @@ -267,12 +310,10 @@ void JobQueue::threadEntry () WriteLog (lsTRACE, JobQueue) << "Doing " << Job::toString (type) << " job"; job.doJob (); } // must destroy job without holding lock + sl.lock (); -- (mJobCounts[type].second); } - - --mThreadCount; - mJobCond.notify_all (); } // vim:ts=4 diff --git a/modules/ripple_core/functional/ripple_JobQueue.h b/modules/ripple_core/functional/ripple_JobQueue.h index 3c4f80d173..30f877197b 100644 --- a/modules/ripple_core/functional/ripple_JobQueue.h +++ b/modules/ripple_core/functional/ripple_JobQueue.h @@ -12,7 +12,11 @@ class JobQueue public: explicit JobQueue (boost::asio::io_service&); + // VFALCO TODO make convenience functions that allow the caller to not + // have to call bind. + // void addJob (JobType type, const std::string& name, const FUNCTION_TYPE& job); + void addLimitJob (JobType type, const std::string& name, int limit, const FUNCTION_TYPE& job); int getJobCount (JobType t); // Jobs waiting at this priority int getJobCountTotal (JobType t); // Jobs waiting plus running at this priority @@ -56,6 +60,8 @@ private: boost::asio::io_service& mIOService; std::map > mJobCounts; + + bool getJob (Job& job); }; #endif diff --git a/modules/ripple_core/functional/ripple_LoadEvent.h b/modules/ripple_core/functional/ripple_LoadEvent.h index a2abf8492e..d665ec0171 100644 --- a/modules/ripple_core/functional/ripple_LoadEvent.h +++ b/modules/ripple_core/functional/ripple_LoadEvent.h @@ -10,7 +10,7 @@ class LoadMonitor; // VFALCO NOTE What is the difference between a LoadEvent and a LoadMonitor? -// VFALCO TODO Rename LoadEvent to LoadMonitor::Event +// VFALCO TODO Rename LoadEvent to ScopedLoadSample // // This looks like a scoped elapsed time measuring class // diff --git a/modules/ripple_core/functional/ripple_LoadFeeTrack.cpp b/modules/ripple_core/functional/ripple_LoadFeeTrack.cpp index 35e14b8efd..630f24a38b 100644 --- a/modules/ripple_core/functional/ripple_LoadFeeTrack.cpp +++ b/modules/ripple_core/functional/ripple_LoadFeeTrack.cpp @@ -4,209 +4,7 @@ */ //============================================================================== -class LoadManager; - -class LoadFeeTrack : public ILoadFeeTrack -{ -public: - LoadFeeTrack () - : mLocalTxnLoadFee (lftNormalFee) - , mRemoteTxnLoadFee (lftNormalFee) - , raiseCount (0) - { - } - - // Scale using load as well as base rate - uint64 scaleFeeLoad (uint64 fee, uint64 baseFee, uint32 referenceFeeUnits, bool bAdmin) - { - static uint64 midrange (0x00000000FFFFFFFF); - - bool big = (fee > midrange); - - if (big) // big fee, divide first to avoid overflow - fee /= baseFee; - else // normal fee, multiply first for accuracy - fee *= referenceFeeUnits; - - uint32 feeFactor = std::max (mLocalTxnLoadFee, mRemoteTxnLoadFee); - - // Let admins pay the normal fee until the local load exceeds four times the remote - if (bAdmin && (feeFactor > mRemoteTxnLoadFee) && (feeFactor < (4 * mRemoteTxnLoadFee))) - feeFactor = mRemoteTxnLoadFee; - - { - boost::mutex::scoped_lock sl (mLock); - fee = mulDiv (fee, feeFactor, lftNormalFee); - } - - if (big) // Fee was big to start, must now multiply - fee *= referenceFeeUnits; - else // Fee was small to start, mst now divide - fee /= baseFee; - - return fee; - } - - // Scale from fee units to millionths of a ripple - uint64 scaleFeeBase (uint64 fee, uint64 baseFee, uint32 referenceFeeUnits) - { - return mulDiv (fee, referenceFeeUnits, baseFee); - } - - uint32 getRemoteFee () - { - boost::mutex::scoped_lock sl (mLock); - return mRemoteTxnLoadFee; - } - - uint32 getLocalFee () - { - boost::mutex::scoped_lock sl (mLock); - return mLocalTxnLoadFee; - } - - uint32 getLoadBase () - { - return lftNormalFee; - } - - uint32 getLoadFactor () - { - boost::mutex::scoped_lock sl (mLock); - return std::max (mLocalTxnLoadFee, mRemoteTxnLoadFee); - } - - bool isLoaded () - { - boost::mutex::scoped_lock sl (mLock); - return (raiseCount != 0) || (mLocalTxnLoadFee != lftNormalFee); - } - - void setRemoteFee (uint32 f) - { - boost::mutex::scoped_lock sl (mLock); - mRemoteTxnLoadFee = f; - } - - bool raiseLocalFee () - { - boost::mutex::scoped_lock sl (mLock); - - if (++raiseCount < 2) - return false; - - uint32 origFee = mLocalTxnLoadFee; - - if (mLocalTxnLoadFee < mRemoteTxnLoadFee) // make sure this fee takes effect - mLocalTxnLoadFee = mRemoteTxnLoadFee; - - mLocalTxnLoadFee += (mLocalTxnLoadFee / lftFeeIncFraction); // increment by 1/16th - - if (mLocalTxnLoadFee > lftFeeMax) - mLocalTxnLoadFee = lftFeeMax; - - if (origFee == mLocalTxnLoadFee) - return false; - - WriteLog (lsDEBUG, LoadManager) << "Local load fee raised from " << origFee << " to " << mLocalTxnLoadFee; - return true; - } - - bool lowerLocalFee () - { - boost::mutex::scoped_lock sl (mLock); - uint32 origFee = mLocalTxnLoadFee; - raiseCount = 0; - - mLocalTxnLoadFee -= (mLocalTxnLoadFee / lftFeeDecFraction ); // reduce by 1/4 - - if (mLocalTxnLoadFee < lftNormalFee) - mLocalTxnLoadFee = lftNormalFee; - - if (origFee == mLocalTxnLoadFee) - return false; - - WriteLog (lsDEBUG, LoadManager) << "Local load fee lowered from " << origFee << " to " << mLocalTxnLoadFee; - return true; - } - - Json::Value getJson (uint64 baseFee, uint32 referenceFeeUnits) - { - Json::Value j (Json::objectValue); - - { - boost::mutex::scoped_lock sl (mLock); - - // base_fee = The cost to send a "reference" transaction under no load, in millionths of a Ripple - j["base_fee"] = Json::Value::UInt (baseFee); - - // load_fee = The cost to send a "reference" transaction now, in millionths of a Ripple - j["load_fee"] = Json::Value::UInt ( - mulDiv (baseFee, std::max (mLocalTxnLoadFee, mRemoteTxnLoadFee), lftNormalFee)); - } - - return j; - } - -private: - // VFALCO TODO Move this function to some "math utilities" file - // compute (value)*(mul)/(div) - avoid overflow but keep precision - uint64 mulDiv (uint64 value, uint32 mul, uint64 div) - { - // VFALCO TODO replace with beast::literal64bitUnsigned () - // - static uint64 boundary = (0x00000000FFFFFFFF); - - if (value > boundary) // Large value, avoid overflow - return (value / div) * mul; - else // Normal value, preserve accuracy - return (value * mul) / div; - } - -private: - static const int lftNormalFee = 256; // 256 is the minimum/normal load factor - static const int lftFeeIncFraction = 16; // increase fee by 1/16 - static const int lftFeeDecFraction = 4; // decrease fee by 1/4 - static const int lftFeeMax = lftNormalFee * 1000000; - - uint32 mLocalTxnLoadFee; // Scale factor, lftNormalFee = normal fee - uint32 mRemoteTxnLoadFee; // Scale factor, lftNormalFee = normal fee - int raiseCount; - - boost::mutex mLock; -}; - -//------------------------------------------------------------------------------ - ILoadFeeTrack* ILoadFeeTrack::New () { return new LoadFeeTrack; } - -//------------------------------------------------------------------------------ - -BOOST_AUTO_TEST_SUITE (LoadManager_test) - -BOOST_AUTO_TEST_CASE (LoadFeeTrack_test) -{ - WriteLog (lsDEBUG, LoadManager) << "Running load fee track test"; - - Config d; // get a default configuration object - LoadFeeTrack l; - - BOOST_REQUIRE_EQUAL (l.scaleFeeBase (10000, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 10000); - BOOST_REQUIRE_EQUAL (l.scaleFeeLoad (10000, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE, false), 10000); - BOOST_REQUIRE_EQUAL (l.scaleFeeBase (1, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 1); - BOOST_REQUIRE_EQUAL (l.scaleFeeLoad (1, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE, false), 1); - - // Check new default fee values give same fees as old defaults - BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_DEFAULT, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 10); - BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_ACCOUNT_RESERVE, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 200 * SYSTEM_CURRENCY_PARTS); - BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_OWNER_RESERVE, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 50 * SYSTEM_CURRENCY_PARTS); - BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_NICKNAME_CREATE, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 1000); - BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_OFFER, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 10); - BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_CONTRACT_OPERATION, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 1); - -} - -BOOST_AUTO_TEST_SUITE_END () diff --git a/modules/ripple_core/functional/ripple_LoadFeeTrack.h b/modules/ripple_core/functional/ripple_LoadFeeTrack.h new file mode 100644 index 0000000000..d65d296cd8 --- /dev/null +++ b/modules/ripple_core/functional/ripple_LoadFeeTrack.h @@ -0,0 +1,216 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_LOADFEETRACK_RIPPLEHEADER +#define RIPPLE_LOADFEETRACK_RIPPLEHEADER + +// PRIVATE HEADER +class LoadManager; + +class LoadFeeTrack : public ILoadFeeTrack +{ +public: + LoadFeeTrack () + : mLocalTxnLoadFee (lftNormalFee) + , mRemoteTxnLoadFee (lftNormalFee) + , mClusterTxnLoadFee (lftNormalFee) + , raiseCount (0) + { + } + + // Scale using load as well as base rate + uint64 scaleFeeLoad (uint64 fee, uint64 baseFee, uint32 referenceFeeUnits, bool bAdmin) + { + static uint64 midrange (0x00000000FFFFFFFF); + + bool big = (fee > midrange); + + if (big) // big fee, divide first to avoid overflow + fee /= baseFee; + else // normal fee, multiply first for accuracy + fee *= referenceFeeUnits; + + uint32 feeFactor = std::max (mLocalTxnLoadFee, mRemoteTxnLoadFee); + + // Let admins pay the normal fee until the local load exceeds four times the remote + uint32 uRemFee = std::max(mRemoteTxnLoadFee, mClusterTxnLoadFee); + if (bAdmin && (feeFactor > uRemFee) && (feeFactor < (4 * uRemFee))) + feeFactor = uRemFee; + + { + boost::mutex::scoped_lock sl (mLock); + fee = mulDiv (fee, feeFactor, lftNormalFee); + } + + if (big) // Fee was big to start, must now multiply + fee *= referenceFeeUnits; + else // Fee was small to start, mst now divide + fee /= baseFee; + + return fee; + } + + // Scale from fee units to millionths of a ripple + uint64 scaleFeeBase (uint64 fee, uint64 baseFee, uint32 referenceFeeUnits) + { + return mulDiv (fee, referenceFeeUnits, baseFee); + } + + uint32 getRemoteFee () + { + boost::mutex::scoped_lock sl (mLock); + return mRemoteTxnLoadFee; + } + + uint32 getLocalFee () + { + boost::mutex::scoped_lock sl (mLock); + return mLocalTxnLoadFee; + } + + uint32 getLoadBase () + { + return lftNormalFee; + } + + uint32 getLoadFactor () + { + boost::mutex::scoped_lock sl (mLock); + return std::max(mClusterTxnLoadFee, std::max (mLocalTxnLoadFee, mRemoteTxnLoadFee)); + } + + void setClusterFee (uint32 fee) + { + boost::mutex::scoped_lock sl (mLock); + mClusterTxnLoadFee = fee; + } + + uint32 getClusterFee () + { + boost::mutex::scoped_lock sl (mLock); + return mClusterTxnLoadFee; + } + + bool isLoadedLocal () + { + // VFALCO TODO This could be replaced with a SharedData and + // using a read/write lock instead of a critical section. + // + // NOTE This applies to all the locking in this class. + // + // + boost::mutex::scoped_lock sl (mLock); + return (raiseCount != 0) || (mLocalTxnLoadFee != lftNormalFee); + } + + bool isLoadedCluster () + { + // VFALCO TODO This could be replaced with a SharedData and + // using a read/write lock instead of a critical section. + // + // NOTE This applies to all the locking in this class. + // + // + boost::mutex::scoped_lock sl (mLock); + return (raiseCount != 0) || (mLocalTxnLoadFee != lftNormalFee) || (mClusterTxnLoadFee != lftNormalFee); + } + + void setRemoteFee (uint32 f) + { + boost::mutex::scoped_lock sl (mLock); + mRemoteTxnLoadFee = f; + } + + bool raiseLocalFee () + { + boost::mutex::scoped_lock sl (mLock); + + if (++raiseCount < 2) + return false; + + uint32 origFee = mLocalTxnLoadFee; + + if (mLocalTxnLoadFee < mRemoteTxnLoadFee) // make sure this fee takes effect + mLocalTxnLoadFee = mRemoteTxnLoadFee; + + mLocalTxnLoadFee += (mLocalTxnLoadFee / lftFeeIncFraction); // increment by 1/16th + + if (mLocalTxnLoadFee > lftFeeMax) + mLocalTxnLoadFee = lftFeeMax; + + if (origFee == mLocalTxnLoadFee) + return false; + + WriteLog (lsDEBUG, LoadManager) << "Local load fee raised from " << origFee << " to " << mLocalTxnLoadFee; + return true; + } + + bool lowerLocalFee () + { + boost::mutex::scoped_lock sl (mLock); + uint32 origFee = mLocalTxnLoadFee; + raiseCount = 0; + + mLocalTxnLoadFee -= (mLocalTxnLoadFee / lftFeeDecFraction ); // reduce by 1/4 + + if (mLocalTxnLoadFee < lftNormalFee) + mLocalTxnLoadFee = lftNormalFee; + + if (origFee == mLocalTxnLoadFee) + return false; + + WriteLog (lsDEBUG, LoadManager) << "Local load fee lowered from " << origFee << " to " << mLocalTxnLoadFee; + return true; + } + + Json::Value getJson (uint64 baseFee, uint32 referenceFeeUnits) + { + Json::Value j (Json::objectValue); + + { + boost::mutex::scoped_lock sl (mLock); + + // base_fee = The cost to send a "reference" transaction under no load, in millionths of a Ripple + j["base_fee"] = Json::Value::UInt (baseFee); + + // load_fee = The cost to send a "reference" transaction now, in millionths of a Ripple + j["load_fee"] = Json::Value::UInt ( + mulDiv (baseFee, std::max (mLocalTxnLoadFee, mRemoteTxnLoadFee), lftNormalFee)); + } + + return j; + } + +private: + // VFALCO TODO Move this function to some "math utilities" file + // compute (value)*(mul)/(div) - avoid overflow but keep precision + uint64 mulDiv (uint64 value, uint32 mul, uint64 div) + { + // VFALCO TODO replace with beast::literal64bitUnsigned () + // + static uint64 boundary = (0x00000000FFFFFFFF); + + if (value > boundary) // Large value, avoid overflow + return (value / div) * mul; + else // Normal value, preserve accuracy + return (value * mul) / div; + } + +private: + static const int lftNormalFee = 256; // 256 is the minimum/normal load factor + static const int lftFeeIncFraction = 4; // increase fee by 1/4 + static const int lftFeeDecFraction = 4; // decrease fee by 1/4 + static const int lftFeeMax = lftNormalFee * 1000000; + + uint32 mLocalTxnLoadFee; // Scale factor, lftNormalFee = normal fee + uint32 mRemoteTxnLoadFee; // Scale factor, lftNormalFee = normal fee + uint32 mClusterTxnLoadFee; // Scale factor, lftNormalFee = normal fee + int raiseCount; + + boost::mutex mLock; +}; + +#endif diff --git a/modules/ripple_core/functional/ripple_LoadFeeTrackUnitTests.cpp b/modules/ripple_core/functional/ripple_LoadFeeTrackUnitTests.cpp new file mode 100644 index 0000000000..167c08be3a --- /dev/null +++ b/modules/ripple_core/functional/ripple_LoadFeeTrackUnitTests.cpp @@ -0,0 +1,32 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== +BOOST_AUTO_TEST_SUITE (LoadManager_test) + +BOOST_AUTO_TEST_CASE (LoadFeeTrack_test) +{ + using namespace ripple; + + WriteLog (lsDEBUG, LoadManager) << "Running load fee track test"; + + Config d; // get a default configuration object + LoadFeeTrack l; + + BOOST_REQUIRE_EQUAL (l.scaleFeeBase (10000, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 10000); + BOOST_REQUIRE_EQUAL (l.scaleFeeLoad (10000, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE, false), 10000); + BOOST_REQUIRE_EQUAL (l.scaleFeeBase (1, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 1); + BOOST_REQUIRE_EQUAL (l.scaleFeeLoad (1, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE, false), 1); + + // Check new default fee values give same fees as old defaults + BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_DEFAULT, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 10); + BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_ACCOUNT_RESERVE, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 200 * SYSTEM_CURRENCY_PARTS); + BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_OWNER_RESERVE, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 50 * SYSTEM_CURRENCY_PARTS); + BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_NICKNAME_CREATE, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 1000); + BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_OFFER, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 10); + BOOST_REQUIRE_EQUAL (l.scaleFeeBase (d.FEE_CONTRACT_OPERATION, d.FEE_DEFAULT, d.TRANSACTION_FEE_BASE), 1); + +} + +BOOST_AUTO_TEST_SUITE_END () diff --git a/modules/ripple_core/ripple_core.cpp b/modules/ripple_core/ripple_core.cpp index 0c43bd21ca..96b4a79e13 100644 --- a/modules/ripple_core/ripple_core.cpp +++ b/modules/ripple_core/ripple_core.cpp @@ -10,18 +10,29 @@ @ingroup ripple_core */ +#include "BeastConfig.h" + #include "ripple_core.h" #include #include #include -#include #include +namespace ripple +{ + #include "functional/ripple_Config.cpp" +#include "functional/ripple_LoadFeeTrack.h" // private #include "functional/ripple_LoadFeeTrack.cpp" #include "functional/ripple_Job.cpp" #include "functional/ripple_JobQueue.cpp" #include "functional/ripple_LoadEvent.cpp" #include "functional/ripple_LoadMonitor.cpp" + +} + +// These must be outside the namespace + +#include "functional/ripple_LoadFeeTrackUnitTests.cpp" diff --git a/modules/ripple_core/ripple_core.h b/modules/ripple_core/ripple_core.h index eb5c6f58df..0535edad3f 100644 --- a/modules/ripple_core/ripple_core.h +++ b/modules/ripple_core/ripple_core.h @@ -22,15 +22,11 @@ #define RIPPLE_CORE_RIPPLEHEADER #include "../ripple_basics/ripple_basics.h" + #include "../ripple_data/ripple_data.h" -#include -#include - -#include -#include -#include -#include +namespace ripple +{ // VFALCO NOTE Indentation shows dependency hierarchy // @@ -41,4 +37,6 @@ /*.*/#include "functional/ripple_Job.h" /**/#include "functional/ripple_JobQueue.h" +} + #endif diff --git a/modules/ripple_data/crypto/ripple_CKeyDeterministic.cpp b/modules/ripple_data/crypto/ripple_CKeyDeterministic.cpp index c60ed4e3fe..a07bac3231 100644 --- a/modules/ripple_data/crypto/ripple_CKeyDeterministic.cpp +++ b/modules/ripple_data/crypto/ripple_CKeyDeterministic.cpp @@ -346,38 +346,3 @@ EC_KEY* CKey::GeneratePrivateDeterministicKey (const RippleAddress& pubGen, cons return pkey; } - -BOOST_AUTO_TEST_SUITE (DeterministicKeys_test) - -BOOST_AUTO_TEST_CASE (DeterminsticKeys_test1) -{ - Log (lsDEBUG) << "Beginning deterministic key test"; - - uint128 seed1, seed2; - seed1.SetHex ("71ED064155FFADFA38782C5E0158CB26"); - seed2.SetHex ("CF0C3BE4485961858C4198515AE5B965"); - CKey root1 (seed1), root2 (seed2); - - uint256 priv1, priv2; - root1.GetPrivateKeyU (priv1); - root2.GetPrivateKeyU (priv2); - - if (priv1.GetHex () != "7CFBA64F771E93E817E15039215430B53F7401C34931D111EAB3510B22DBB0D8") - BOOST_FAIL ("Incorrect private key for generator"); - - if (priv2.GetHex () != "98BC2EACB26EB021D1A6293C044D88BA2F0B6729A2772DEEBF2E21A263C1740B") - BOOST_FAIL ("Incorrect private key for generator"); - - RippleAddress nSeed; - nSeed.setSeed (seed1); - - if (nSeed.humanSeed () != "shHM53KPZ87Gwdqarm1bAmPeXg8Tn") - BOOST_FAIL ("Incorrect human seed"); - - if (nSeed.humanSeed1751 () != "MAD BODY ACE MINT OKAY HUB WHAT DATA SACK FLAT DANA MATH") - BOOST_FAIL ("Incorrect 1751 seed"); -} - -BOOST_AUTO_TEST_SUITE_END (); - -// vim:ts=4 diff --git a/modules/ripple_data/crypto/ripple_CKeyDeterministicUnitTests.cpp b/modules/ripple_data/crypto/ripple_CKeyDeterministicUnitTests.cpp new file mode 100644 index 0000000000..0f37dd795c --- /dev/null +++ b/modules/ripple_data/crypto/ripple_CKeyDeterministicUnitTests.cpp @@ -0,0 +1,39 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== +BOOST_AUTO_TEST_SUITE (DeterministicKeys_test) + +BOOST_AUTO_TEST_CASE (DeterminsticKeys_test1) +{ + using namespace ripple; + + Log (lsDEBUG) << "Beginning deterministic key test"; + + uint128 seed1, seed2; + seed1.SetHex ("71ED064155FFADFA38782C5E0158CB26"); + seed2.SetHex ("CF0C3BE4485961858C4198515AE5B965"); + CKey root1 (seed1), root2 (seed2); + + uint256 priv1, priv2; + root1.GetPrivateKeyU (priv1); + root2.GetPrivateKeyU (priv2); + + if (priv1.GetHex () != "7CFBA64F771E93E817E15039215430B53F7401C34931D111EAB3510B22DBB0D8") + BOOST_FAIL ("Incorrect private key for generator"); + + if (priv2.GetHex () != "98BC2EACB26EB021D1A6293C044D88BA2F0B6729A2772DEEBF2E21A263C1740B") + BOOST_FAIL ("Incorrect private key for generator"); + + RippleAddress nSeed; + nSeed.setSeed (seed1); + + if (nSeed.humanSeed () != "shHM53KPZ87Gwdqarm1bAmPeXg8Tn") + BOOST_FAIL ("Incorrect human seed"); + + if (nSeed.humanSeed1751 () != "MAD BODY ACE MINT OKAY HUB WHAT DATA SACK FLAT DANA MATH") + BOOST_FAIL ("Incorrect 1751 seed"); +} + +BOOST_AUTO_TEST_SUITE_END (); diff --git a/modules/ripple_data/crypto/ripple_CKeyECIES.cpp b/modules/ripple_data/crypto/ripple_CKeyECIES.cpp index 0903e01272..1c4ba5184f 100644 --- a/modules/ripple_data/crypto/ripple_CKeyECIES.cpp +++ b/modules/ripple_data/crypto/ripple_CKeyECIES.cpp @@ -277,7 +277,7 @@ bool checkECIES (void) if ((i % 100) == 0) { // generate new keys every 100 times - // std::cerr << "new keys" << std::endl; + // Log::out() << "new keys"; senderPriv.MakeNewKey (); recipientPriv.MakeNewKey (); @@ -307,7 +307,7 @@ bool checkECIES (void) return false; } - // std::cerr << "Msg(" << msglen << ") ok " << ciphertext.size() << std::endl; + //Log::out() << "Msg(" << msglen << ") ok " << ciphertext.size(); } return true; diff --git a/modules/ripple_data/protocol/ripple_KnownFormats.h b/modules/ripple_data/protocol/ripple_KnownFormats.h new file mode 100644 index 0000000000..72a78eb34b --- /dev/null +++ b/modules/ripple_data/protocol/ripple_KnownFormats.h @@ -0,0 +1,172 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_KNOWNFORMATS_H_INCLUDED +#define RIPPLE_KNOWNFORMATS_H_INCLUDED + +/** Manages a list of known formats. + + Each format has a name, an associated KeyType (typically an enumeration), + and a predefined @ref SOElement. + + @tparam KeyType The type of key identifying the format. +*/ +template +class KnownFormats +{ +public: + /** A known format. + */ + class Item + { + public: + Item (char const* name, KeyType type) + : m_name (name) + , m_type (type) + { + } + + Item& operator<< (SOElement const& el) + { + elements.push_back (el); + + return *this; + } + + /** Retrieve the name of the format. + */ + std::string const& getName () const noexcept + { + return m_name; + } + + /** Retrieve the transaction type this format represents. + */ + KeyType getType () const noexcept + { + return m_type; + } + + public: + // VFALCO TODO make an accessor for this + SOTemplate elements; + + private: + std::string const m_name; + KeyType const m_type; + }; + +private: + // VFALCO TODO use String instead of std::string + typedef std::map NameMap; + typedef std::map TypeMap; + +public: + /** Create the known formats object. + + Derived classes will load the object will all the known formats. + */ + KnownFormats () + { + } + + /** Destroy the known formats object. + + The defined formats are deleted. + */ + ~KnownFormats () + { + } + + /** Retrieve the type for a format specified by name. + + If the format name is unknown, an exception is thrown. + + @param name The name of the type. + @return The type. + */ + KeyType findTypeByName (std::string const name) const + { + Item const* const result = findByName (name); + + if (result != nullptr) + { + return result->getType (); + } + else + { + throw std::runtime_error ("Unknown format name"); + } + } + + /** Retrieve a format based on its type. + */ + // VFALCO TODO Can we just return the SOElement& ? + Item const* findByType (KeyType type) const noexcept + { + Item* result = nullptr; + + typename TypeMap::const_iterator const iter = m_types.find (type); + + if (iter != m_types.end ()) + { + result = iter->second; + } + + return result; + } + +protected: + /** Retrieve a format based on its name. + */ + Item const* findByName (std::string const& name) const noexcept + { + Item* result = nullptr; + + typename NameMap::const_iterator const iter = m_names.find (name); + + if (iter != m_names.end ()) + { + result = iter->second; + } + + return result; + } + + /** Add a new format. + + The new format has the set of common fields already added. + + @param name The name of this format. + @param type The type of this format. + + @return The created format. + */ + Item& add (char const* name, KeyType type) + { + Item& item = *m_formats.add (new Item (name, type)); + + addCommonFields (item); + + m_types [item.getType ()] = &item; + m_names [item.getName ()] = &item; + + return item; + } + + /** Adds common fields. + + This is called for every new item. + */ + virtual void addCommonFields (Item& item) = 0; + +private: + OwnedArray m_formats; + NameMap m_names; + TypeMap m_types; +}; + +#endif diff --git a/modules/ripple_data/protocol/ripple_LedgerFormat.cpp b/modules/ripple_data/protocol/ripple_LedgerFormats.cpp similarity index 74% rename from modules/ripple_data/protocol/ripple_LedgerFormat.cpp rename to modules/ripple_data/protocol/ripple_LedgerFormats.cpp index 5231ea182b..4c5182e6f1 100644 --- a/modules/ripple_data/protocol/ripple_LedgerFormat.cpp +++ b/modules/ripple_data/protocol/ripple_LedgerFormats.cpp @@ -4,22 +4,10 @@ */ //============================================================================== -std::map LedgerEntryFormat::byType; - -std::map LedgerEntryFormat::byName; - -#define LEF_BASE \ - << SOElement(sfLedgerIndex, SOE_OPTIONAL) \ - << SOElement(sfLedgerEntryType, SOE_REQUIRED) \ - << SOElement(sfFlags, SOE_REQUIRED) - -#define DECLARE_LEF(name, type) lef = new LedgerEntryFormat(#name, type); (*lef) LEF_BASE - -void LEFInit () +LedgerFormats::LedgerFormats () + : SharedSingleton (SingletonLifetime::persistAfterCreation) { - LedgerEntryFormat* lef; - - DECLARE_LEF (AccountRoot, ltACCOUNT_ROOT) + add ("AccountRoot", ltACCOUNT_ROOT) << SOElement (sfAccount, SOE_REQUIRED) << SOElement (sfSequence, SOE_REQUIRED) << SOElement (sfBalance, SOE_REQUIRED) @@ -35,7 +23,7 @@ void LEFInit () << SOElement (sfDomain, SOE_OPTIONAL) ; - DECLARE_LEF (Contract, ltCONTRACT) + add ("Contract", ltCONTRACT) << SOElement (sfAccount, SOE_REQUIRED) << SOElement (sfBalance, SOE_REQUIRED) << SOElement (sfPreviousTxnID, SOE_REQUIRED) @@ -50,7 +38,7 @@ void LEFInit () << SOElement (sfExpireCode, SOE_OPTIONAL) ; - DECLARE_LEF (DirectoryNode, ltDIR_NODE) + add ("DirectoryNode", ltDIR_NODE) << SOElement (sfOwner, SOE_OPTIONAL) // for owner directories << SOElement (sfTakerPaysCurrency, SOE_OPTIONAL) // for order book directories << SOElement (sfTakerPaysIssuer, SOE_OPTIONAL) // for order book directories @@ -63,16 +51,16 @@ void LEFInit () << SOElement (sfIndexPrevious, SOE_OPTIONAL) ; - DECLARE_LEF (GeneratorMap, ltGENERATOR_MAP) + add ("GeneratorMap", ltGENERATOR_MAP) << SOElement (sfGenerator, SOE_REQUIRED) ; - DECLARE_LEF (Nickname, ltNICKNAME) + add ("Nickname", ltNICKNAME) << SOElement (sfAccount, SOE_REQUIRED) << SOElement (sfMinimumOffer, SOE_OPTIONAL) ; - DECLARE_LEF (Offer, ltOFFER) + add ("Offer", ltOFFER) << SOElement (sfAccount, SOE_REQUIRED) << SOElement (sfSequence, SOE_REQUIRED) << SOElement (sfTakerPays, SOE_REQUIRED) @@ -85,7 +73,7 @@ void LEFInit () << SOElement (sfExpiration, SOE_OPTIONAL) ; - DECLARE_LEF (RippleState, ltRIPPLE_STATE) + add ("RippleState", ltRIPPLE_STATE) << SOElement (sfBalance, SOE_REQUIRED) << SOElement (sfLowLimit, SOE_REQUIRED) << SOElement (sfHighLimit, SOE_REQUIRED) @@ -99,17 +87,17 @@ void LEFInit () << SOElement (sfHighQualityOut, SOE_OPTIONAL) ; - DECLARE_LEF (LedgerHashes, ltLEDGER_HASHES) + add ("LedgerHashes", ltLEDGER_HASHES) << SOElement (sfFirstLedgerSequence, SOE_OPTIONAL) // Remove if we do a ledger restart << SOElement (sfLastLedgerSequence, SOE_OPTIONAL) << SOElement (sfHashes, SOE_REQUIRED) ; - DECLARE_LEF (EnabledFeatures, ltFEATURES) + add ("EnabledFeatures", ltFEATURES) << SOElement (sfFeatures, SOE_REQUIRED) ; - DECLARE_LEF (FeeSettings, ltFEE_SETTINGS) + add ("FeeSettings", ltFEE_SETTINGS) << SOElement (sfBaseFee, SOE_REQUIRED) << SOElement (sfReferenceFeeUnits, SOE_REQUIRED) << SOElement (sfReserveBase, SOE_REQUIRED) @@ -117,34 +105,16 @@ void LEFInit () ; } -LedgerEntryFormat* LedgerEntryFormat::getLgrFormat (LedgerEntryType t) +LedgerFormats* LedgerFormats::createInstance () { - std::map::iterator it = byType.find (static_cast (t)); - - if (it == byType.end ()) - return NULL; - - return it->second; + return new LedgerFormats; } -LedgerEntryFormat* LedgerEntryFormat::getLgrFormat (int t) +void LedgerFormats::addCommonFields (Item& item) { - std::map::iterator it = byType.find ((t)); - - if (it == byType.end ()) - return NULL; - - return it->second; + item + << SOElement(sfLedgerIndex, SOE_OPTIONAL) + << SOElement(sfLedgerEntryType, SOE_REQUIRED) + << SOElement(sfFlags, SOE_REQUIRED) + ; } - -LedgerEntryFormat* LedgerEntryFormat::getLgrFormat (const std::string& t) -{ - std::map::iterator it = byName.find ((t)); - - if (it == byName.end ()) - return NULL; - - return it->second; -} - -// vim:ts=4 diff --git a/modules/ripple_data/protocol/ripple_LedgerFormat.h b/modules/ripple_data/protocol/ripple_LedgerFormats.h similarity index 74% rename from modules/ripple_data/protocol/ripple_LedgerFormat.h rename to modules/ripple_data/protocol/ripple_LedgerFormats.h index eb8e0f4a8f..a2a55e9e91 100644 --- a/modules/ripple_data/protocol/ripple_LedgerFormat.h +++ b/modules/ripple_data/protocol/ripple_LedgerFormats.h @@ -4,13 +4,15 @@ */ //============================================================================== -#ifndef RIPPLE_LEDGERFORMAT_H -#define RIPPLE_LEDGERFORMAT_H +#ifndef RIPPLE_LEDGERFORMATS_H_INCLUDED +#define RIPPLE_LEDGERFORMATS_H_INCLUDED -/** +/** Ledger entry types. These are stored in serialized data. + @note Changing these values results in a hard fork. + @ingroup protocol */ // Used as the type of a transaction or the type of a ledger entry. @@ -97,33 +99,23 @@ enum LedgerSpecificFlags lsfHighAuth = 0x00080000, }; -// VFALCO TODO See if we can merge LedgerEntryFormat with TxFormat -// -class LedgerEntryFormat +//------------------------------------------------------------------------------ + +/** Holds the list of known ledger entry formats. +*/ +class LedgerFormats + : public KnownFormats + , public SharedSingleton { +private: + LedgerFormats (); + public: - std::string t_name; - LedgerEntryType t_type; - SOTemplate elements; + static LedgerFormats* createInstance (); - static std::map byType; - static std::map byName; - - LedgerEntryFormat (const char* name, LedgerEntryType type) : t_name (name), t_type (type) - { - byName[name] = this; - byType[type] = this; - } - LedgerEntryFormat& operator<< (const SOElement& el) - { - elements.push_back (el); - return *this; - } - - static LedgerEntryFormat* getLgrFormat (LedgerEntryType t); - static LedgerEntryFormat* getLgrFormat (const std::string& t); - static LedgerEntryFormat* getLgrFormat (int t); +private: + void addCommonFields (Item& item); }; #endif -// vim:ts=4 + diff --git a/modules/ripple_data/protocol/ripple_PackedMessage.cpp b/modules/ripple_data/protocol/ripple_PackedMessage.cpp index 3b60f9a69c..4b2dc38732 100644 --- a/modules/ripple_data/protocol/ripple_PackedMessage.cpp +++ b/modules/ripple_data/protocol/ripple_PackedMessage.cpp @@ -18,8 +18,8 @@ PackedMessage::PackedMessage (::google::protobuf::Message const& message, int ty { message.SerializeToArray (&mBuffer [PackedMessage::kHeaderBytes], messageBytes); -#ifdef DEBUG - // std::cerr << "PackedMessage: type=" << type << ", datalen=" << msg_size << std::endl; +#ifdef BEAST_DEBUG + //Log::out() << "PackedMessage: type=" << type << ", datalen=" << msg_size; #endif } } diff --git a/modules/ripple_data/protocol/ripple_RippleAddress.cpp b/modules/ripple_data/protocol/ripple_RippleAddress.cpp index c0dd4da9d4..05c1437417 100644 --- a/modules/ripple_data/protocol/ripple_RippleAddress.cpp +++ b/modules/ripple_data/protocol/ripple_RippleAddress.cpp @@ -814,17 +814,17 @@ bool RippleAddress::setSeedGeneric (const std::string& strText) } else if (setSeed (strText)) { - // std::cerr << "Recognized seed." << std::endl; + // Log::out() << "Recognized seed."; nothing (); } else if (1 == setSeed1751 (strText)) { - // std::cerr << "Recognized 1751 seed." << std::endl; + // Log::out() << "Recognized 1751 seed."; nothing (); } else { - // std::cerr << "Creating seed from pass phrase." << std::endl; + // Log::out() << "Creating seed from pass phrase."; setSeed (CKey::PassPhraseToKey (strText)); } @@ -865,71 +865,3 @@ RippleAddress RippleAddress::createSeedGeneric (const std::string& strText) return naNew; } - -BOOST_AUTO_TEST_SUITE (ripple_address) - -BOOST_AUTO_TEST_CASE ( check_crypto ) -{ - // Construct a seed. - RippleAddress naSeed; - - BOOST_CHECK (naSeed.setSeedGeneric ("masterpassphrase")); - BOOST_CHECK_MESSAGE (naSeed.humanSeed () == "snoPBrXtMeMyMHUVTgbuqAfg1SUTb", naSeed.humanSeed ()); - - // Create node public/private key pair - RippleAddress naNodePublic = RippleAddress::createNodePublic (naSeed); - RippleAddress naNodePrivate = RippleAddress::createNodePrivate (naSeed); - - BOOST_CHECK_MESSAGE (naNodePublic.humanNodePublic () == "n94a1u4jAz288pZLtw6yFWVbi89YamiC6JBXPVUj5zmExe5fTVg9", naNodePublic.humanNodePublic ()); - BOOST_CHECK_MESSAGE (naNodePrivate.humanNodePrivate () == "pnen77YEeUd4fFKG7iycBWcwKpTaeFRkW2WFostaATy1DSupwXe", naNodePrivate.humanNodePrivate ()); - - // Check node signing. - Blob vucTextSrc = strCopy ("Hello, nurse!"); - uint256 uHash = Serializer::getSHA512Half (vucTextSrc); - Blob vucTextSig; - - naNodePrivate.signNodePrivate (uHash, vucTextSig); - BOOST_CHECK_MESSAGE (naNodePublic.verifyNodePublic (uHash, vucTextSig), "Verify failed."); - - // Construct a public generator from the seed. - RippleAddress naGenerator = RippleAddress::createGeneratorPublic (naSeed); - - BOOST_CHECK_MESSAGE (naGenerator.humanGenerator () == "fhuJKrhSDzV2SkjLn9qbwm5AaRmrxDPfFsHDCP6yfDZWcxDFz4mt", naGenerator.humanGenerator ()); - - // Create account #0 public/private key pair. - RippleAddress naAccountPublic0 = RippleAddress::createAccountPublic (naGenerator, 0); - RippleAddress naAccountPrivate0 = RippleAddress::createAccountPrivate (naGenerator, naSeed, 0); - - BOOST_CHECK_MESSAGE (naAccountPublic0.humanAccountID () == "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", naAccountPublic0.humanAccountID ()); - BOOST_CHECK_MESSAGE (naAccountPublic0.humanAccountPublic () == "aBQG8RQAzjs1eTKFEAQXr2gS4utcDiEC9wmi7pfUPTi27VCahwgw", naAccountPublic0.humanAccountPublic ()); - BOOST_CHECK_MESSAGE (naAccountPrivate0.humanAccountPrivate () == "p9JfM6HHi64m6mvB6v5k7G2b1cXzGmYiCNJf6GHPKvFTWdeRVjh", naAccountPrivate0.humanAccountPrivate ()); - - // Create account #1 public/private key pair. - RippleAddress naAccountPublic1 = RippleAddress::createAccountPublic (naGenerator, 1); - RippleAddress naAccountPrivate1 = RippleAddress::createAccountPrivate (naGenerator, naSeed, 1); - - BOOST_CHECK_MESSAGE (naAccountPublic1.humanAccountID () == "r4bYF7SLUMD7QgSLLpgJx38WJSY12ViRjP", naAccountPublic1.humanAccountID ()); - BOOST_CHECK_MESSAGE (naAccountPublic1.humanAccountPublic () == "aBPXpTfuLy1Bhk3HnGTTAqnovpKWQ23NpFMNkAF6F1Atg5vDyPrw", naAccountPublic1.humanAccountPublic ()); - BOOST_CHECK_MESSAGE (naAccountPrivate1.humanAccountPrivate () == "p9JEm822LMrzJii1k7TvdphfENTp6G5jr253Xa5rkzUWVr8ogQt", naAccountPrivate1.humanAccountPrivate ()); - - // Check account signing. - BOOST_CHECK_MESSAGE (naAccountPrivate0.accountPrivateSign (uHash, vucTextSig), "Signing failed."); - BOOST_CHECK_MESSAGE (naAccountPublic0.accountPublicVerify (uHash, vucTextSig), "Verify failed."); - BOOST_CHECK_MESSAGE (!naAccountPublic1.accountPublicVerify (uHash, vucTextSig), "Anti-verify failed."); - - BOOST_CHECK_MESSAGE (naAccountPrivate1.accountPrivateSign (uHash, vucTextSig), "Signing failed."); - BOOST_CHECK_MESSAGE (naAccountPublic1.accountPublicVerify (uHash, vucTextSig), "Verify failed."); - BOOST_CHECK_MESSAGE (!naAccountPublic0.accountPublicVerify (uHash, vucTextSig), "Anti-verify failed."); - - // Check account encryption. - Blob vucTextCipher - = naAccountPrivate0.accountPrivateEncrypt (naAccountPublic1, vucTextSrc); - Blob vucTextRecovered - = naAccountPrivate1.accountPrivateDecrypt (naAccountPublic0, vucTextCipher); - - BOOST_CHECK_MESSAGE (vucTextSrc == vucTextRecovered, "Encrypt-decrypt failed."); -} - -BOOST_AUTO_TEST_SUITE_END () - -// vim:ts=4 diff --git a/modules/ripple_data/protocol/ripple_RippleAddressUnitTests.cpp b/modules/ripple_data/protocol/ripple_RippleAddressUnitTests.cpp new file mode 100644 index 0000000000..b35acb09d0 --- /dev/null +++ b/modules/ripple_data/protocol/ripple_RippleAddressUnitTests.cpp @@ -0,0 +1,72 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== +BOOST_AUTO_TEST_SUITE (ripple_address) + +BOOST_AUTO_TEST_CASE ( check_crypto ) +{ + using namespace ripple; + + // Construct a seed. + RippleAddress naSeed; + + BOOST_CHECK (naSeed.setSeedGeneric ("masterpassphrase")); + BOOST_CHECK_MESSAGE (naSeed.humanSeed () == "snoPBrXtMeMyMHUVTgbuqAfg1SUTb", naSeed.humanSeed ()); + + // Create node public/private key pair + RippleAddress naNodePublic = RippleAddress::createNodePublic (naSeed); + RippleAddress naNodePrivate = RippleAddress::createNodePrivate (naSeed); + + BOOST_CHECK_MESSAGE (naNodePublic.humanNodePublic () == "n94a1u4jAz288pZLtw6yFWVbi89YamiC6JBXPVUj5zmExe5fTVg9", naNodePublic.humanNodePublic ()); + BOOST_CHECK_MESSAGE (naNodePrivate.humanNodePrivate () == "pnen77YEeUd4fFKG7iycBWcwKpTaeFRkW2WFostaATy1DSupwXe", naNodePrivate.humanNodePrivate ()); + + // Check node signing. + Blob vucTextSrc = strCopy ("Hello, nurse!"); + uint256 uHash = Serializer::getSHA512Half (vucTextSrc); + Blob vucTextSig; + + naNodePrivate.signNodePrivate (uHash, vucTextSig); + BOOST_CHECK_MESSAGE (naNodePublic.verifyNodePublic (uHash, vucTextSig), "Verify failed."); + + // Construct a public generator from the seed. + RippleAddress naGenerator = RippleAddress::createGeneratorPublic (naSeed); + + BOOST_CHECK_MESSAGE (naGenerator.humanGenerator () == "fhuJKrhSDzV2SkjLn9qbwm5AaRmrxDPfFsHDCP6yfDZWcxDFz4mt", naGenerator.humanGenerator ()); + + // Create account #0 public/private key pair. + RippleAddress naAccountPublic0 = RippleAddress::createAccountPublic (naGenerator, 0); + RippleAddress naAccountPrivate0 = RippleAddress::createAccountPrivate (naGenerator, naSeed, 0); + + BOOST_CHECK_MESSAGE (naAccountPublic0.humanAccountID () == "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", naAccountPublic0.humanAccountID ()); + BOOST_CHECK_MESSAGE (naAccountPublic0.humanAccountPublic () == "aBQG8RQAzjs1eTKFEAQXr2gS4utcDiEC9wmi7pfUPTi27VCahwgw", naAccountPublic0.humanAccountPublic ()); + BOOST_CHECK_MESSAGE (naAccountPrivate0.humanAccountPrivate () == "p9JfM6HHi64m6mvB6v5k7G2b1cXzGmYiCNJf6GHPKvFTWdeRVjh", naAccountPrivate0.humanAccountPrivate ()); + + // Create account #1 public/private key pair. + RippleAddress naAccountPublic1 = RippleAddress::createAccountPublic (naGenerator, 1); + RippleAddress naAccountPrivate1 = RippleAddress::createAccountPrivate (naGenerator, naSeed, 1); + + BOOST_CHECK_MESSAGE (naAccountPublic1.humanAccountID () == "r4bYF7SLUMD7QgSLLpgJx38WJSY12ViRjP", naAccountPublic1.humanAccountID ()); + BOOST_CHECK_MESSAGE (naAccountPublic1.humanAccountPublic () == "aBPXpTfuLy1Bhk3HnGTTAqnovpKWQ23NpFMNkAF6F1Atg5vDyPrw", naAccountPublic1.humanAccountPublic ()); + BOOST_CHECK_MESSAGE (naAccountPrivate1.humanAccountPrivate () == "p9JEm822LMrzJii1k7TvdphfENTp6G5jr253Xa5rkzUWVr8ogQt", naAccountPrivate1.humanAccountPrivate ()); + + // Check account signing. + BOOST_CHECK_MESSAGE (naAccountPrivate0.accountPrivateSign (uHash, vucTextSig), "Signing failed."); + BOOST_CHECK_MESSAGE (naAccountPublic0.accountPublicVerify (uHash, vucTextSig), "Verify failed."); + BOOST_CHECK_MESSAGE (!naAccountPublic1.accountPublicVerify (uHash, vucTextSig), "Anti-verify failed."); + + BOOST_CHECK_MESSAGE (naAccountPrivate1.accountPrivateSign (uHash, vucTextSig), "Signing failed."); + BOOST_CHECK_MESSAGE (naAccountPublic1.accountPublicVerify (uHash, vucTextSig), "Verify failed."); + BOOST_CHECK_MESSAGE (!naAccountPublic0.accountPublicVerify (uHash, vucTextSig), "Anti-verify failed."); + + // Check account encryption. + Blob vucTextCipher + = naAccountPrivate0.accountPrivateEncrypt (naAccountPublic1, vucTextSrc); + Blob vucTextRecovered + = naAccountPrivate1.accountPrivateDecrypt (naAccountPublic0, vucTextCipher); + + BOOST_CHECK_MESSAGE (vucTextSrc == vucTextRecovered, "Encrypt-decrypt failed."); +} + +BOOST_AUTO_TEST_SUITE_END () diff --git a/modules/ripple_data/protocol/ripple_STAmount.cpp b/modules/ripple_data/protocol/ripple_STAmount.cpp index 467e7ea398..7a586f949b 100644 --- a/modules/ripple_data/protocol/ripple_STAmount.cpp +++ b/modules/ripple_data/protocol/ripple_STAmount.cpp @@ -50,7 +50,7 @@ bool STAmount::currencyFromString (uint160& uDstCurrency, const std::string& sCu // std::string sIso; // sIso.assign(vucIso.begin(), vucIso.end()); - // std::cerr << "currency: " << sIso << std::endl; + // Log::out() << "currency: " << sIso; Serializer s; @@ -1298,521 +1298,3 @@ Json::Value STAmount::getJson (int) const setJson (elem); return elem; } - -// For unit tests: -static STAmount serdes (const STAmount& s) -{ - Serializer ser; - - s.add (ser); - - SerializerIterator sit (ser); - - return STAmount::deserialize (sit); -} - -BOOST_AUTO_TEST_SUITE (amount) - -BOOST_AUTO_TEST_CASE ( setValue_test ) -{ - STAmount saTmp; - -#if 0 - // Check native floats - saTmp.setFullValue ("1^0"); - BOOST_CHECK_MESSAGE (SYSTEM_CURRENCY_PARTS == saTmp.getNValue (), "float integer failed"); - saTmp.setFullValue ("0^1"); - BOOST_CHECK_MESSAGE (SYSTEM_CURRENCY_PARTS / 10 == saTmp.getNValue (), "float fraction failed"); - saTmp.setFullValue ("0^12"); - BOOST_CHECK_MESSAGE (12 * SYSTEM_CURRENCY_PARTS / 100 == saTmp.getNValue (), "float fraction failed"); - saTmp.setFullValue ("1^2"); - BOOST_CHECK_MESSAGE (SYSTEM_CURRENCY_PARTS + (2 * SYSTEM_CURRENCY_PARTS / 10) == saTmp.getNValue (), "float combined failed"); -#endif - - // Check native integer - saTmp.setFullValue ("1"); - BOOST_CHECK_MESSAGE (1 == saTmp.getNValue (), "integer failed"); -} - -BOOST_AUTO_TEST_CASE ( NativeCurrency_test ) -{ - STAmount zero, one (1), hundred (100); - - if (serdes (zero) != zero) BOOST_FAIL ("STAmount fail"); - - if (serdes (one) != one) BOOST_FAIL ("STAmount fail"); - - if (serdes (hundred) != hundred) BOOST_FAIL ("STAmount fail"); - - if (!zero.isNative ()) BOOST_FAIL ("STAmount fail"); - - if (!hundred.isNative ()) BOOST_FAIL ("STAmount fail"); - - if (!zero.isZero ()) BOOST_FAIL ("STAmount fail"); - - if (one.isZero ()) BOOST_FAIL ("STAmount fail"); - - if (hundred.isZero ()) BOOST_FAIL ("STAmount fail"); - - if ((zero < zero)) BOOST_FAIL ("STAmount fail"); - - if (! (zero < one)) BOOST_FAIL ("STAmount fail"); - - if (! (zero < hundred)) BOOST_FAIL ("STAmount fail"); - - if ((one < zero)) BOOST_FAIL ("STAmount fail"); - - if ((one < one)) BOOST_FAIL ("STAmount fail"); - - if (! (one < hundred)) BOOST_FAIL ("STAmount fail"); - - if ((hundred < zero)) BOOST_FAIL ("STAmount fail"); - - if ((hundred < one)) BOOST_FAIL ("STAmount fail"); - - if ((hundred < hundred)) BOOST_FAIL ("STAmount fail"); - - if ((zero > zero)) BOOST_FAIL ("STAmount fail"); - - if ((zero > one)) BOOST_FAIL ("STAmount fail"); - - if ((zero > hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (one > zero)) BOOST_FAIL ("STAmount fail"); - - if ((one > one)) BOOST_FAIL ("STAmount fail"); - - if ((one > hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred > zero)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred > one)) BOOST_FAIL ("STAmount fail"); - - if ((hundred > hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (zero <= zero)) BOOST_FAIL ("STAmount fail"); - - if (! (zero <= one)) BOOST_FAIL ("STAmount fail"); - - if (! (zero <= hundred)) BOOST_FAIL ("STAmount fail"); - - if ((one <= zero)) BOOST_FAIL ("STAmount fail"); - - if (! (one <= one)) BOOST_FAIL ("STAmount fail"); - - if (! (one <= hundred)) BOOST_FAIL ("STAmount fail"); - - if ((hundred <= zero)) BOOST_FAIL ("STAmount fail"); - - if ((hundred <= one)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred <= hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (zero >= zero)) BOOST_FAIL ("STAmount fail"); - - if ((zero >= one)) BOOST_FAIL ("STAmount fail"); - - if ((zero >= hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (one >= zero)) BOOST_FAIL ("STAmount fail"); - - if (! (one >= one)) BOOST_FAIL ("STAmount fail"); - - if ((one >= hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred >= zero)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred >= one)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred >= hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (zero == zero)) BOOST_FAIL ("STAmount fail"); - - if ((zero == one)) BOOST_FAIL ("STAmount fail"); - - if ((zero == hundred)) BOOST_FAIL ("STAmount fail"); - - if ((one == zero)) BOOST_FAIL ("STAmount fail"); - - if (! (one == one)) BOOST_FAIL ("STAmount fail"); - - if ((one == hundred)) BOOST_FAIL ("STAmount fail"); - - if ((hundred == zero)) BOOST_FAIL ("STAmount fail"); - - if ((hundred == one)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred == hundred)) BOOST_FAIL ("STAmount fail"); - - if ((zero != zero)) BOOST_FAIL ("STAmount fail"); - - if (! (zero != one)) BOOST_FAIL ("STAmount fail"); - - if (! (zero != hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (one != zero)) BOOST_FAIL ("STAmount fail"); - - if ((one != one)) BOOST_FAIL ("STAmount fail"); - - if (! (one != hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred != zero)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred != one)) BOOST_FAIL ("STAmount fail"); - - if ((hundred != hundred)) BOOST_FAIL ("STAmount fail"); - - if (STAmount ().getText () != "0") BOOST_FAIL ("STAmount fail"); - - if (STAmount (31).getText () != "31") BOOST_FAIL ("STAmount fail"); - - if (STAmount (310).getText () != "310") BOOST_FAIL ("STAmount fail"); - - BOOST_TEST_MESSAGE ("Amount NC Complete"); -} - -BOOST_AUTO_TEST_CASE ( CustomCurrency_test ) -{ - STAmount zero (CURRENCY_ONE, ACCOUNT_ONE), one (CURRENCY_ONE, ACCOUNT_ONE, 1), hundred (CURRENCY_ONE, ACCOUNT_ONE, 100); - - serdes (one).getRaw (); - - if (serdes (zero) != zero) BOOST_FAIL ("STAmount fail"); - - if (serdes (one) != one) BOOST_FAIL ("STAmount fail"); - - if (serdes (hundred) != hundred) BOOST_FAIL ("STAmount fail"); - - if (zero.isNative ()) BOOST_FAIL ("STAmount fail"); - - if (hundred.isNative ()) BOOST_FAIL ("STAmount fail"); - - if (!zero.isZero ()) BOOST_FAIL ("STAmount fail"); - - if (one.isZero ()) BOOST_FAIL ("STAmount fail"); - - if (hundred.isZero ()) BOOST_FAIL ("STAmount fail"); - - if ((zero < zero)) BOOST_FAIL ("STAmount fail"); - - if (! (zero < one)) BOOST_FAIL ("STAmount fail"); - - if (! (zero < hundred)) BOOST_FAIL ("STAmount fail"); - - if ((one < zero)) BOOST_FAIL ("STAmount fail"); - - if ((one < one)) BOOST_FAIL ("STAmount fail"); - - if (! (one < hundred)) BOOST_FAIL ("STAmount fail"); - - if ((hundred < zero)) BOOST_FAIL ("STAmount fail"); - - if ((hundred < one)) BOOST_FAIL ("STAmount fail"); - - if ((hundred < hundred)) BOOST_FAIL ("STAmount fail"); - - if ((zero > zero)) BOOST_FAIL ("STAmount fail"); - - if ((zero > one)) BOOST_FAIL ("STAmount fail"); - - if ((zero > hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (one > zero)) BOOST_FAIL ("STAmount fail"); - - if ((one > one)) BOOST_FAIL ("STAmount fail"); - - if ((one > hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred > zero)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred > one)) BOOST_FAIL ("STAmount fail"); - - if ((hundred > hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (zero <= zero)) BOOST_FAIL ("STAmount fail"); - - if (! (zero <= one)) BOOST_FAIL ("STAmount fail"); - - if (! (zero <= hundred)) BOOST_FAIL ("STAmount fail"); - - if ((one <= zero)) BOOST_FAIL ("STAmount fail"); - - if (! (one <= one)) BOOST_FAIL ("STAmount fail"); - - if (! (one <= hundred)) BOOST_FAIL ("STAmount fail"); - - if ((hundred <= zero)) BOOST_FAIL ("STAmount fail"); - - if ((hundred <= one)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred <= hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (zero >= zero)) BOOST_FAIL ("STAmount fail"); - - if ((zero >= one)) BOOST_FAIL ("STAmount fail"); - - if ((zero >= hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (one >= zero)) BOOST_FAIL ("STAmount fail"); - - if (! (one >= one)) BOOST_FAIL ("STAmount fail"); - - if ((one >= hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred >= zero)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred >= one)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred >= hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (zero == zero)) BOOST_FAIL ("STAmount fail"); - - if ((zero == one)) BOOST_FAIL ("STAmount fail"); - - if ((zero == hundred)) BOOST_FAIL ("STAmount fail"); - - if ((one == zero)) BOOST_FAIL ("STAmount fail"); - - if (! (one == one)) BOOST_FAIL ("STAmount fail"); - - if ((one == hundred)) BOOST_FAIL ("STAmount fail"); - - if ((hundred == zero)) BOOST_FAIL ("STAmount fail"); - - if ((hundred == one)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred == hundred)) BOOST_FAIL ("STAmount fail"); - - if ((zero != zero)) BOOST_FAIL ("STAmount fail"); - - if (! (zero != one)) BOOST_FAIL ("STAmount fail"); - - if (! (zero != hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (one != zero)) BOOST_FAIL ("STAmount fail"); - - if ((one != one)) BOOST_FAIL ("STAmount fail"); - - if (! (one != hundred)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred != zero)) BOOST_FAIL ("STAmount fail"); - - if (! (hundred != one)) BOOST_FAIL ("STAmount fail"); - - if ((hundred != hundred)) BOOST_FAIL ("STAmount fail"); - - if (STAmount (CURRENCY_ONE, ACCOUNT_ONE).getText () != "0") BOOST_FAIL ("STAmount fail"); - - if (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 31).getText () != "31") BOOST_FAIL ("STAmount fail"); - - if (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 31, 1).getText () != "310") BOOST_FAIL ("STAmount fail"); - - if (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 31, -1).getText () != "3.1") BOOST_FAIL ("STAmount fail"); - - if (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 31, -2).getText () != "0.31") BOOST_FAIL ("STAmount fail"); - - if (STAmount::multiply (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 20), STAmount (3), CURRENCY_ONE, ACCOUNT_ONE).getText () != "60") - BOOST_FAIL ("STAmount multiply fail 1"); - - if (STAmount::multiply (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 20), STAmount (3), uint160 (), ACCOUNT_XRP).getText () != "60") - BOOST_FAIL ("STAmount multiply fail 2"); - - if (STAmount::multiply (STAmount (20), STAmount (3), CURRENCY_ONE, ACCOUNT_ONE).getText () != "60") - BOOST_FAIL ("STAmount multiply fail 3"); - - if (STAmount::multiply (STAmount (20), STAmount (3), uint160 (), ACCOUNT_XRP).getText () != "60") - BOOST_FAIL ("STAmount multiply fail 4"); - - if (STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), STAmount (3), CURRENCY_ONE, ACCOUNT_ONE).getText () != "20") - { - WriteLog (lsFATAL, STAmount) << "60/3 = " << - STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), - STAmount (3), CURRENCY_ONE, ACCOUNT_ONE).getText (); - BOOST_FAIL ("STAmount divide fail"); - } - - if (STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), STAmount (3), uint160 (), ACCOUNT_XRP).getText () != "20") - BOOST_FAIL ("STAmount divide fail"); - - if (STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 3), CURRENCY_ONE, ACCOUNT_ONE).getText () != "20") - BOOST_FAIL ("STAmount divide fail"); - - if (STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 3), uint160 (), ACCOUNT_XRP).getText () != "20") - BOOST_FAIL ("STAmount divide fail"); - - STAmount a1 (CURRENCY_ONE, ACCOUNT_ONE, 60), a2 (CURRENCY_ONE, ACCOUNT_ONE, 10, -1); - - if (STAmount::divide (a2, a1, CURRENCY_ONE, ACCOUNT_ONE) != STAmount::setRate (STAmount::getRate (a1, a2))) - BOOST_FAIL ("STAmount setRate(getRate) fail"); - - if (STAmount::divide (a1, a2, CURRENCY_ONE, ACCOUNT_ONE) != STAmount::setRate (STAmount::getRate (a2, a1))) - BOOST_FAIL ("STAmount setRate(getRate) fail"); - - BOOST_TEST_MESSAGE ("Amount CC Complete"); -} - -static bool roundTest (int n, int d, int m) -{ - // check STAmount rounding - STAmount num (CURRENCY_ONE, ACCOUNT_ONE, n); - STAmount den (CURRENCY_ONE, ACCOUNT_ONE, d); - STAmount mul (CURRENCY_ONE, ACCOUNT_ONE, m); - STAmount quot = STAmount::divide (n, d, CURRENCY_ONE, ACCOUNT_ONE); - STAmount res = STAmount::multiply (quot, mul, CURRENCY_ONE, ACCOUNT_ONE); - - if (res.isNative ()) - BOOST_FAIL ("Product is native"); - - res.roundSelf (); - - STAmount cmp (CURRENCY_ONE, ACCOUNT_ONE, (n * m) / d); - - if (cmp.isNative ()) - BOOST_FAIL ("Comparison amount is native"); - - if (res == cmp) - return true; - - cmp.throwComparable (res); - WriteLog (lsWARNING, STAmount) << "(" << num.getText () << "/" << den.getText () << ") X " << mul.getText () << " = " - << res.getText () << " not " << cmp.getText (); - BOOST_FAIL ("Round fail"); - return false; -} - -static void mulTest (int a, int b) -{ - STAmount aa (CURRENCY_ONE, ACCOUNT_ONE, a); - STAmount bb (CURRENCY_ONE, ACCOUNT_ONE, b); - STAmount prod1 (STAmount::multiply (aa, bb, CURRENCY_ONE, ACCOUNT_ONE)); - - if (prod1.isNative ()) - BOOST_FAIL ("product is native"); - - STAmount prod2 (CURRENCY_ONE, ACCOUNT_ONE, static_cast (a) * static_cast (b)); - - if (prod1 != prod2) - { - WriteLog (lsWARNING, STAmount) << "nn(" << aa.getFullText () << " * " << bb.getFullText () << ") = " << prod1.getFullText () - << " not " << prod2.getFullText (); - BOOST_WARN ("Multiplication result is not exact"); - } - - aa = a; - prod1 = STAmount::multiply (aa, bb, CURRENCY_ONE, ACCOUNT_ONE); - - if (prod1 != prod2) - { - WriteLog (lsWARNING, STAmount) << "n(" << aa.getFullText () << " * " << bb.getFullText () << ") = " << prod1.getFullText () - << " not " << prod2.getFullText (); - BOOST_WARN ("Multiplication result is not exact"); - } - -} - -BOOST_AUTO_TEST_CASE ( CurrencyMulDivTests ) -{ - CBigNum b; - - for (int i = 0; i < 16; ++i) - { - uint64 r = rand (); - r <<= 32; - r |= rand (); - b.setuint64 (r); - - if (b.getuint64 () != r) - { - WriteLog (lsFATAL, STAmount) << r << " != " << b.getuint64 () << " " << b.ToString (16); - BOOST_FAIL ("setull64/getull64 failure"); - } - } - - // Test currency multiplication and division operations such as - // convertToDisplayAmount, convertToInternalAmount, getRate, getClaimed, and getNeeded - - if (STAmount::getRate (STAmount (1), STAmount (10)) != (((100ull - 14) << (64 - 8)) | 1000000000000000ull)) - BOOST_FAIL ("STAmount getRate fail 1"); - - if (STAmount::getRate (STAmount (10), STAmount (1)) != (((100ull - 16) << (64 - 8)) | 1000000000000000ull)) - BOOST_FAIL ("STAmount getRate fail 2"); - - if (STAmount::getRate (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 1), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 10)) != (((100ull - 14) << (64 - 8)) | 1000000000000000ull)) - BOOST_FAIL ("STAmount getRate fail 3"); - - if (STAmount::getRate (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 10), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 1)) != (((100ull - 16) << (64 - 8)) | 1000000000000000ull)) - BOOST_FAIL ("STAmount getRate fail 4"); - - if (STAmount::getRate (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 1), STAmount (10)) != (((100ull - 14) << (64 - 8)) | 1000000000000000ull)) - BOOST_FAIL ("STAmount getRate fail 5"); - - if (STAmount::getRate (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 10), STAmount (1)) != (((100ull - 16) << (64 - 8)) | 1000000000000000ull)) - BOOST_FAIL ("STAmount getRate fail 6"); - - if (STAmount::getRate (STAmount (1), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 10)) != (((100ull - 14) << (64 - 8)) | 1000000000000000ull)) - BOOST_FAIL ("STAmount getRate fail 7"); - - if (STAmount::getRate (STAmount (10), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 1)) != (((100ull - 16) << (64 - 8)) | 1000000000000000ull)) - BOOST_FAIL ("STAmount getRate fail 8"); - - roundTest (1, 3, 3); - roundTest (2, 3, 9); - roundTest (1, 7, 21); - roundTest (1, 2, 4); - roundTest (3, 9, 18); - roundTest (7, 11, 44); - - for (int i = 0; i <= 100000; ++i) - mulTest (rand () % 10000000, rand () % 10000000); -} - -BOOST_AUTO_TEST_CASE ( UnderFlowTests ) -{ - STAmount bigNative (STAmount::cMaxNative / 2); - STAmount bigValue (CURRENCY_ONE, ACCOUNT_ONE, - (STAmount::cMinValue + STAmount::cMaxValue) / 2, STAmount::cMaxOffset - 1); - STAmount smallValue (CURRENCY_ONE, ACCOUNT_ONE, - (STAmount::cMinValue + STAmount::cMaxValue) / 2, STAmount::cMinOffset + 1); - STAmount zero (CURRENCY_ONE, ACCOUNT_ONE, 0); - - STAmount smallXsmall = STAmount::multiply (smallValue, smallValue, CURRENCY_ONE, ACCOUNT_ONE); - - if (!smallXsmall.isZero ()) - BOOST_FAIL ("STAmount: smallXsmall != 0"); - - STAmount bigDsmall = STAmount::divide (smallValue, bigValue, CURRENCY_ONE, ACCOUNT_ONE); - - if (!bigDsmall.isZero ()) - BOOST_FAIL ("STAmount: small/big != 0: " << bigDsmall); - - bigDsmall = STAmount::divide (smallValue, bigNative, CURRENCY_ONE, uint160 ()); - - if (!bigDsmall.isZero ()) - BOOST_FAIL ("STAmount: small/bigNative != 0: " << bigDsmall); - - bigDsmall = STAmount::divide (smallValue, bigValue, uint160 (), uint160 ()); - - if (!bigDsmall.isZero ()) - BOOST_FAIL ("STAmount: (small/big)->N != 0: " << bigDsmall); - - bigDsmall = STAmount::divide (smallValue, bigNative, uint160 (), uint160 ()); - - if (!bigDsmall.isZero ()) - BOOST_FAIL ("STAmount: (small/bigNative)->N != 0: " << bigDsmall); - - // very bad offer - uint64 r = STAmount::getRate (smallValue, bigValue); - - if (r != 0) - BOOST_FAIL ("STAmount: getRate(smallOut/bigIn) != 0" << r); - - // very good offer - r = STAmount::getRate (bigValue, smallValue); - - if (r != 0) - BOOST_FAIL ("STAmount:: getRate(smallIn/bigOUt) != 0" << r); -} - -BOOST_AUTO_TEST_SUITE_END () - -// vim:ts=4 diff --git a/modules/ripple_data/protocol/ripple_STAmountRound.cpp b/modules/ripple_data/protocol/ripple_STAmountRound.cpp index 3d90ad699e..b865e16dc3 100644 --- a/modules/ripple_data/protocol/ripple_STAmountRound.cpp +++ b/modules/ripple_data/protocol/ripple_STAmountRound.cpp @@ -4,7 +4,7 @@ */ //============================================================================== -static void canonicalizeRound (bool isNative, uint64& value, int& offset, bool roundUp) +void STAmount::canonicalizeRound (bool isNative, uint64& value, int& offset, bool roundUp) { if (!roundUp) // canonicalize already rounds down return; @@ -298,54 +298,3 @@ STAmount STAmount::divRound (const STAmount& num, const STAmount& den, return STAmount (uCurrencyID, uIssuerID, amount, offset, resultNegative); } -BOOST_AUTO_TEST_SUITE (amountRound) - -BOOST_AUTO_TEST_CASE ( amountRound_test ) -{ - uint64 value = 25000000000000000ull; - int offset = -14; - canonicalizeRound (false, value, offset, true); - - STAmount one (CURRENCY_ONE, ACCOUNT_ONE, 1); - STAmount two (CURRENCY_ONE, ACCOUNT_ONE, 2); - STAmount three (CURRENCY_ONE, ACCOUNT_ONE, 3); - - STAmount oneThird1 = STAmount::divRound (one, three, CURRENCY_ONE, ACCOUNT_ONE, false); - STAmount oneThird2 = STAmount::divide (one, three, CURRENCY_ONE, ACCOUNT_ONE); - STAmount oneThird3 = STAmount::divRound (one, three, CURRENCY_ONE, ACCOUNT_ONE, true); - WriteLog (lsINFO, STAmount) << oneThird1; - WriteLog (lsINFO, STAmount) << oneThird2; - WriteLog (lsINFO, STAmount) << oneThird3; - - STAmount twoThird1 = STAmount::divRound (two, three, CURRENCY_ONE, ACCOUNT_ONE, false); - STAmount twoThird2 = STAmount::divide (two, three, CURRENCY_ONE, ACCOUNT_ONE); - STAmount twoThird3 = STAmount::divRound (two, three, CURRENCY_ONE, ACCOUNT_ONE, true); - WriteLog (lsINFO, STAmount) << twoThird1; - WriteLog (lsINFO, STAmount) << twoThird2; - WriteLog (lsINFO, STAmount) << twoThird3; - - STAmount oneA = STAmount::mulRound (oneThird1, three, CURRENCY_ONE, ACCOUNT_ONE, false); - STAmount oneB = STAmount::multiply (oneThird2, three, CURRENCY_ONE, ACCOUNT_ONE); - STAmount oneC = STAmount::mulRound (oneThird3, three, CURRENCY_ONE, ACCOUNT_ONE, true); - WriteLog (lsINFO, STAmount) << oneA; - WriteLog (lsINFO, STAmount) << oneB; - WriteLog (lsINFO, STAmount) << oneC; - - STAmount fourThirdsA = STAmount::addRound (twoThird2, twoThird2, false); - STAmount fourThirdsB = twoThird2 + twoThird2; - STAmount fourThirdsC = STAmount::addRound (twoThird2, twoThird2, true); - WriteLog (lsINFO, STAmount) << fourThirdsA; - WriteLog (lsINFO, STAmount) << fourThirdsB; - WriteLog (lsINFO, STAmount) << fourThirdsC; - - STAmount dripTest1 = STAmount::mulRound (twoThird2, two, uint160 (), uint160 (), false); - STAmount dripTest2 = STAmount::multiply (twoThird2, two, uint160 (), uint160 ()); - STAmount dripTest3 = STAmount::mulRound (twoThird2, two, uint160 (), uint160 (), true); - WriteLog (lsINFO, STAmount) << dripTest1; - WriteLog (lsINFO, STAmount) << dripTest2; - WriteLog (lsINFO, STAmount) << dripTest3; -} - -BOOST_AUTO_TEST_SUITE_END () - -// vim:ts=4 diff --git a/modules/ripple_data/protocol/ripple_STAmountUnitTests.cpp b/modules/ripple_data/protocol/ripple_STAmountUnitTests.cpp new file mode 100644 index 0000000000..fdc0960ef2 --- /dev/null +++ b/modules/ripple_data/protocol/ripple_STAmountUnitTests.cpp @@ -0,0 +1,600 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +// For unit tests: +namespace ripple +{ + +static STAmount serdes (const STAmount& s) +{ + Serializer ser; + + s.add (ser); + + SerializerIterator sit (ser); + + return STAmount::deserialize (sit); +} + +static bool roundTest (int n, int d, int m) +{ + // check STAmount rounding + STAmount num (CURRENCY_ONE, ACCOUNT_ONE, n); + STAmount den (CURRENCY_ONE, ACCOUNT_ONE, d); + STAmount mul (CURRENCY_ONE, ACCOUNT_ONE, m); + STAmount quot = STAmount::divide (n, d, CURRENCY_ONE, ACCOUNT_ONE); + STAmount res = STAmount::multiply (quot, mul, CURRENCY_ONE, ACCOUNT_ONE); + + if (res.isNative ()) + BOOST_FAIL ("Product is native"); + + res.roundSelf (); + + STAmount cmp (CURRENCY_ONE, ACCOUNT_ONE, (n * m) / d); + + if (cmp.isNative ()) + BOOST_FAIL ("Comparison amount is native"); + + if (res == cmp) + return true; + + cmp.throwComparable (res); + WriteLog (lsWARNING, STAmount) << "(" << num.getText () << "/" << den.getText () << ") X " << mul.getText () << " = " + << res.getText () << " not " << cmp.getText (); + BOOST_FAIL ("Round fail"); + return false; +} + +static void mulTest (int a, int b) +{ + STAmount aa (CURRENCY_ONE, ACCOUNT_ONE, a); + STAmount bb (CURRENCY_ONE, ACCOUNT_ONE, b); + STAmount prod1 (STAmount::multiply (aa, bb, CURRENCY_ONE, ACCOUNT_ONE)); + + if (prod1.isNative ()) + BOOST_FAIL ("product is native"); + + STAmount prod2 (CURRENCY_ONE, ACCOUNT_ONE, static_cast (a) * static_cast (b)); + + if (prod1 != prod2) + { + WriteLog (lsWARNING, STAmount) << "nn(" << aa.getFullText () << " * " << bb.getFullText () << ") = " << prod1.getFullText () + << " not " << prod2.getFullText (); + BOOST_WARN ("Multiplication result is not exact"); + } + + aa = a; + prod1 = STAmount::multiply (aa, bb, CURRENCY_ONE, ACCOUNT_ONE); + + if (prod1 != prod2) + { + WriteLog (lsWARNING, STAmount) << "n(" << aa.getFullText () << " * " << bb.getFullText () << ") = " << prod1.getFullText () + << " not " << prod2.getFullText (); + BOOST_WARN ("Multiplication result is not exact"); + } + +} + +} + +//------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_SUITE (amount) + +BOOST_AUTO_TEST_CASE ( setValue_test ) +{ + using namespace ripple; + + STAmount saTmp; + +#if 0 + // Check native floats + saTmp.setFullValue ("1^0"); + BOOST_CHECK_MESSAGE (SYSTEM_CURRENCY_PARTS == saTmp.getNValue (), "float integer failed"); + saTmp.setFullValue ("0^1"); + BOOST_CHECK_MESSAGE (SYSTEM_CURRENCY_PARTS / 10 == saTmp.getNValue (), "float fraction failed"); + saTmp.setFullValue ("0^12"); + BOOST_CHECK_MESSAGE (12 * SYSTEM_CURRENCY_PARTS / 100 == saTmp.getNValue (), "float fraction failed"); + saTmp.setFullValue ("1^2"); + BOOST_CHECK_MESSAGE (SYSTEM_CURRENCY_PARTS + (2 * SYSTEM_CURRENCY_PARTS / 10) == saTmp.getNValue (), "float combined failed"); +#endif + + // Check native integer + saTmp.setFullValue ("1"); + BOOST_CHECK_MESSAGE (1 == saTmp.getNValue (), "integer failed"); +} + +//------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_CASE ( NativeCurrency_test ) +{ + using namespace ripple; + + STAmount zero, one (1), hundred (100); + + if (serdes (zero) != zero) BOOST_FAIL ("STAmount fail"); + + if (serdes (one) != one) BOOST_FAIL ("STAmount fail"); + + if (serdes (hundred) != hundred) BOOST_FAIL ("STAmount fail"); + + if (!zero.isNative ()) BOOST_FAIL ("STAmount fail"); + + if (!hundred.isNative ()) BOOST_FAIL ("STAmount fail"); + + if (!zero.isZero ()) BOOST_FAIL ("STAmount fail"); + + if (one.isZero ()) BOOST_FAIL ("STAmount fail"); + + if (hundred.isZero ()) BOOST_FAIL ("STAmount fail"); + + if ((zero < zero)) BOOST_FAIL ("STAmount fail"); + + if (! (zero < one)) BOOST_FAIL ("STAmount fail"); + + if (! (zero < hundred)) BOOST_FAIL ("STAmount fail"); + + if ((one < zero)) BOOST_FAIL ("STAmount fail"); + + if ((one < one)) BOOST_FAIL ("STAmount fail"); + + if (! (one < hundred)) BOOST_FAIL ("STAmount fail"); + + if ((hundred < zero)) BOOST_FAIL ("STAmount fail"); + + if ((hundred < one)) BOOST_FAIL ("STAmount fail"); + + if ((hundred < hundred)) BOOST_FAIL ("STAmount fail"); + + if ((zero > zero)) BOOST_FAIL ("STAmount fail"); + + if ((zero > one)) BOOST_FAIL ("STAmount fail"); + + if ((zero > hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (one > zero)) BOOST_FAIL ("STAmount fail"); + + if ((one > one)) BOOST_FAIL ("STAmount fail"); + + if ((one > hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred > zero)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred > one)) BOOST_FAIL ("STAmount fail"); + + if ((hundred > hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (zero <= zero)) BOOST_FAIL ("STAmount fail"); + + if (! (zero <= one)) BOOST_FAIL ("STAmount fail"); + + if (! (zero <= hundred)) BOOST_FAIL ("STAmount fail"); + + if ((one <= zero)) BOOST_FAIL ("STAmount fail"); + + if (! (one <= one)) BOOST_FAIL ("STAmount fail"); + + if (! (one <= hundred)) BOOST_FAIL ("STAmount fail"); + + if ((hundred <= zero)) BOOST_FAIL ("STAmount fail"); + + if ((hundred <= one)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred <= hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (zero >= zero)) BOOST_FAIL ("STAmount fail"); + + if ((zero >= one)) BOOST_FAIL ("STAmount fail"); + + if ((zero >= hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (one >= zero)) BOOST_FAIL ("STAmount fail"); + + if (! (one >= one)) BOOST_FAIL ("STAmount fail"); + + if ((one >= hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred >= zero)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred >= one)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred >= hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (zero == zero)) BOOST_FAIL ("STAmount fail"); + + if ((zero == one)) BOOST_FAIL ("STAmount fail"); + + if ((zero == hundred)) BOOST_FAIL ("STAmount fail"); + + if ((one == zero)) BOOST_FAIL ("STAmount fail"); + + if (! (one == one)) BOOST_FAIL ("STAmount fail"); + + if ((one == hundred)) BOOST_FAIL ("STAmount fail"); + + if ((hundred == zero)) BOOST_FAIL ("STAmount fail"); + + if ((hundred == one)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred == hundred)) BOOST_FAIL ("STAmount fail"); + + if ((zero != zero)) BOOST_FAIL ("STAmount fail"); + + if (! (zero != one)) BOOST_FAIL ("STAmount fail"); + + if (! (zero != hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (one != zero)) BOOST_FAIL ("STAmount fail"); + + if ((one != one)) BOOST_FAIL ("STAmount fail"); + + if (! (one != hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred != zero)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred != one)) BOOST_FAIL ("STAmount fail"); + + if ((hundred != hundred)) BOOST_FAIL ("STAmount fail"); + + if (STAmount ().getText () != "0") BOOST_FAIL ("STAmount fail"); + + if (STAmount (31).getText () != "31") BOOST_FAIL ("STAmount fail"); + + if (STAmount (310).getText () != "310") BOOST_FAIL ("STAmount fail"); + + BOOST_TEST_MESSAGE ("Amount NC Complete"); +} + +//------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_CASE ( CustomCurrency_test ) +{ + using namespace ripple; + + STAmount zero (CURRENCY_ONE, ACCOUNT_ONE), one (CURRENCY_ONE, ACCOUNT_ONE, 1), hundred (CURRENCY_ONE, ACCOUNT_ONE, 100); + + serdes (one).getRaw (); + + if (serdes (zero) != zero) BOOST_FAIL ("STAmount fail"); + + if (serdes (one) != one) BOOST_FAIL ("STAmount fail"); + + if (serdes (hundred) != hundred) BOOST_FAIL ("STAmount fail"); + + if (zero.isNative ()) BOOST_FAIL ("STAmount fail"); + + if (hundred.isNative ()) BOOST_FAIL ("STAmount fail"); + + if (!zero.isZero ()) BOOST_FAIL ("STAmount fail"); + + if (one.isZero ()) BOOST_FAIL ("STAmount fail"); + + if (hundred.isZero ()) BOOST_FAIL ("STAmount fail"); + + if ((zero < zero)) BOOST_FAIL ("STAmount fail"); + + if (! (zero < one)) BOOST_FAIL ("STAmount fail"); + + if (! (zero < hundred)) BOOST_FAIL ("STAmount fail"); + + if ((one < zero)) BOOST_FAIL ("STAmount fail"); + + if ((one < one)) BOOST_FAIL ("STAmount fail"); + + if (! (one < hundred)) BOOST_FAIL ("STAmount fail"); + + if ((hundred < zero)) BOOST_FAIL ("STAmount fail"); + + if ((hundred < one)) BOOST_FAIL ("STAmount fail"); + + if ((hundred < hundred)) BOOST_FAIL ("STAmount fail"); + + if ((zero > zero)) BOOST_FAIL ("STAmount fail"); + + if ((zero > one)) BOOST_FAIL ("STAmount fail"); + + if ((zero > hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (one > zero)) BOOST_FAIL ("STAmount fail"); + + if ((one > one)) BOOST_FAIL ("STAmount fail"); + + if ((one > hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred > zero)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred > one)) BOOST_FAIL ("STAmount fail"); + + if ((hundred > hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (zero <= zero)) BOOST_FAIL ("STAmount fail"); + + if (! (zero <= one)) BOOST_FAIL ("STAmount fail"); + + if (! (zero <= hundred)) BOOST_FAIL ("STAmount fail"); + + if ((one <= zero)) BOOST_FAIL ("STAmount fail"); + + if (! (one <= one)) BOOST_FAIL ("STAmount fail"); + + if (! (one <= hundred)) BOOST_FAIL ("STAmount fail"); + + if ((hundred <= zero)) BOOST_FAIL ("STAmount fail"); + + if ((hundred <= one)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred <= hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (zero >= zero)) BOOST_FAIL ("STAmount fail"); + + if ((zero >= one)) BOOST_FAIL ("STAmount fail"); + + if ((zero >= hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (one >= zero)) BOOST_FAIL ("STAmount fail"); + + if (! (one >= one)) BOOST_FAIL ("STAmount fail"); + + if ((one >= hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred >= zero)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred >= one)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred >= hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (zero == zero)) BOOST_FAIL ("STAmount fail"); + + if ((zero == one)) BOOST_FAIL ("STAmount fail"); + + if ((zero == hundred)) BOOST_FAIL ("STAmount fail"); + + if ((one == zero)) BOOST_FAIL ("STAmount fail"); + + if (! (one == one)) BOOST_FAIL ("STAmount fail"); + + if ((one == hundred)) BOOST_FAIL ("STAmount fail"); + + if ((hundred == zero)) BOOST_FAIL ("STAmount fail"); + + if ((hundred == one)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred == hundred)) BOOST_FAIL ("STAmount fail"); + + if ((zero != zero)) BOOST_FAIL ("STAmount fail"); + + if (! (zero != one)) BOOST_FAIL ("STAmount fail"); + + if (! (zero != hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (one != zero)) BOOST_FAIL ("STAmount fail"); + + if ((one != one)) BOOST_FAIL ("STAmount fail"); + + if (! (one != hundred)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred != zero)) BOOST_FAIL ("STAmount fail"); + + if (! (hundred != one)) BOOST_FAIL ("STAmount fail"); + + if ((hundred != hundred)) BOOST_FAIL ("STAmount fail"); + + if (STAmount (CURRENCY_ONE, ACCOUNT_ONE).getText () != "0") BOOST_FAIL ("STAmount fail"); + + if (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 31).getText () != "31") BOOST_FAIL ("STAmount fail"); + + if (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 31, 1).getText () != "310") BOOST_FAIL ("STAmount fail"); + + if (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 31, -1).getText () != "3.1") BOOST_FAIL ("STAmount fail"); + + if (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 31, -2).getText () != "0.31") BOOST_FAIL ("STAmount fail"); + + if (STAmount::multiply (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 20), STAmount (3), CURRENCY_ONE, ACCOUNT_ONE).getText () != "60") + BOOST_FAIL ("STAmount multiply fail 1"); + + if (STAmount::multiply (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 20), STAmount (3), uint160 (), ACCOUNT_XRP).getText () != "60") + BOOST_FAIL ("STAmount multiply fail 2"); + + if (STAmount::multiply (STAmount (20), STAmount (3), CURRENCY_ONE, ACCOUNT_ONE).getText () != "60") + BOOST_FAIL ("STAmount multiply fail 3"); + + if (STAmount::multiply (STAmount (20), STAmount (3), uint160 (), ACCOUNT_XRP).getText () != "60") + BOOST_FAIL ("STAmount multiply fail 4"); + + if (STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), STAmount (3), CURRENCY_ONE, ACCOUNT_ONE).getText () != "20") + { + WriteLog (lsFATAL, STAmount) << "60/3 = " << + STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), + STAmount (3), CURRENCY_ONE, ACCOUNT_ONE).getText (); + BOOST_FAIL ("STAmount divide fail"); + } + + if (STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), STAmount (3), uint160 (), ACCOUNT_XRP).getText () != "20") + BOOST_FAIL ("STAmount divide fail"); + + if (STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 3), CURRENCY_ONE, ACCOUNT_ONE).getText () != "20") + BOOST_FAIL ("STAmount divide fail"); + + if (STAmount::divide (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 60), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 3), uint160 (), ACCOUNT_XRP).getText () != "20") + BOOST_FAIL ("STAmount divide fail"); + + STAmount a1 (CURRENCY_ONE, ACCOUNT_ONE, 60), a2 (CURRENCY_ONE, ACCOUNT_ONE, 10, -1); + + if (STAmount::divide (a2, a1, CURRENCY_ONE, ACCOUNT_ONE) != STAmount::setRate (STAmount::getRate (a1, a2))) + BOOST_FAIL ("STAmount setRate(getRate) fail"); + + if (STAmount::divide (a1, a2, CURRENCY_ONE, ACCOUNT_ONE) != STAmount::setRate (STAmount::getRate (a2, a1))) + BOOST_FAIL ("STAmount setRate(getRate) fail"); + + BOOST_TEST_MESSAGE ("Amount CC Complete"); +} + +//------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_CASE ( CurrencyMulDivTests ) +{ + using namespace ripple; + + CBigNum b; + + for (int i = 0; i < 16; ++i) + { + uint64 r = rand (); + r <<= 32; + r |= rand (); + b.setuint64 (r); + + if (b.getuint64 () != r) + { + WriteLog (lsFATAL, STAmount) << r << " != " << b.getuint64 () << " " << b.ToString (16); + BOOST_FAIL ("setull64/getull64 failure"); + } + } + + // Test currency multiplication and division operations such as + // convertToDisplayAmount, convertToInternalAmount, getRate, getClaimed, and getNeeded + + if (STAmount::getRate (STAmount (1), STAmount (10)) != (((100ull - 14) << (64 - 8)) | 1000000000000000ull)) + BOOST_FAIL ("STAmount getRate fail 1"); + + if (STAmount::getRate (STAmount (10), STAmount (1)) != (((100ull - 16) << (64 - 8)) | 1000000000000000ull)) + BOOST_FAIL ("STAmount getRate fail 2"); + + if (STAmount::getRate (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 1), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 10)) != (((100ull - 14) << (64 - 8)) | 1000000000000000ull)) + BOOST_FAIL ("STAmount getRate fail 3"); + + if (STAmount::getRate (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 10), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 1)) != (((100ull - 16) << (64 - 8)) | 1000000000000000ull)) + BOOST_FAIL ("STAmount getRate fail 4"); + + if (STAmount::getRate (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 1), STAmount (10)) != (((100ull - 14) << (64 - 8)) | 1000000000000000ull)) + BOOST_FAIL ("STAmount getRate fail 5"); + + if (STAmount::getRate (STAmount (CURRENCY_ONE, ACCOUNT_ONE, 10), STAmount (1)) != (((100ull - 16) << (64 - 8)) | 1000000000000000ull)) + BOOST_FAIL ("STAmount getRate fail 6"); + + if (STAmount::getRate (STAmount (1), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 10)) != (((100ull - 14) << (64 - 8)) | 1000000000000000ull)) + BOOST_FAIL ("STAmount getRate fail 7"); + + if (STAmount::getRate (STAmount (10), STAmount (CURRENCY_ONE, ACCOUNT_ONE, 1)) != (((100ull - 16) << (64 - 8)) | 1000000000000000ull)) + BOOST_FAIL ("STAmount getRate fail 8"); + + roundTest (1, 3, 3); + roundTest (2, 3, 9); + roundTest (1, 7, 21); + roundTest (1, 2, 4); + roundTest (3, 9, 18); + roundTest (7, 11, 44); + + for (int i = 0; i <= 100000; ++i) + mulTest (rand () % 10000000, rand () % 10000000); +} + +//------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_CASE ( UnderFlowTests ) +{ + using namespace ripple; + + STAmount bigNative (STAmount::cMaxNative / 2); + STAmount bigValue (CURRENCY_ONE, ACCOUNT_ONE, + (STAmount::cMinValue + STAmount::cMaxValue) / 2, STAmount::cMaxOffset - 1); + STAmount smallValue (CURRENCY_ONE, ACCOUNT_ONE, + (STAmount::cMinValue + STAmount::cMaxValue) / 2, STAmount::cMinOffset + 1); + STAmount zero (CURRENCY_ONE, ACCOUNT_ONE, 0); + + STAmount smallXsmall = STAmount::multiply (smallValue, smallValue, CURRENCY_ONE, ACCOUNT_ONE); + + if (!smallXsmall.isZero ()) + BOOST_FAIL ("STAmount: smallXsmall != 0"); + + STAmount bigDsmall = STAmount::divide (smallValue, bigValue, CURRENCY_ONE, ACCOUNT_ONE); + + if (!bigDsmall.isZero ()) + BOOST_FAIL ("STAmount: small/big != 0: " << bigDsmall); + + bigDsmall = STAmount::divide (smallValue, bigNative, CURRENCY_ONE, uint160 ()); + + if (!bigDsmall.isZero ()) + BOOST_FAIL ("STAmount: small/bigNative != 0: " << bigDsmall); + + bigDsmall = STAmount::divide (smallValue, bigValue, uint160 (), uint160 ()); + + if (!bigDsmall.isZero ()) + BOOST_FAIL ("STAmount: (small/big)->N != 0: " << bigDsmall); + + bigDsmall = STAmount::divide (smallValue, bigNative, uint160 (), uint160 ()); + + if (!bigDsmall.isZero ()) + BOOST_FAIL ("STAmount: (small/bigNative)->N != 0: " << bigDsmall); + + // very bad offer + uint64 r = STAmount::getRate (smallValue, bigValue); + + if (r != 0) + BOOST_FAIL ("STAmount: getRate(smallOut/bigIn) != 0" << r); + + // very good offer + r = STAmount::getRate (bigValue, smallValue); + + if (r != 0) + BOOST_FAIL ("STAmount:: getRate(smallIn/bigOUt) != 0" << r); +} + +BOOST_AUTO_TEST_SUITE_END () + +//------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_SUITE (amountRound) + +BOOST_AUTO_TEST_CASE ( amountRound_test ) +{ + using namespace ripple; + + uint64 value = 25000000000000000ull; + int offset = -14; + STAmount::canonicalizeRound (false, value, offset, true); + + STAmount one (CURRENCY_ONE, ACCOUNT_ONE, 1); + STAmount two (CURRENCY_ONE, ACCOUNT_ONE, 2); + STAmount three (CURRENCY_ONE, ACCOUNT_ONE, 3); + + STAmount oneThird1 = STAmount::divRound (one, three, CURRENCY_ONE, ACCOUNT_ONE, false); + STAmount oneThird2 = STAmount::divide (one, three, CURRENCY_ONE, ACCOUNT_ONE); + STAmount oneThird3 = STAmount::divRound (one, three, CURRENCY_ONE, ACCOUNT_ONE, true); + WriteLog (lsINFO, STAmount) << oneThird1; + WriteLog (lsINFO, STAmount) << oneThird2; + WriteLog (lsINFO, STAmount) << oneThird3; + + STAmount twoThird1 = STAmount::divRound (two, three, CURRENCY_ONE, ACCOUNT_ONE, false); + STAmount twoThird2 = STAmount::divide (two, three, CURRENCY_ONE, ACCOUNT_ONE); + STAmount twoThird3 = STAmount::divRound (two, three, CURRENCY_ONE, ACCOUNT_ONE, true); + WriteLog (lsINFO, STAmount) << twoThird1; + WriteLog (lsINFO, STAmount) << twoThird2; + WriteLog (lsINFO, STAmount) << twoThird3; + + STAmount oneA = STAmount::mulRound (oneThird1, three, CURRENCY_ONE, ACCOUNT_ONE, false); + STAmount oneB = STAmount::multiply (oneThird2, three, CURRENCY_ONE, ACCOUNT_ONE); + STAmount oneC = STAmount::mulRound (oneThird3, three, CURRENCY_ONE, ACCOUNT_ONE, true); + WriteLog (lsINFO, STAmount) << oneA; + WriteLog (lsINFO, STAmount) << oneB; + WriteLog (lsINFO, STAmount) << oneC; + + STAmount fourThirdsA = STAmount::addRound (twoThird2, twoThird2, false); + STAmount fourThirdsB = twoThird2 + twoThird2; + STAmount fourThirdsC = STAmount::addRound (twoThird2, twoThird2, true); + WriteLog (lsINFO, STAmount) << fourThirdsA; + WriteLog (lsINFO, STAmount) << fourThirdsB; + WriteLog (lsINFO, STAmount) << fourThirdsC; + + STAmount dripTest1 = STAmount::mulRound (twoThird2, two, uint160 (), uint160 (), false); + STAmount dripTest2 = STAmount::multiply (twoThird2, two, uint160 (), uint160 ()); + STAmount dripTest3 = STAmount::mulRound (twoThird2, two, uint160 (), uint160 (), true); + WriteLog (lsINFO, STAmount) << dripTest1; + WriteLog (lsINFO, STAmount) << dripTest2; + WriteLog (lsINFO, STAmount) << dripTest3; +} + +BOOST_AUTO_TEST_SUITE_END () diff --git a/modules/ripple_data/protocol/ripple_SerializedObject.cpp b/modules/ripple_data/protocol/ripple_SerializedObject.cpp index 5a07dd568e..dada73dce5 100644 --- a/modules/ripple_data/protocol/ripple_SerializedObject.cpp +++ b/modules/ripple_data/protocol/ripple_SerializedObject.cpp @@ -1287,24 +1287,19 @@ UPTR_T STObject::parseJson (const Json::Value& object, SField::ref inN { if (field == sfTransactionType) { - TxFormat* f = TxFormats::getInstance ().findByName (strValue); + // Retrieve type from name. Throws if not found. + TxType const txType = TxFormats::getInstance()->findTypeByName (strValue); - if (!f) - throw std::runtime_error ("Unknown transaction type"); - - data.push_back (new STUInt16 (field, static_cast (f->getType ()))); + data.push_back (new STUInt16 (field, static_cast (txType))); if (*name == sfGeneric) name = &sfTransaction; } else if (field == sfLedgerEntryType) { - LedgerEntryFormat* f = LedgerEntryFormat::getLgrFormat (strValue); + LedgerEntryType const type = LedgerFormats::getInstance()->findTypeByName (strValue); - if (!f) - throw std::runtime_error ("Unknown ledger entry type"); - - data.push_back (new STUInt16 (field, static_cast (f->t_type))); + data.push_back (new STUInt16 (field, static_cast (type))); if (*name == sfGeneric) name = &sfLedgerEntry; @@ -1565,84 +1560,3 @@ UPTR_T STObject::parseJson (const Json::Value& object, SField::ref inN return UPTR_T (new STObject (*name, data)); } - -BOOST_AUTO_TEST_SUITE (SerializedObject) - -BOOST_AUTO_TEST_CASE ( FieldManipulation_test ) -{ - if (sfGeneric.isUseful ()) - BOOST_FAIL ("sfGeneric must not be useful"); - - SField sfTestVL (STI_VL, 255, "TestVL"); - SField sfTestH256 (STI_HASH256, 255, "TestH256"); - SField sfTestU32 (STI_UINT32, 255, "TestU32"); - SField sfTestObject (STI_OBJECT, 255, "TestObject"); - - SOTemplate elements; - elements.push_back (SOElement (sfFlags, SOE_REQUIRED)); - elements.push_back (SOElement (sfTestVL, SOE_REQUIRED)); - elements.push_back (SOElement (sfTestH256, SOE_OPTIONAL)); - elements.push_back (SOElement (sfTestU32, SOE_REQUIRED)); - - STObject object1 (elements, sfTestObject); - STObject object2 (object1); - - if (object1.getSerializer () != object2.getSerializer ()) BOOST_FAIL ("STObject error 1"); - - if (object1.isFieldPresent (sfTestH256) || !object1.isFieldPresent (sfTestVL)) - BOOST_FAIL ("STObject error"); - - object1.makeFieldPresent (sfTestH256); - - if (!object1.isFieldPresent (sfTestH256)) BOOST_FAIL ("STObject Error 2"); - - if (object1.getFieldH256 (sfTestH256) != uint256 ()) BOOST_FAIL ("STObject error 3"); - - if (object1.getSerializer () == object2.getSerializer ()) - { - WriteLog (lsINFO, STObject) << "O1: " << object1.getJson (0); - WriteLog (lsINFO, STObject) << "O2: " << object2.getJson (0); - BOOST_FAIL ("STObject error 4"); - } - - object1.makeFieldAbsent (sfTestH256); - - if (object1.isFieldPresent (sfTestH256)) BOOST_FAIL ("STObject error 5"); - - if (object1.getFlags () != 0) BOOST_FAIL ("STObject error 6"); - - if (object1.getSerializer () != object2.getSerializer ()) BOOST_FAIL ("STObject error 7"); - - STObject copy (object1); - - if (object1.isFieldPresent (sfTestH256)) BOOST_FAIL ("STObject error 8"); - - if (copy.isFieldPresent (sfTestH256)) BOOST_FAIL ("STObject error 9"); - - if (object1.getSerializer () != copy.getSerializer ()) BOOST_FAIL ("STObject error 10"); - - copy.setFieldU32 (sfTestU32, 1); - - if (object1.getSerializer () == copy.getSerializer ()) BOOST_FAIL ("STObject error 11"); - - for (int i = 0; i < 1000; i++) - { - Blob j (i, 2); - - object1.setFieldVL (sfTestVL, j); - - Serializer s; - object1.add (s); - SerializerIterator it (s); - - STObject object3 (elements, it, sfTestObject); - - if (object1.getFieldVL (sfTestVL) != j) BOOST_FAIL ("STObject error"); - - if (object3.getFieldVL (sfTestVL) != j) BOOST_FAIL ("STObject error"); - } -} - -BOOST_AUTO_TEST_SUITE_END (); - -// vim:ts=4 diff --git a/modules/ripple_data/protocol/ripple_SerializedObject.h b/modules/ripple_data/protocol/ripple_SerializedObject.h index bcc4fd426a..0e8bd79bec 100644 --- a/modules/ripple_data/protocol/ripple_SerializedObject.h +++ b/modules/ripple_data/protocol/ripple_SerializedObject.h @@ -12,6 +12,8 @@ class STObject , public CountedObject { public: + static char const* getCountedObjectName () { return "STObject"; } + STObject () : mType (NULL) { ; @@ -323,6 +325,8 @@ class STArray , public CountedObject { public: + static char const* getCountedObjectName () { return "STArray"; } + typedef boost::ptr_vector vector; typedef boost::ptr_vector::iterator iterator; typedef boost::ptr_vector::const_iterator const_iterator; diff --git a/modules/ripple_data/protocol/ripple_SerializedObjectUnitTests.cpp b/modules/ripple_data/protocol/ripple_SerializedObjectUnitTests.cpp new file mode 100644 index 0000000000..821328a68c --- /dev/null +++ b/modules/ripple_data/protocol/ripple_SerializedObjectUnitTests.cpp @@ -0,0 +1,86 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +BOOST_AUTO_TEST_SUITE (SerializedObject) + +BOOST_AUTO_TEST_CASE ( FieldManipulation_test ) +{ + using namespace ripple; + + if (sfGeneric.isUseful ()) + BOOST_FAIL ("sfGeneric must not be useful"); + + SField sfTestVL (STI_VL, 255, "TestVL"); + SField sfTestH256 (STI_HASH256, 255, "TestH256"); + SField sfTestU32 (STI_UINT32, 255, "TestU32"); + SField sfTestObject (STI_OBJECT, 255, "TestObject"); + + SOTemplate elements; + elements.push_back (SOElement (sfFlags, SOE_REQUIRED)); + elements.push_back (SOElement (sfTestVL, SOE_REQUIRED)); + elements.push_back (SOElement (sfTestH256, SOE_OPTIONAL)); + elements.push_back (SOElement (sfTestU32, SOE_REQUIRED)); + + STObject object1 (elements, sfTestObject); + STObject object2 (object1); + + if (object1.getSerializer () != object2.getSerializer ()) BOOST_FAIL ("STObject error 1"); + + if (object1.isFieldPresent (sfTestH256) || !object1.isFieldPresent (sfTestVL)) + BOOST_FAIL ("STObject error"); + + object1.makeFieldPresent (sfTestH256); + + if (!object1.isFieldPresent (sfTestH256)) BOOST_FAIL ("STObject Error 2"); + + if (object1.getFieldH256 (sfTestH256) != uint256 ()) BOOST_FAIL ("STObject error 3"); + + if (object1.getSerializer () == object2.getSerializer ()) + { + WriteLog (lsINFO, STObject) << "O1: " << object1.getJson (0); + WriteLog (lsINFO, STObject) << "O2: " << object2.getJson (0); + BOOST_FAIL ("STObject error 4"); + } + + object1.makeFieldAbsent (sfTestH256); + + if (object1.isFieldPresent (sfTestH256)) BOOST_FAIL ("STObject error 5"); + + if (object1.getFlags () != 0) BOOST_FAIL ("STObject error 6"); + + if (object1.getSerializer () != object2.getSerializer ()) BOOST_FAIL ("STObject error 7"); + + STObject copy (object1); + + if (object1.isFieldPresent (sfTestH256)) BOOST_FAIL ("STObject error 8"); + + if (copy.isFieldPresent (sfTestH256)) BOOST_FAIL ("STObject error 9"); + + if (object1.getSerializer () != copy.getSerializer ()) BOOST_FAIL ("STObject error 10"); + + copy.setFieldU32 (sfTestU32, 1); + + if (object1.getSerializer () == copy.getSerializer ()) BOOST_FAIL ("STObject error 11"); + + for (int i = 0; i < 1000; i++) + { + Blob j (i, 2); + + object1.setFieldVL (sfTestVL, j); + + Serializer s; + object1.add (s); + SerializerIterator it (s); + + STObject object3 (elements, it, sfTestObject); + + if (object1.getFieldVL (sfTestVL) != j) BOOST_FAIL ("STObject error"); + + if (object3.getFieldVL (sfTestVL) != j) BOOST_FAIL ("STObject error"); + } +} + +BOOST_AUTO_TEST_SUITE_END (); diff --git a/modules/ripple_data/protocol/ripple_SerializedTypes.cpp b/modules/ripple_data/protocol/ripple_SerializedTypes.cpp index eaacad42e2..7d60b2791d 100644 --- a/modules/ripple_data/protocol/ripple_SerializedTypes.cpp +++ b/modules/ripple_data/protocol/ripple_SerializedTypes.cpp @@ -33,6 +33,8 @@ bool SerializedType::isEquivalent (const SerializedType& t) const void STPathSet::printDebug () { + // VFALCO NOTE Can't use Log::out() because of std::endl + // for (int i = 0; i < value.size (); i++) { std::cerr << i << ": "; @@ -53,13 +55,13 @@ void STPathSet::printDebug () void STPath::printDebug () { - std::cerr << "STPath:" << std::endl; + Log::out() << "STPath:"; for (int i = 0; i < mPath.size (); i++) { RippleAddress nad; nad.setAccountID (mPath[i].mAccountID); - std::cerr << " " << i << ": " << nad.humanAccountID () << std::endl; + Log::out() << " " << i << ": " << nad.humanAccountID (); } } @@ -129,18 +131,20 @@ std::string STUInt16::getText () const { if (getFName () == sfLedgerEntryType) { - LedgerEntryFormat* f = LedgerEntryFormat::getLgrFormat (value); + LedgerFormats::Item const* const item = + LedgerFormats::getInstance ()->findByType (static_cast (value)); - if (f != NULL) - return f->t_name; + if (item != nullptr) + return item->getName (); } if (getFName () == sfTransactionType) { - TxFormat* f = TxFormats::getInstance ().findByType (static_cast (value)); + TxFormats::Item const* const item = + TxFormats::getInstance()->findByType (static_cast (value)); - if (f != NULL) - return f->getName (); + if (item != nullptr) + return item->getName (); } return boost::lexical_cast (value); @@ -150,18 +154,20 @@ Json::Value STUInt16::getJson (int) const { if (getFName () == sfLedgerEntryType) { - LedgerEntryFormat* f = LedgerEntryFormat::getLgrFormat (value); + LedgerFormats::Item const* const item = + LedgerFormats::getInstance ()->findByType (static_cast (value)); - if (f != NULL) - return f->t_name; + if (item != nullptr) + return item->getName (); } if (getFName () == sfTransactionType) { - TxFormat* f = TxFormats::getInstance ().findByType (static_cast (value)); + TxFormats::Item const* const item = + TxFormats::getInstance()->findByType (static_cast (value)); - if (f != NULL) - return f->getName (); + if (item != nullptr) + return item->getName (); } return value; diff --git a/modules/ripple_data/protocol/ripple_SerializedTypes.h b/modules/ripple_data/protocol/ripple_SerializedTypes.h index 5137d21bea..2c01a4f47c 100644 --- a/modules/ripple_data/protocol/ripple_SerializedTypes.h +++ b/modules/ripple_data/protocol/ripple_SerializedTypes.h @@ -723,6 +723,8 @@ public: STAmount getRound () const; void roundSelf (); + + static void canonicalizeRound (bool isNative, uint64& value, int& offset, bool roundUp); private: template diff --git a/modules/ripple_data/protocol/ripple_Serializer.cpp b/modules/ripple_data/protocol/ripple_Serializer.cpp index 2debcb5329..6f9ec1eb74 100644 --- a/modules/ripple_data/protocol/ripple_Serializer.cpp +++ b/modules/ripple_data/protocol/ripple_Serializer.cpp @@ -704,24 +704,3 @@ Blob SerializerIterator::getRaw (int iLength) return mSerializer.getRaw (iPos, iLength); } - -BOOST_AUTO_TEST_SUITE (Serializer_suite) - -BOOST_AUTO_TEST_CASE ( Serializer_PrefixHash_test ) -{ - Serializer s1; - s1.add32 (3); - s1.add256 (uint256 ()); - - Serializer s2; - s2.add32 (0x12345600); - s2.addRaw (s1.peekData ()); - - if (s1.getPrefixHash (0x12345600) != s2.getSHA512Half ()) - BOOST_FAIL ("Prefix hash does not work"); -} - - -BOOST_AUTO_TEST_SUITE_END (); - -// vim:ts=4 diff --git a/modules/ripple_data/protocol/ripple_SerializerUnitTests.cpp b/modules/ripple_data/protocol/ripple_SerializerUnitTests.cpp new file mode 100644 index 0000000000..df6bc3b834 --- /dev/null +++ b/modules/ripple_data/protocol/ripple_SerializerUnitTests.cpp @@ -0,0 +1,25 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +BOOST_AUTO_TEST_SUITE (Serializer_suite) + +BOOST_AUTO_TEST_CASE ( Serializer_PrefixHash_test ) +{ + using namespace ripple; + + Serializer s1; + s1.add32 (3); + s1.add256 (uint256 ()); + + Serializer s2; + s2.add32 (0x12345600); + s2.addRaw (s1.peekData ()); + + if (s1.getPrefixHash (0x12345600) != s2.getSHA512Half ()) + BOOST_FAIL ("Prefix hash does not work"); +} + +BOOST_AUTO_TEST_SUITE_END (); diff --git a/modules/ripple_data/protocol/ripple_TER.cpp b/modules/ripple_data/protocol/ripple_TER.cpp index fa2418a9e1..8e2d1e69a6 100644 --- a/modules/ripple_data/protocol/ripple_TER.cpp +++ b/modules/ripple_data/protocol/ripple_TER.cpp @@ -31,6 +31,7 @@ bool transResultInfo (TER terCode, std::string& strToken, std::string& strHuman) { tecUNFUNDED_ADD, "tecUNFUNDED_ADD", "Insufficient XRP balance for WalletAdd." }, { tecUNFUNDED_OFFER, "tecUNFUNDED_OFFER", "Insufficient balance to fund created offer." }, { tecUNFUNDED_PAYMENT, "tecUNFUNDED_PAYMENT", "Insufficient XRP balance to send." }, + { tecOWNERS, "tecOWNERS", "Non-zero owner count." }, { tefFAILURE, "tefFAILURE", "Failed to apply." }, { tefALREADY, "tefALREADY", "The exact transaction was already in this ledger." }, diff --git a/modules/ripple_data/protocol/ripple_TER.h b/modules/ripple_data/protocol/ripple_TER.h index b51a229454..4ba0387b55 100644 --- a/modules/ripple_data/protocol/ripple_TER.h +++ b/modules/ripple_data/protocol/ripple_TER.h @@ -150,6 +150,7 @@ enum TER // aka TransactionEngineResult tecUNFUNDED = 129, // Deprecated, old ambiguous unfunded. tecMASTER_DISABLED = 130, tecNO_REGULAR_KEY = 131, + tecOWNERS = 132, }; // VFALCO TODO change these to normal functions. diff --git a/modules/ripple_data/protocol/ripple_TxFormat.cpp b/modules/ripple_data/protocol/ripple_TxFormat.cpp deleted file mode 100644 index 49eee8679b..0000000000 --- a/modules/ripple_data/protocol/ripple_TxFormat.cpp +++ /dev/null @@ -1,93 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -// VFALCO TODO Find a way to not use macros. inline function? - -#define TF_BASE \ - << SOElement(sfTransactionType, SOE_REQUIRED) \ - << SOElement(sfFlags, SOE_OPTIONAL) \ - << SOElement(sfSourceTag, SOE_OPTIONAL) \ - << SOElement(sfAccount, SOE_REQUIRED) \ - << SOElement(sfSequence, SOE_REQUIRED) \ - << SOElement(sfPreviousTxnID, SOE_OPTIONAL) \ - << SOElement(sfFee, SOE_REQUIRED) \ - << SOElement(sfOperationLimit, SOE_OPTIONAL) \ - << SOElement(sfSigningPubKey, SOE_REQUIRED) \ - << SOElement(sfTxnSignature, SOE_OPTIONAL) - -#define DECLARE_TF(name, type) tf = TxFormats::getInstance().add (new TxFormat(#name, type)); (*tf) TF_BASE - -void TFInit () -{ - TxFormat* tf; - - DECLARE_TF (AccountSet, ttACCOUNT_SET) - << SOElement (sfEmailHash, SOE_OPTIONAL) - << SOElement (sfWalletLocator, SOE_OPTIONAL) - << SOElement (sfWalletSize, SOE_OPTIONAL) - << SOElement (sfMessageKey, SOE_OPTIONAL) - << SOElement (sfDomain, SOE_OPTIONAL) - << SOElement (sfTransferRate, SOE_OPTIONAL) - << SOElement (sfSetFlag, SOE_OPTIONAL) - << SOElement (sfClearFlag, SOE_OPTIONAL) - ; - - DECLARE_TF (TrustSet, ttTRUST_SET) - << SOElement (sfLimitAmount, SOE_OPTIONAL) - << SOElement (sfQualityIn, SOE_OPTIONAL) - << SOElement (sfQualityOut, SOE_OPTIONAL) - ; - - DECLARE_TF (OfferCreate, ttOFFER_CREATE) - << SOElement (sfTakerPays, SOE_REQUIRED) - << SOElement (sfTakerGets, SOE_REQUIRED) - << SOElement (sfExpiration, SOE_OPTIONAL) - << SOElement (sfOfferSequence, SOE_OPTIONAL) - ; - - DECLARE_TF (OfferCancel, ttOFFER_CANCEL) - << SOElement (sfOfferSequence, SOE_REQUIRED) - ; - - DECLARE_TF (SetRegularKey, ttREGULAR_KEY_SET) - << SOElement (sfRegularKey, SOE_OPTIONAL) - ; - - DECLARE_TF (Payment, ttPAYMENT) - << SOElement (sfDestination, SOE_REQUIRED) - << SOElement (sfAmount, SOE_REQUIRED) - << SOElement (sfSendMax, SOE_OPTIONAL) - << SOElement (sfPaths, SOE_DEFAULT) - << SOElement (sfInvoiceID, SOE_OPTIONAL) - << SOElement (sfDestinationTag, SOE_OPTIONAL) - ; - - DECLARE_TF (Contract, ttCONTRACT) - << SOElement (sfExpiration, SOE_REQUIRED) - << SOElement (sfBondAmount, SOE_REQUIRED) - << SOElement (sfStampEscrow, SOE_REQUIRED) - << SOElement (sfRippleEscrow, SOE_REQUIRED) - << SOElement (sfCreateCode, SOE_OPTIONAL) - << SOElement (sfFundCode, SOE_OPTIONAL) - << SOElement (sfRemoveCode, SOE_OPTIONAL) - << SOElement (sfExpireCode, SOE_OPTIONAL) - ; - - DECLARE_TF (RemoveContract, ttCONTRACT_REMOVE) - << SOElement (sfTarget, SOE_REQUIRED) - ; - - DECLARE_TF (EnableFeature, ttFEATURE) - << SOElement (sfFeature, SOE_REQUIRED) - ; - - DECLARE_TF (SetFee, ttFEE) - << SOElement (sfBaseFee, SOE_REQUIRED) - << SOElement (sfReferenceFeeUnits, SOE_REQUIRED) - << SOElement (sfReserveBase, SOE_REQUIRED) - << SOElement (sfReserveIncrement, SOE_REQUIRED) - ; -} diff --git a/modules/ripple_data/protocol/ripple_TxFormat.h b/modules/ripple_data/protocol/ripple_TxFormat.h deleted file mode 100644 index 09ff0ea2a2..0000000000 --- a/modules/ripple_data/protocol/ripple_TxFormat.h +++ /dev/null @@ -1,76 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_TXFORMAT_H -#define RIPPLE_TXFORMAT_H - -// VFALCO TODO Rename to TxType -// Be aware there are some strings "TransactionType" -// And also we have TransactionType in ripple_SerializeDeclarations.h -// -/** Transaction type identifiers. - - These are part of the binary message format. - - @ingroup protocol -*/ -enum TransactionType -{ - ttINVALID = -1, - - ttPAYMENT = 0, - ttCLAIM = 1, // open - ttWALLET_ADD = 2, - ttACCOUNT_SET = 3, - ttPASSWORD_FUND = 4, // open - ttREGULAR_KEY_SET = 5, - ttNICKNAME_SET = 6, // open - ttOFFER_CREATE = 7, - ttOFFER_CANCEL = 8, - ttCONTRACT = 9, - ttCONTRACT_REMOVE = 10, // can we use the same msg as offer cancel - - ttTRUST_SET = 20, - - ttFEATURE = 100, - ttFEE = 101, -}; - -class TxFormat -{ -public: - TxFormat (char const* name, TransactionType type) - : m_name (name) - , m_type (type) - { - } - - TxFormat& operator<< (SOElement const& el) - { - elements.push_back (el); - - return *this; - } - - /** Retrieve the name of the format. - */ - std::string const& getName () const { return m_name; } - - /** Retrieve the transaction type this format represents. - */ - TransactionType getType () const { return m_type; } - -public: - // VFALCO TODO make an accessor for this - SOTemplate elements; - -private: - std::string const m_name; - TransactionType const m_type; -}; - -#endif -// vim:ts=4 diff --git a/modules/ripple_data/protocol/ripple_TxFormats.cpp b/modules/ripple_data/protocol/ripple_TxFormats.cpp index 14c248c21e..865ca4fbda 100644 --- a/modules/ripple_data/protocol/ripple_TxFormats.cpp +++ b/modules/ripple_data/protocol/ripple_TxFormats.cpp @@ -4,51 +4,94 @@ */ //============================================================================== -TxFormats& TxFormats::getInstance () -{ - static TxFormats instance; - - return instance; -} - -TxFormat* TxFormats::add (TxFormat* txFormat) -{ - // VFALCO TODO Figure out when and how to delete the TxFormat objects later? - m_types [txFormat->getType ()] = txFormat; - m_names [txFormat->getName ()] = txFormat; - - return txFormat; -} - -TxFormat* TxFormats::findByType (TransactionType type) -{ - TxFormat* result = NULL; - - TypeMap::iterator const iter = m_types.find (type); - - if (iter != m_types.end ()) - { - result = iter->second; - } - - return result; -} - -TxFormat* TxFormats::findByName (std::string const& name) -{ - TxFormat* result = NULL; // VFALCO TODO replace all NULL with nullptr - - NameMap::iterator const iter = m_names.find (name); - - if (iter != m_names.end ()) - { - result = iter->second; - } - - return result; -} - TxFormats::TxFormats () + : SharedSingleton (SingletonLifetime::persistAfterCreation) { + add ("AccountSet", ttACCOUNT_SET) + << SOElement (sfEmailHash, SOE_OPTIONAL) + << SOElement (sfWalletLocator, SOE_OPTIONAL) + << SOElement (sfWalletSize, SOE_OPTIONAL) + << SOElement (sfMessageKey, SOE_OPTIONAL) + << SOElement (sfDomain, SOE_OPTIONAL) + << SOElement (sfTransferRate, SOE_OPTIONAL) + << SOElement (sfSetFlag, SOE_OPTIONAL) + << SOElement (sfClearFlag, SOE_OPTIONAL) + ; + + add ("TrustSet", ttTRUST_SET) + << SOElement (sfLimitAmount, SOE_OPTIONAL) + << SOElement (sfQualityIn, SOE_OPTIONAL) + << SOElement (sfQualityOut, SOE_OPTIONAL) + ; + + add ("OfferCreate", ttOFFER_CREATE) + << SOElement (sfTakerPays, SOE_REQUIRED) + << SOElement (sfTakerGets, SOE_REQUIRED) + << SOElement (sfExpiration, SOE_OPTIONAL) + << SOElement (sfOfferSequence, SOE_OPTIONAL) + ; + + add ("OfferCancel", ttOFFER_CANCEL) + << SOElement (sfOfferSequence, SOE_REQUIRED) + ; + + add ("SetRegularKey", ttREGULAR_KEY_SET) + << SOElement (sfRegularKey, SOE_OPTIONAL) + ; + + add ("Payment", ttPAYMENT) + << SOElement (sfDestination, SOE_REQUIRED) + << SOElement (sfAmount, SOE_REQUIRED) + << SOElement (sfSendMax, SOE_OPTIONAL) + << SOElement (sfPaths, SOE_DEFAULT) + << SOElement (sfInvoiceID, SOE_OPTIONAL) + << SOElement (sfDestinationTag, SOE_OPTIONAL) + ; + + add ("Contract", ttCONTRACT) + << SOElement (sfExpiration, SOE_REQUIRED) + << SOElement (sfBondAmount, SOE_REQUIRED) + << SOElement (sfStampEscrow, SOE_REQUIRED) + << SOElement (sfRippleEscrow, SOE_REQUIRED) + << SOElement (sfCreateCode, SOE_OPTIONAL) + << SOElement (sfFundCode, SOE_OPTIONAL) + << SOElement (sfRemoveCode, SOE_OPTIONAL) + << SOElement (sfExpireCode, SOE_OPTIONAL) + ; + + add ("RemoveContract", ttCONTRACT_REMOVE) + << SOElement (sfTarget, SOE_REQUIRED) + ; + + add ("EnableFeature", ttFEATURE) + << SOElement (sfFeature, SOE_REQUIRED) + ; + + add ("SetFee", ttFEE) + << SOElement (sfBaseFee, SOE_REQUIRED) + << SOElement (sfReferenceFeeUnits, SOE_REQUIRED) + << SOElement (sfReserveBase, SOE_REQUIRED) + << SOElement (sfReserveIncrement, SOE_REQUIRED) + ; } +void TxFormats::addCommonFields (Item& item) +{ + item + << SOElement(sfTransactionType, SOE_REQUIRED) + << SOElement(sfFlags, SOE_OPTIONAL) + << SOElement(sfSourceTag, SOE_OPTIONAL) + << SOElement(sfAccount, SOE_REQUIRED) + << SOElement(sfSequence, SOE_REQUIRED) + << SOElement(sfPreviousTxnID, SOE_OPTIONAL) + << SOElement(sfFee, SOE_REQUIRED) + << SOElement(sfOperationLimit, SOE_OPTIONAL) + << SOElement(sfSigningPubKey, SOE_REQUIRED) + << SOElement(sfTxnSignature, SOE_OPTIONAL) + ; +} + +TxFormats* TxFormats::createInstance () +{ + return new TxFormats; +} diff --git a/modules/ripple_data/protocol/ripple_TxFormats.h b/modules/ripple_data/protocol/ripple_TxFormats.h index 12878db2ab..d6414ffd3a 100644 --- a/modules/ripple_data/protocol/ripple_TxFormats.h +++ b/modules/ripple_data/protocol/ripple_TxFormats.h @@ -4,43 +4,54 @@ */ //============================================================================== -#ifndef RIPPLE_TXFORMATS_H -#define RIPPLE_TXFORMATS_H +#ifndef RIPPLE_TXFORMATS_H_INCLUDED +#define RIPPLE_TXFORMATS_H_INCLUDED + +/** Transaction type identifiers. + + These are part of the binary message format. + + @ingroup protocol +*/ +enum TxType +{ + ttINVALID = -1, + + ttPAYMENT = 0, + ttCLAIM = 1, // open + ttWALLET_ADD = 2, + ttACCOUNT_SET = 3, + ttPASSWORD_FUND = 4, // open + ttREGULAR_KEY_SET = 5, + ttNICKNAME_SET = 6, // open + ttOFFER_CREATE = 7, + ttOFFER_CANCEL = 8, + ttCONTRACT = 9, + ttCONTRACT_REMOVE = 10, // can we use the same msg as offer cancel + + ttTRUST_SET = 20, + + ttFEATURE = 100, + ttFEE = 101, +}; /** Manages the list of known transaction formats. */ class TxFormats + : public KnownFormats + , public SharedSingleton { -public: - // VFALCO TODO Make this a member of the Application object instead of a singleton? - static TxFormats& getInstance (); - - /** Add a format. - - The caller is responsible for freeing the memory. - - @return The passed format. - */ - TxFormat* add (TxFormat* txFormat); - - /** Retrieve a format based on its transaction type. - */ - TxFormat* findByType (TransactionType type); - - /** Retrieve a format based on its name. - */ - TxFormat* findByName (std::string const& name); - private: + /** Create the object. + + This will load the object will all the known transaction formats. + */ TxFormats (); -private: - typedef std::map NameMap; - typedef std::map TypeMap; + void addCommonFields (Item& item); - NameMap m_names; - TypeMap m_types; +public: + static TxFormats* createInstance (); }; #endif -// vim:ts=4 diff --git a/modules/ripple_data/ripple_data.cpp b/modules/ripple_data/ripple_data.cpp index 9cb8edb024..12b2d5d5fb 100644 --- a/modules/ripple_data/ripple_data.cpp +++ b/modules/ripple_data/ripple_data.cpp @@ -10,6 +10,8 @@ @ingroup ripple_data */ +#include "BeastConfig.h" + #include "ripple_data.h" #include @@ -33,7 +35,6 @@ #include #include #include -#include #include #include @@ -54,10 +55,8 @@ #undef min #endif -#if RIPPLE_USE_NAMESPACE namespace ripple { -#endif #include "crypto/ripple_Base58.h" // for RippleAddress #include "crypto/ripple_CKey.h" // needs RippleAddress VFALCO TODO remove this dependency cycle @@ -72,7 +71,7 @@ namespace ripple #include "crypto/ripple_RFC1751.cpp" #include "protocol/ripple_FieldNames.cpp" -#include "protocol/ripple_LedgerFormat.cpp" +#include "protocol/ripple_LedgerFormats.cpp" #include "protocol/ripple_PackedMessage.cpp" #include "protocol/ripple_RippleAddress.cpp" #include "protocol/ripple_SerializedTypes.cpp" @@ -80,7 +79,6 @@ namespace ripple #include "protocol/ripple_SerializedObjectTemplate.cpp" #include "protocol/ripple_SerializedObject.cpp" #include "protocol/ripple_TER.cpp" -#include "protocol/ripple_TxFormat.cpp" #include "protocol/ripple_TxFormats.cpp" // These are for STAmount @@ -91,11 +89,14 @@ static const uint64 tenTo17m1 = tenTo17 - 1; #include "protocol/ripple_STAmount.cpp" #include "protocol/ripple_STAmountRound.cpp" -#include "utility/ripple_JSONCache.cpp" - -#if RIPPLE_USE_NAMESPACE } -#endif + +// These must be outside the namespace because of boost +#include "crypto/ripple_CKeyDeterministicUnitTests.cpp" +#include "protocol/ripple_RippleAddressUnitTests.cpp" +#include "protocol/ripple_SerializedObjectUnitTests.cpp" +#include "protocol/ripple_SerializerUnitTests.cpp" +#include "protocol/ripple_STAmountUnitTests.cpp" // VFALCO TODO Fix this for SConstruct #if BEAST_MSVC diff --git a/modules/ripple_data/ripple_data.h b/modules/ripple_data/ripple_data.h index bfef508e51..82fd48c93c 100644 --- a/modules/ripple_data/ripple_data.h +++ b/modules/ripple_data/ripple_data.h @@ -20,42 +20,15 @@ #ifndef RIPPLE_DATA_RIPPLEHEADER #define RIPPLE_DATA_RIPPLEHEADER -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include // VFALCO NOTE this looks like junk - -#include -#include -#include -#include - -//------------------------------------------------------------------------------ - // VFALCO TODO try to reduce these dependencies #include "../ripple_basics/ripple_basics.h" -// VFALCO TODO don't expose leveldb throughout the headers -#include "../ripple_leveldb/ripple_leveldb.h" - // VFALCO TODO figure out a good place for this file, perhaps give it some // additional hierarchy via directories. #include "ripple.pb.h" -#if RIPPLE_USE_NAMESPACE namespace ripple { -#endif #include "crypto/ripple_CBigNum.h" #include "crypto/ripple_Base58.h" // VFALCO TODO Can be moved to .cpp if we clean up setAlphabet stuff @@ -71,20 +44,16 @@ namespace ripple #include "protocol/ripple_TER.h" #include "protocol/ripple_SerializedTypes.h" // needs Serializer, TER #include "protocol/ripple_SerializedObjectTemplate.h" + #include "protocol/ripple_KnownFormats.h" + #include "protocol/ripple_LedgerFormats.h" // needs SOTemplate from SerializedObjectTemplate + #include "protocol/ripple_TxFormats.h" #include "protocol/ripple_SerializedObject.h" -#include "protocol/ripple_LedgerFormat.h" // needs SOTemplate from SerializedObject #include "protocol/ripple_TxFlags.h" -#include "protocol/ripple_TxFormat.h" -#include "protocol/ripple_TxFormats.h" -#include "utility/ripple_JSONCache.h" #include "utility/ripple_UptimeTimerAdapter.h" -#if RIPPLE_USE_NAMESPACE } -#endif -#if RIPPLE_USE_NAMESPACE namespace boost { template <> @@ -135,57 +104,5 @@ namespace boost typedef ripple::STArray::const_iterator type; }; } -#else -namespace boost -{ - template <> - struct range_mutable_iterator - { - typedef std::vector ::iterator type; - }; - - template <> - struct range_const_iterator - { - typedef std::vector ::const_iterator type; - }; - - template <> - struct range_mutable_iterator - { - typedef std::vector ::iterator type; - }; - - template <> - struct range_const_iterator - { - typedef std::vector ::const_iterator type; - }; - - template <> - struct range_mutable_iterator - { - typedef STObject::iterator type; - }; - - template <> - struct range_const_iterator - { - typedef STObject::const_iterator type; - }; - - template <> - struct range_mutable_iterator - { - typedef STArray::iterator type; - }; - - template <> - struct range_const_iterator - { - typedef STArray::const_iterator type; - }; -} -#endif #endif diff --git a/modules/ripple_data/utility/ripple_JSONCache.cpp b/modules/ripple_data/utility/ripple_JSONCache.cpp deleted file mode 100644 index 1eaed909cb..0000000000 --- a/modules/ripple_data/utility/ripple_JSONCache.cpp +++ /dev/null @@ -1,186 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -JSONCache::Key::Key (int op, uint256 const& ledger, uint160 const& object, int lastUse) - : mLedger (ledger) - , mObject (object) - , mOperation (op) - , mLastUse (lastUse) -{ - mHash = static_cast (mOperation); - - mLedger.hash_combine (mHash); - - mObject.hash_combine (mHash); -} - -int JSONCache::Key::compare (Key const& other) const -{ - if (mHash < other.mHash) return -1; - - if (mHash > other.mHash) return 1; - - if (mOperation < other.mOperation) return -1; - - if (mOperation > other.mOperation) return 1; - - if (mLedger < other.mLedger) return -1; - - if (mLedger > other.mLedger) return 1; - - if (mObject < other.mObject) return -1; - - if (mObject > other.mObject) return 1; - - return 0; -} - -bool JSONCache::Key::operator< (Key const& rhs) const -{ - return compare (rhs) < 0; -} -bool JSONCache::Key::operator> (Key const& rhs) const -{ - return compare (rhs) > 0; -} -bool JSONCache::Key::operator<= (Key const& rhs) const -{ - return compare (rhs) <= 0; -} -bool JSONCache::Key::operator>= (Key const& rhs) const -{ - return compare (rhs) >= 0; -} -bool JSONCache::Key::operator!= (Key const& rhs) const -{ - return compare (rhs) != 0; -} -bool JSONCache::Key::operator== (Key const& rhs) const -{ - return compare (rhs) == 0; -} - -void JSONCache::Key::touch (Key const& key) const -{ - mLastUse = key.mLastUse; -} - -bool JSONCache::Key::isExpired (int expireTimeSeconds) const -{ - return mLastUse < expireTimeSeconds; -} - -std::size_t JSONCache::Key::getHash () const -{ - return mHash; -} - -//------------------------------------------------------------------------------ - -JSONCache::JSONCache (int expirationTimeInSeconds) - : m_expirationTime (expirationTimeInSeconds) - , mHits (0) - , mMisses (0) -{ -} - -//------------------------------------------------------------------------------ - -float JSONCache::getHitRate () -{ - boost::recursive_mutex::scoped_lock sl (m_lock); - - return (static_cast (mHits) * 100.f) / (1.0f + mHits + mMisses); -} - -//------------------------------------------------------------------------------ - -int JSONCache::getNumberOfEntries () -{ - boost::recursive_mutex::scoped_lock sl (m_lock); - - return m_cache.size (); -} - -//------------------------------------------------------------------------------ - -JSONCache::data_t JSONCache::getEntry (Kind kind, LedgerHash const& ledger, uint160 const& object) -{ - JSONCache::data_t result; // default constructor indicates not found - - Key key (kind, ledger, object, getUptime ()); - - { - boost::recursive_mutex::scoped_lock sl (m_lock); - - boost::unordered_map ::iterator it = m_cache.find (key); - - if (it != m_cache.end ()) - { - ++mHits; - - it->first.touch (key); - - result = it->second; - } - else - { - ++mMisses; - } - } - - return result; -} - -//------------------------------------------------------------------------------ - -void JSONCache::storeEntry (Kind kind, uint256 const& ledger, uint160 const& object, data_t const& data) -{ - Key key (kind, ledger, object, getUptime ()); - - { - boost::recursive_mutex::scoped_lock sl (m_lock); - - m_cache.insert (std::pair (key, data)); - } -} - -//------------------------------------------------------------------------------ - -void JSONCache::sweep () -{ - int sweepTime = getUptime (); - - if (sweepTime >= m_expirationTime) - { - sweepTime -= m_expirationTime; - - { - boost::recursive_mutex::scoped_lock sl (m_lock); - - boost::unordered_map ::iterator it = m_cache.begin (); - - while (it != m_cache.end ()) - { - if (it->first.isExpired (sweepTime)) - { - it = m_cache.erase (it); - } - else - { - ++it; - } - } - } - } -} - -//------------------------------------------------------------------------------ - -int JSONCache::getUptime () const -{ - return UptimeTimer::getInstance ().getElapsedSeconds (); -} diff --git a/modules/ripple_data/utility/ripple_JSONCache.h b/modules/ripple_data/utility/ripple_JSONCache.h deleted file mode 100644 index b86bdc150e..0000000000 --- a/modules/ripple_data/utility/ripple_JSONCache.h +++ /dev/null @@ -1,98 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_JSCONCACHE_H -#define RIPPLE_JSCONCACHE_H - -/** A simple cache for JSON. - - @note All member functions are thread-safe. -*/ -class JSONCache -{ -public: - class Key - { - public: - Key (int op, const uint256& ledger, const uint160& object, int lastUse); - int compare (const Key& k) const; - bool operator< (const Key& k) const; - bool operator> (const Key& k) const; - bool operator<= (const Key& k) const; - bool operator>= (const Key& k) const; - bool operator!= (const Key& k) const; - bool operator== (const Key& k) const; - - void touch (Key const& key) const; - bool isExpired (int expireTime) const; - - std::size_t getHash () const; - - private: - uint256 mLedger; - uint160 mObject; - int mOperation; - mutable int mLastUse; - std::size_t mHash; - }; - -public: - typedef boost::shared_ptr data_t; - -public: - enum Kind - { - kindLines, - kindOffers - }; - - /** Construct the cache. - - @param expirationTimeInSeconds The time until cached items expire, in seconds. - */ - explicit JSONCache (int expirationTimeInSeconds); - - /** Return the fraction of cache hits. - */ - float getHitRate (); - - /** Return the number of cached items. - */ - int getNumberOfEntries (); - - /** Retrieve a cached item. - - @return The item, or a default constructed container if it was not found. - */ - data_t getEntry (Kind kind, LedgerHash const& ledger, uint160 const& object); - - /** Store an item in the cache. - */ - void storeEntry (Kind kind, LedgerHash const& ledger, uint160 const& object, data_t const& data); - - /** Purge expired items. - - This must be called periodically. - */ - void sweep (); - -private: - int getUptime () const; - -private: - int const m_expirationTime; - boost::unordered_map m_cache; - boost::recursive_mutex m_lock; - uint64 mHits; - uint64 mMisses; -}; - -inline std::size_t hash_value (JSONCache::Key const& key) -{ - return key.getHash (); -} - -#endif diff --git a/modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp b/modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp new file mode 100644 index 0000000000..5b6cf45203 --- /dev/null +++ b/modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp @@ -0,0 +1,97 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +// Unity build file for LevelDB + +#include "BeastConfig.h" + +#include "ripple_hyperleveldb.h" + +#if RIPPLE_HYPERLEVELDB_AVAILABLE + +// Set the appropriate LevelDB platform macro based on our platform. +// +#if BEAST_WIN32 + #define LEVELDB_PLATFORM_WINDOWS + +#else + #define LEVELDB_PLATFORM_POSIX + + #if BEAST_MAC || BEAST_IOS + #define OS_MACOSX + + // VFALCO TODO Distinguish between BEAST_BSD and BEAST_FREEBSD + #elif BEAST_BSD + #define OS_FREEBSD + + #endif + +#endif + +#if BEAST_MSVC +#pragma warning (push) +#pragma warning (disable: 4018) // signed/unsigned mismatch +#pragma warning (disable: 4244) // conversion, possible loss of data +#endif + +#include "hyperleveldb/db/builder.cc" +#include "hyperleveldb/db/db_impl.cc" +#include "hyperleveldb/db/db_iter.cc" +#include "hyperleveldb/db/dbformat.cc" +#include "hyperleveldb/db/filename.cc" +#include "hyperleveldb/db/log_reader.cc" +#include "hyperleveldb/db/log_writer.cc" +#include "hyperleveldb/db/memtable.cc" +#include "hyperleveldb/db/repair.cc" +#include "hyperleveldb/db/table_cache.cc" +#include "hyperleveldb/db/version_edit.cc" +#include "hyperleveldb/db/version_set.cc" +#include "hyperleveldb/db/write_batch.cc" + +#include "hyperleveldb/table/block.cc" +#include "hyperleveldb/table/block_builder.cc" +#include "hyperleveldb/table/filter_block.cc" +#include "hyperleveldb/table/format.cc" +#include "hyperleveldb/table/iterator.cc" +#include "hyperleveldb/table/merger.cc" +#include "hyperleveldb/table/table.cc" +#include "hyperleveldb/table/table_builder.cc" +#include "hyperleveldb/table/two_level_iterator.cc" + +#include "hyperleveldb/util/arena.cc" +#include "hyperleveldb/util/bloom.cc" +#include "hyperleveldb/util/cache.cc" +#include "hyperleveldb/util/coding.cc" +#include "hyperleveldb/util/comparator.cc" +#include "hyperleveldb/util/crc32c.cc" +#include "hyperleveldb/util/env.cc" +#include "hyperleveldb/util/filter_policy.cc" +#include "hyperleveldb/util/hash.cc" +#include "hyperleveldb/util/histogram.cc" +#include "hyperleveldb/util/logging.cc" +#include "hyperleveldb/util/options.cc" +#include "hyperleveldb/util/status.cc" + +// Platform Specific + +#if defined (LEVELDB_PLATFORM_WINDOWS) +# include "hyperleveldb/util/env_win.cc" +# include "hyperleveldb/port/port_win.cc" + +#elif defined (LEVELDB_PLATFORM_POSIX) +# include "hyperleveldb/util/env_posix.cc" +# include "hyperleveldb/port/port_posix.cc" + +#elif defined (LEVELDB_PLATFORM_ANDROID) +# error Missing Android port! + +#endif + +#ifdef BEAST_MSVC +#pragma warning (pop) +#endif + +#endif diff --git a/modules/ripple_hyperleveldb/ripple_hyperleveldb.h b/modules/ripple_hyperleveldb/ripple_hyperleveldb.h new file mode 100644 index 0000000000..10e9b2a5d6 --- /dev/null +++ b/modules/ripple_hyperleveldb/ripple_hyperleveldb.h @@ -0,0 +1,27 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_HYPERLEVELDB_RIPPLEHEADER +#define RIPPLE_HYPERLEVELDB_RIPPLEHEADER + +#include "beast/modules/beast_core/system/beast_TargetPlatform.h" + +#if ! BEAST_WIN32 + +#define RIPPLE_HYPERLEVELDB_AVAILABLE 1 + +#include "hyperleveldb/hyperleveldb/cache.h" +#include "hyperleveldb/hyperleveldb/filter_policy.h" +#include "hyperleveldb/hyperleveldb/db.h" +#include "hyperleveldb/hyperleveldb/write_batch.h" + +#else + +#define RIPPLE_HYPERLEVELDB_AVAILABLE 0 + +#endif + +#endif diff --git a/modules/ripple_json/ripple_json.cpp b/modules/ripple_json/ripple_json.cpp index 6dabe12ff8..419905dcc7 100644 --- a/modules/ripple_json/ripple_json.cpp +++ b/modules/ripple_json/ripple_json.cpp @@ -10,6 +10,8 @@ @ingroup ripple_json */ +#include "BeastConfig.h" + #include "ripple_json.h" #include @@ -30,15 +32,11 @@ #define JSON_ASSERT( condition ) assert( condition ); // @todo <= change this into an exception throw #define JSON_ASSERT_MESSAGE( condition, message ) if (!( condition )) throw std::runtime_error( message ); -#if RIPPLE_USE_NAMESPACE namespace ripple { -#endif #include "json/json_reader.cpp" #include "json/json_value.cpp" #include "json/json_writer.cpp" -#if RIPPLE_USE_NAMESPACE } -#endif diff --git a/modules/ripple_json/ripple_json.h b/modules/ripple_json/ripple_json.h index ecbfa700f8..7bda52df0d 100644 --- a/modules/ripple_json/ripple_json.h +++ b/modules/ripple_json/ripple_json.h @@ -20,10 +20,7 @@ #ifndef RIPPLE_JSON_RIPPLEHEADER #define RIPPLE_JSON_RIPPLEHEADER -#include -#include -#include -#include +#include "../modules/ripple_basics/ripple_basics.h" #include "json/json_config.h" // Needed before these cpptl includes @@ -37,10 +34,8 @@ # include #endif -#if RIPPLE_USE_NAMESPACE namespace ripple { -#endif #include "json/json_forwards.h" #include "json/json_features.h" @@ -48,8 +43,6 @@ namespace ripple #include "json/json_reader.h" #include "json/json_writer.h" -#if RIPPLE_USE_NAMESPACE } -#endif #endif diff --git a/modules/ripple_leveldb/ripple_leveldb.cpp b/modules/ripple_leveldb/ripple_leveldb.cpp index fcf011da74..abc46ef9f8 100644 --- a/modules/ripple_leveldb/ripple_leveldb.cpp +++ b/modules/ripple_leveldb/ripple_leveldb.cpp @@ -6,6 +6,8 @@ // Unity build file for LevelDB +#include "BeastConfig.h" + #include "ripple_leveldb.h" #include "beast/modules/beast_core/system/beast_TargetPlatform.h" @@ -15,16 +17,18 @@ #if BEAST_WIN32 #define LEVELDB_PLATFORM_WINDOWS -#elif BEAST_MAC || BEAST_IOS - #define OS_MACOSX - -// VFALCO TODO Distinguish between BEAST_BSD and BEAST_FREEBSD -#elif BEAST_BSD - #define OS_FREEBSD - #else #define LEVELDB_PLATFORM_POSIX + #if BEAST_MAC || BEAST_IOS + #define OS_MACOSX + + // VFALCO TODO Distinguish between BEAST_BSD and BEAST_FREEBSD + #elif BEAST_BSD + #define OS_FREEBSD + + #endif + #endif #if BEAST_MSVC diff --git a/modules/ripple_mdb/ripple_mdb.c b/modules/ripple_mdb/ripple_mdb.c new file mode 100644 index 0000000000..8f66bbcf83 --- /dev/null +++ b/modules/ripple_mdb/ripple_mdb.c @@ -0,0 +1,18 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +// Unity build file for MDB + +#include "BeastConfig.h" + +#include "ripple_mdb.h" + +#if RIPPLE_MDB_AVAILABLE + +#include "libraries/liblmdb/mdb.c" +#include "libraries/liblmdb/midl.c" + +#endif diff --git a/modules/ripple_mdb/ripple_mdb.h b/modules/ripple_mdb/ripple_mdb.h new file mode 100644 index 0000000000..faefb502b3 --- /dev/null +++ b/modules/ripple_mdb/ripple_mdb.h @@ -0,0 +1,23 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_MDB_H_INCLUDED +#define RIPPLE_MDB_H_INCLUDED + +#include "beast/modules/beast_core/system/beast_TargetPlatform.h" + +#if ! BEAST_WIN32 +#define RIPPLE_MDB_AVAILABLE 1 + +#include "libraries/liblmdb/lmdb.h" + +#else +// mdb is unsupported on Win32 +#define RIPPLE_MDB_AVAILBLE 0 + +#endif + +#endif diff --git a/src/cpp/ripple/HTTPRequest.cpp b/modules/ripple_net/basics/ripple_HTTPRequest.cpp similarity index 95% rename from src/cpp/ripple/HTTPRequest.cpp rename to modules/ripple_net/basics/ripple_HTTPRequest.cpp index 15ec644b02..2c066c7991 100644 --- a/src/cpp/ripple/HTTPRequest.cpp +++ b/modules/ripple_net/basics/ripple_HTTPRequest.cpp @@ -18,7 +18,7 @@ void HTTPRequest::reset () eState = await_request; } -HTTPRequestAction HTTPRequest::requestDone (bool forceClose) +HTTPRequest::Action HTTPRequest::requestDone (bool forceClose) { if (forceClose || bShouldClose) return haCLOSE_CONN; @@ -35,7 +35,7 @@ std::string HTTPRequest::getReplyHeaders (bool forceClose) return "Connection: Keep-Alive\r\n"; } -HTTPRequestAction HTTPRequest::consume (boost::asio::streambuf& buf) +HTTPRequest::Action HTTPRequest::consume (boost::asio::streambuf& buf) { std::string line; std::istream is (&buf); diff --git a/src/cpp/ripple/HTTPRequest.h b/modules/ripple_net/basics/ripple_HTTPRequest.h similarity index 76% rename from src/cpp/ripple/HTTPRequest.h rename to modules/ripple_net/basics/ripple_HTTPRequest.h index 9219eba3ab..b272cc24b9 100644 --- a/src/cpp/ripple/HTTPRequest.h +++ b/modules/ripple_net/basics/ripple_HTTPRequest.h @@ -4,23 +4,23 @@ */ //============================================================================== -#ifndef HTTPREQUEST__HPP -#define HTTPREQUEST__HPP - -enum HTTPRequestAction -{ - // What the application code needs to do - haERROR = 0, - haREAD_LINE = 1, - haREAD_RAW = 2, - haDO_REQUEST = 3, - haCLOSE_CONN = 4 -}; +#ifndef RIPPLE_HTTPREQUEST_H_INCLUDED +#define RIPPLE_HTTPREQUEST_H_INCLUDED +/** An HTTP request we are handling from a client. +*/ class HTTPRequest { - // an HTTP request we are handling from a client public: + enum Action + { + // What the application code needs to do + haERROR = 0, + haREAD_LINE = 1, + haREAD_RAW = 2, + haDO_REQUEST = 3, + haCLOSE_CONN = 4 + }; HTTPRequest () : eState (await_request), iDataSize (0), bShouldClose (true) { @@ -59,8 +59,8 @@ public: } std::string getReplyHeaders (bool forceClose); - HTTPRequestAction consume (boost::asio::streambuf&); - HTTPRequestAction requestDone (bool forceClose); // call after reply is sent + Action consume (boost::asio::streambuf&); + Action requestDone (bool forceClose); // call after reply is sent int getDataSize () { diff --git a/src/cpp/ripple/HttpsClient.cpp b/modules/ripple_net/basics/ripple_HttpsClient.cpp similarity index 97% rename from src/cpp/ripple/HttpsClient.cpp rename to modules/ripple_net/basics/ripple_HttpsClient.cpp index c90979fd89..793d3acd7d 100644 --- a/src/cpp/ripple/HttpsClient.cpp +++ b/modules/ripple_net/basics/ripple_HttpsClient.cpp @@ -10,6 +10,9 @@ SETUP_LOG (HttpsClient) +// VFALCO NOTE Why use theConfig.SSL_CONTEXT instead of just passing it? +// TODO Remove all theConfig deps from this file +// HttpsClient::HttpsClient (boost::asio::io_service& io_service, const unsigned short port, std::size_t responseMax) @@ -313,11 +316,18 @@ void HttpsClient::handleHeader (const boost::system::error_code& ecResult, std:: mResponseMax = size; } - if (mResponseMax == 0) // no body wanted or available + if (mResponseMax == 0) + { + // no body wanted or available invokeComplete (ecResult, mStatus); - else if (mBody.size () >= mResponseMax) // we got the whole thing + } + else if (mBody.size () >= mResponseMax) + { + // we got the whole thing invokeComplete (ecResult, mStatus, mBody); + } else + { mSocket.async_read ( mResponse.prepare (mResponseMax - mBody.size ()), boost::asio::transfer_all (), @@ -325,6 +335,7 @@ void HttpsClient::handleHeader (const boost::system::error_code& ecResult, std:: shared_from_this (), boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); + } } void HttpsClient::handleData (const boost::system::error_code& ecResult, std::size_t bytes_transferred) diff --git a/src/cpp/ripple/HttpsClient.h b/modules/ripple_net/basics/ripple_HttpsClient.h similarity index 73% rename from src/cpp/ripple/HttpsClient.h rename to modules/ripple_net/basics/ripple_HttpsClient.h index 2bb2eabd57..59920f1426 100644 --- a/src/cpp/ripple/HttpsClient.h +++ b/modules/ripple_net/basics/ripple_HttpsClient.h @@ -4,14 +4,16 @@ */ //============================================================================== -#ifndef _HTTPS_CLIENT_ -#define _HTTPS_CLIENT_ +#ifndef RIPPLE_HTTPSCLIENT_H_INCLUDED +#define RIPPLE_HTTPSCLIENT_H_INCLUDED +// VFALCO TODO Make this an abstract interface. // -// Async https client. -// - -class HttpsClient : public boost::enable_shared_from_this +/** Provides an asynchronous HTTPS client implementation. +*/ +class HttpsClient + : public boost::enable_shared_from_this + , LeakChecked { public: HttpsClient ( @@ -20,29 +22,36 @@ public: std::size_t responseMax ); + // VFALCO NOTE Putting "https" is redundant, the class is + // already called HttpsClient. + // + // VFALCO TODO Rename these to request, get, and next. + // void httpsRequest ( bool bSSL, std::deque deqSites, - FUNCTION_TYPE build, + FUNCTION_TYPE build, boost::posix_time::time_duration timeout, - FUNCTION_TYPE complete); + FUNCTION_TYPE complete); void httpsGet ( bool bSSL, std::deque deqSites, const std::string& strPath, boost::posix_time::time_duration timeout, - FUNCTION_TYPE complete); + FUNCTION_TYPE complete); + // VFALCO TODO These statics all belong in some HttpsClientOperations class + // static void httpsGet ( bool bSSL, boost::asio::io_service& io_service, - std::deque deqSites, + std::deque deqSites, const unsigned short port, const std::string& strPath, std::size_t responseMax, boost::posix_time::time_duration timeout, - FUNCTION_TYPE complete); + FUNCTION_TYPE complete); static void httpsGet ( bool bSSL, @@ -52,17 +61,17 @@ public: const std::string& strPath, std::size_t responseMax, boost::posix_time::time_duration timeout, - FUNCTION_TYPE complete); + FUNCTION_TYPE complete); static void httpsRequest ( bool bSSL, boost::asio::io_service& io_service, std::string strSite, const unsigned short port, - FUNCTION_TYPE build, + FUNCTION_TYPE build, std::size_t responseMax, boost::posix_time::time_duration timeout, - FUNCTION_TYPE complete); + FUNCTION_TYPE complete); static void sendSMS (boost::asio::io_service& io_service, const std::string& strText); @@ -117,4 +126,4 @@ private: }; #endif -// vim:ts=4 + diff --git a/modules/ripple_net/basics/ripple_RPCServer.cpp b/modules/ripple_net/basics/ripple_RPCServer.cpp new file mode 100644 index 0000000000..6aa5c803c0 --- /dev/null +++ b/modules/ripple_net/basics/ripple_RPCServer.cpp @@ -0,0 +1,268 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +SETUP_LOG (RPCServer) + +class RPCServerImp : public RPCServer, LeakChecked +{ +public: + RPCServerImp ( + boost::asio::io_service& io_service, + boost::asio::ssl::context& context, + Handler& handler) + : m_handler (handler) + , mSocket (io_service, context) + , mStrand (io_service) + { + } + + //-------------------------------------------------------------------------- +private: + enum + { + maxQueryBytes = 1024 * 1024 + }; + + void connected () + { + boost::asio::async_read_until ( + mSocket, + mLineBuffer, + "\r\n", + mStrand.wrap (boost::bind ( + &RPCServerImp::handle_read_line, + boost::static_pointer_cast (shared_from_this ()), + boost::asio::placeholders::error))); + } + + //-------------------------------------------------------------------------- + + void handle_write (const boost::system::error_code& e) + { + if (!e) + { + HTTPRequest::Action action = mHTTPRequest.requestDone (false); + + if (action == HTTPRequest::haCLOSE_CONN) + { + mSocket.async_shutdown (mStrand.wrap (boost::bind ( + &RPCServerImp::handle_shutdown, + boost::static_pointer_cast (shared_from_this()), + boost::asio::placeholders::error))); + } + else + { + boost::asio::async_read_until ( + mSocket, + mLineBuffer, + "\r\n", + mStrand.wrap (boost::bind ( + &RPCServerImp::handle_read_line, + boost::static_pointer_cast (shared_from_this()), + boost::asio::placeholders::error))); + } + } + + if (e != boost::asio::error::operation_aborted) + { + // VFALCO TODO What is this for? It was commented out. + // + //connection_manager_.stop (shared_from_this ()); + } + } + + //-------------------------------------------------------------------------- + + void handle_read_line (const boost::system::error_code& e) + { + if (! e) + { + HTTPRequest::Action action = mHTTPRequest.consume (mLineBuffer); + + if (action == HTTPRequest::haDO_REQUEST) + { + // request with no body + WriteLog (lsWARNING, RPCServer) << "RPC HTTP request with no body"; + + mSocket.async_shutdown (mStrand.wrap (boost::bind ( + &RPCServerImp::handle_shutdown, + boost::static_pointer_cast (shared_from_this ()), + boost::asio::placeholders::error))); + } + else if (action == HTTPRequest::haREAD_LINE) + { + boost::asio::async_read_until ( + mSocket, + mLineBuffer, + "\r\n", + mStrand.wrap (boost::bind ( + &RPCServerImp::handle_read_line, + boost::static_pointer_cast (shared_from_this ()), + boost::asio::placeholders::error))); + } + else if (action == HTTPRequest::haREAD_RAW) + { + int rLen = mHTTPRequest.getDataSize (); + + if ((rLen < 0) || (rLen > maxQueryBytes)) + { + WriteLog (lsWARNING, RPCServer) << "Illegal RPC request length " << rLen; + + mSocket.async_shutdown (mStrand.wrap (boost::bind ( + &RPCServerImp::handle_shutdown, + boost::static_pointer_cast (shared_from_this ()), + boost::asio::placeholders::error))); + } + else + { + int alreadyHave = mLineBuffer.size (); + + if (alreadyHave < rLen) + { + mQueryVec.resize (rLen - alreadyHave); + + boost::asio::async_read ( + mSocket, + boost::asio::buffer (mQueryVec), + mStrand.wrap (boost::bind ( + &RPCServerImp::handle_read_req, + boost::static_pointer_cast (shared_from_this ()), + boost::asio::placeholders::error))); + + WriteLog (lsTRACE, RPCServer) << "Waiting for completed request: " << rLen; + } + else + { + // we have the whole thing + mQueryVec.resize (0); + + handle_read_req (e); + } + } + } + else + { + mSocket.async_shutdown (mStrand.wrap (boost::bind ( + &RPCServerImp::handle_shutdown, + boost::static_pointer_cast (shared_from_this ()), + boost::asio::placeholders::error))); + } + } + } + + //-------------------------------------------------------------------------- + + void handle_read_req (const boost::system::error_code& ec) + { + std::string req; + + if (mLineBuffer.size ()) + { + req.assign (boost::asio::buffer_cast (mLineBuffer.data ()), mLineBuffer.size ()); + + mLineBuffer.consume (mLineBuffer.size ()); + } + + req += strCopy (mQueryVec); + + if (! m_handler.isAuthorized (mHTTPRequest.peekHeaders ())) + { + mReplyStr = m_handler.createResponse (403, "Forbidden"); + } + else + { + mReplyStr = handleRequest (req); + } + + boost::asio::async_write ( + mSocket, + boost::asio::buffer (mReplyStr), + mStrand.wrap (boost::bind ( + &RPCServerImp::handle_write, + boost::static_pointer_cast (shared_from_this ()), + boost::asio::placeholders::error))); + } + + //-------------------------------------------------------------------------- + + void handle_shutdown (const boost::system::error_code& ec) + { + // nothing to do, we just keep the object alive + } + + //-------------------------------------------------------------------------- + + // JSON-RPC request must contain "method", "params", and "id" fields. + // + std::string handleRequest (const std::string& request) + { + WriteLog (lsTRACE, RPCServer) << "handleRequest " << request; + + // Figure out the remote address. + // VFALCO TODO Clean up this try/catch nonsense. + // + std::string remoteAddress; + + try + { + remoteAddress = mSocket.PlainSocket ().remote_endpoint ().address ().to_string (); + } + catch (...) + { + // endpoint already disconnected + return ""; + } + + return m_handler.processRequest (request, remoteAddress); + } + + //-------------------------------------------------------------------------- + + AutoSocket& getSocket () + { + return mSocket; + } + + //-------------------------------------------------------------------------- + + boost::asio::ip::tcp::socket& getRawSocket () + { + return mSocket.PlainSocket (); + } + + //-------------------------------------------------------------------------- + + std::string getRemoteAddressText () + { + std::string address; + + address = mSocket.PlainSocket ().remote_endpoint ().address ().to_string (); + + return address; + } + +private: + Handler& m_handler; + + AutoSocket mSocket; + boost::asio::io_service::strand mStrand; + + boost::asio::streambuf mLineBuffer; + Blob mQueryVec; + std::string mReplyStr; + + HTTPRequest mHTTPRequest; +}; + +//------------------------------------------------------------------------------ + +RPCServer::pointer RPCServer::New ( + boost::asio::io_service& io_service, + boost::asio::ssl::context& context, + Handler& handler) +{ + return pointer (new RPCServerImp (io_service, context, handler)); +} diff --git a/modules/ripple_net/basics/ripple_RPCServer.h b/modules/ripple_net/basics/ripple_RPCServer.h new file mode 100644 index 0000000000..1290180ecd --- /dev/null +++ b/modules/ripple_net/basics/ripple_RPCServer.h @@ -0,0 +1,68 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_RPCSERVER_H_INCLUDED +#define RIPPLE_RPCSERVER_H_INCLUDED + +/** Provides RPC services to a client. + + Each client has a separate instance of this object. +*/ +// VFALCO NOTE This looks like intrusve shared object? +// +class RPCServer : public boost::enable_shared_from_this +{ +public: + typedef boost::shared_ptr pointer; + +public: + /** Handles a RPC client request. + */ + class Handler + { + public: + virtual ~Handler () { } + + /** Construct a HTTP response string. + */ + virtual std::string createResponse (int statusCode, std::string const& description) = 0; + + /** Determine if the connection is authorized. + */ + virtual bool isAuthorized (std::map const& headers) = 0; + + /** Produce a response for a given request. + + @param request The RPC request string. + @return The server's response. + */ + virtual std::string processRequest (std::string const& request, std::string const& remoteAddress) = 0; + }; + + static pointer New ( + boost::asio::io_service& io_service, + boost::asio::ssl::context& context, + Handler& handler); + + /** Called when the connection is established. + */ + virtual void connected () = 0; + + // VFALCO TODO AutoSocket exposes all sorts of boost::asio interface + virtual AutoSocket& getSocket () = 0; + + // VFALCO TODO Remove this since it exposes boost + virtual boost::asio::ip::tcp::socket& getRawSocket () = 0; + + /** Retrieve the remote address as a string. + + @return A std::string representing the remote address. + */ + // VFALCO TODO Replace the return type with a dedicated class. + virtual std::string getRemoteAddressText () = 0; +}; + +#endif diff --git a/src/cpp/ripple/SNTPClient.cpp b/modules/ripple_net/basics/ripple_SNTPClient.cpp similarity index 100% rename from src/cpp/ripple/SNTPClient.cpp rename to modules/ripple_net/basics/ripple_SNTPClient.cpp diff --git a/src/cpp/ripple/SNTPClient.h b/modules/ripple_net/basics/ripple_SNTPClient.h similarity index 75% rename from src/cpp/ripple/SNTPClient.h rename to modules/ripple_net/basics/ripple_SNTPClient.h index 4344522348..87e03c31cc 100644 --- a/src/cpp/ripple/SNTPClient.h +++ b/modules/ripple_net/basics/ripple_SNTPClient.h @@ -4,8 +4,8 @@ */ //============================================================================== -#ifndef __SNTPCLIENT__ -#define __SNTPCLIENT__ +#ifndef RIPPLE_SNTPCLIENT_H_INCLUDED +#define RIPPLE_SNTPCLIENT_H_INCLUDED class SNTPQuery { @@ -20,19 +20,33 @@ public: } }; -class SNTPClient +//------------------------------------------------------------------------------ + +// VFALCO TODO Make an abstract interface for this to hide the boost +// +class SNTPClient : LeakChecked { public: - SNTPClient (boost::asio::io_service& service); - void init (const std::vector& servers); - void addServer (const std::string& mServer); + explicit SNTPClient (boost::asio::io_service& service); + + void init (std::vector const& servers); + + void addServer (std::string const& mServer); void queryAll (); bool doQuery (); bool getOffset (int& offset); private: - std::map mQueries; + void receivePacket (const boost::system::error_code& error, std::size_t bytes); + void resolveComplete (const boost::system::error_code& error, boost::asio::ip::udp::resolver::iterator iterator); + void sentPacket (boost::shared_ptr, const boost::system::error_code&, std::size_t); + void timerEntry (const boost::system::error_code&); + void sendComplete (const boost::system::error_code& error, std::size_t bytesTransferred); + void processReply (); + +private: + std::map mQueries; boost::mutex mLock; boost::asio::ip::udp::socket mSocket; @@ -47,14 +61,6 @@ private: std::vector mReceiveBuffer; boost::asio::ip::udp::endpoint mReceiveEndpoint; - - void receivePacket (const boost::system::error_code& error, std::size_t bytes); - void resolveComplete (const boost::system::error_code& error, boost::asio::ip::udp::resolver::iterator iterator); - void sentPacket (boost::shared_ptr, const boost::system::error_code&, std::size_t); - void timerEntry (const boost::system::error_code&); - void sendComplete (const boost::system::error_code& error, std::size_t bytesTransferred); - void processReply (); }; #endif -// vim:ts=4 diff --git a/modules/ripple_net/ripple_net.cpp b/modules/ripple_net/ripple_net.cpp new file mode 100644 index 0000000000..a2ed5d8037 --- /dev/null +++ b/modules/ripple_net/ripple_net.cpp @@ -0,0 +1,28 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +/** Add this to get the @ref ripple_net module. + + @file ripple_net.cpp + @ingroup ripple_net +*/ + +#include "BeastConfig.h" + +#include "ripple_net.h" + +// VFALCO TODO Remove this dependency on theConfig +#include "../modules/ripple_core/ripple_core.h" // theConfig for HttpsClient + +namespace ripple +{ + +#include "basics/ripple_HTTPRequest.cpp" +#include "basics/ripple_HttpsClient.cpp" +#include "basics/ripple_RPCServer.cpp" +#include "basics/ripple_SNTPClient.cpp" + +} diff --git a/modules/ripple_net/ripple_net.h b/modules/ripple_net/ripple_net.h new file mode 100644 index 0000000000..2452be7f9d --- /dev/null +++ b/modules/ripple_net/ripple_net.h @@ -0,0 +1,37 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NET_H_INCLUDED +#define RIPPLE_NET_H_INCLUDED + +/** Include this to get the @ref ripple_net module. + + @file ripple_net.h + @ingroup ripple_net +*/ + +/** Network classes. + + This module provides classes that handle all network activities. + + @defgroup ripple_net +*/ + +#include "../ripple_basics/ripple_basics.h" + +#include "../ripple_websocket/ripple_websocket.h" + +namespace ripple +{ + +#include "basics/ripple_HTTPRequest.h" +#include "basics/ripple_HttpsClient.h" +#include "basics/ripple_RPCServer.h" +#include "basics/ripple_SNTPClient.h" + +} + +#endif diff --git a/modules/ripple_sqlite/ripple_sqlite.c b/modules/ripple_sqlite/ripple_sqlite.c index 6da2c4012c..ae8c03ab18 100644 --- a/modules/ripple_sqlite/ripple_sqlite.c +++ b/modules/ripple_sqlite/ripple_sqlite.c @@ -10,6 +10,8 @@ @ingroup ripple_sqlite */ +#include "BeastConfig.h" + // This prevents sqlite.h from being included // #define RIPPLE_SQLITE_MODULE_INCLUDED 1 diff --git a/modules/ripple_websocket/autosocket/ripple_AutoSocket.cpp b/modules/ripple_websocket/autosocket/ripple_AutoSocket.cpp index b9d5415fee..84e2714fff 100644 --- a/modules/ripple_websocket/autosocket/ripple_AutoSocket.cpp +++ b/modules/ripple_websocket/autosocket/ripple_AutoSocket.cpp @@ -4,8 +4,4 @@ */ //============================================================================== -#if RIPPLE_USE_NAMESPACE ripple::LogPartition AutoSocket::AutoSocketPartition ("AutoSocket"); -#else -LogPartition AutoSocket::AutoSocketPartition ("AutoSocket"); -#endif diff --git a/modules/ripple_websocket/autosocket/ripple_AutoSocket.h b/modules/ripple_websocket/autosocket/ripple_AutoSocket.h index 30b20116ff..a64b326c6e 100644 --- a/modules/ripple_websocket/autosocket/ripple_AutoSocket.h +++ b/modules/ripple_websocket/autosocket/ripple_AutoSocket.h @@ -12,13 +12,10 @@ // To force a non-SSL connection, just don't call async_handshake. // To force SSL only inbound, call setSSLOnly. -namespace basio = boost::asio; -namespace bassl = basio::ssl; - class AutoSocket { public: - typedef bassl::stream ssl_socket; + typedef boost::asio::ssl::stream ssl_socket; typedef boost::shared_ptr socket_ptr; typedef ssl_socket::next_layer_type plain_socket; typedef ssl_socket::lowest_layer_type lowest_layer_type; @@ -27,12 +24,12 @@ public: typedef boost::function callback; public: - AutoSocket (basio::io_service& s, bassl::context& c) : mSecure (false), mBuffer (4) + AutoSocket (boost::asio::io_service& s, boost::asio::ssl::context& c) : mSecure (false), mBuffer (4) { mSocket = boost::make_shared (boost::ref (s), boost::ref (c)); } - AutoSocket (basio::io_service& s, bassl::context& c, bool secureOnly, bool plainOnly) + AutoSocket (boost::asio::io_service& s, boost::asio::ssl::context& c, bool secureOnly, bool plainOnly) : mSecure (secureOnly), mBuffer ((plainOnly || secureOnly) ? 0 : 4) { mSocket = boost::make_shared (boost::ref (s), boost::ref (c)); @@ -73,9 +70,7 @@ public: static bool rfc2818_verify (const std::string& domain, bool preverified, boost::asio::ssl::verify_context& ctx) { -#if RIPPLE_USE_NAMESPACE using namespace ripple; -#endif if (boost::asio::ssl::rfc2818_verification (domain) (preverified, ctx)) return true; @@ -92,7 +87,7 @@ public: mSocket->set_verify_mode (boost::asio::ssl::verify_peer); // XXX Verify semantics of RFC 2818 are what we want. - mSocket->set_verify_callback (boost::bind (&rfc2818_verify, strDomain, _1, _2), ec); + mSocket->set_verify_callback (boost::bind (&rfc2818_verify, strDomain, boost::placeholders::_1, boost::placeholders::_2), ec); return ec; } @@ -114,13 +109,14 @@ public: else { // autodetect - mSocket->next_layer ().async_receive (basio::buffer (mBuffer), basio::socket_base::message_peek, + mSocket->next_layer ().async_receive (boost::asio::buffer (mBuffer), boost::asio::socket_base::message_peek, boost::bind (&AutoSocket::handle_autodetect, this, cbFunc, - basio::placeholders::error, basio::placeholders::bytes_transferred)); + boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); } } - template void async_shutdown (ShutdownHandler handler) + template + void async_shutdown (ShutdownHandler handler) { if (isSecure ()) mSocket->async_shutdown (handler); @@ -139,7 +135,8 @@ public: } } - template void async_read_some (const Seq& buffers, Handler handler) + template + void async_read_some (const Seq& buffers, Handler handler) { if (isSecure ()) mSocket->async_read_some (buffers, handler); @@ -151,30 +148,31 @@ public: void async_read_until (const Seq& buffers, Condition condition, Handler handler) { if (isSecure ()) - basio::async_read_until (*mSocket, buffers, condition, handler); + boost::asio::async_read_until (*mSocket, buffers, condition, handler); else - basio::async_read_until (PlainSocket (), buffers, condition, handler); + boost::asio::async_read_until (PlainSocket (), buffers, condition, handler); } template - void async_read_until (basio::basic_streambuf& buffers, const std::string& delim, Handler handler) + void async_read_until (boost::asio::basic_streambuf& buffers, const std::string& delim, Handler handler) { if (isSecure ()) - basio::async_read_until (*mSocket, buffers, delim, handler); + boost::asio::async_read_until (*mSocket, buffers, delim, handler); else - basio::async_read_until (PlainSocket (), buffers, delim, handler); + boost::asio::async_read_until (PlainSocket (), buffers, delim, handler); } template - void async_read_until (basio::basic_streambuf& buffers, MatchCondition cond, Handler handler) + void async_read_until (boost::asio::basic_streambuf& buffers, MatchCondition cond, Handler handler) { if (isSecure ()) - basio::async_read_until (*mSocket, buffers, cond, handler); + boost::asio::async_read_until (*mSocket, buffers, cond, handler); else - basio::async_read_until (PlainSocket (), buffers, cond, handler); + boost::asio::async_read_until (PlainSocket (), buffers, cond, handler); } - template void async_write (const Buf& buffers, Handler handler) + template + void async_write (const Buf& buffers, Handler handler) { if (isSecure ()) boost::asio::async_write (*mSocket, buffers, handler); @@ -201,7 +199,7 @@ public: } template - void async_read (basio::basic_streambuf& buffers, Condition cond, Handler handler) + void async_read (boost::asio::basic_streambuf& buffers, Condition cond, Handler handler) { if (isSecure ()) boost::asio::async_read (*mSocket, buffers, cond, handler); @@ -209,7 +207,8 @@ public: boost::asio::async_read (PlainSocket (), buffers, cond, handler); } - template void async_read (const Buf& buffers, Handler handler) + template + void async_read (const Buf& buffers, Handler handler) { if (isSecure ()) boost::asio::async_read (*mSocket, buffers, handler); @@ -217,7 +216,8 @@ public: boost::asio::async_read (PlainSocket (), buffers, handler); } - template void async_write_some (const Seq& buffers, Handler handler) + template + void async_write_some (const Seq& buffers, Handler handler) { if (isSecure ()) mSocket->async_write_some (buffers, handler); @@ -228,9 +228,7 @@ public: protected: void handle_autodetect (callback cbFunc, const error_code& ec, size_t bytesTransferred) { -#if RIPPLE_USE_NAMESPACE using namespace ripple; -#endif if (ec) { @@ -261,11 +259,7 @@ protected: } private: -#if RIPPLE_USE_NAMESPACE static ripple::LogPartition AutoSocketPartition; -#else - static LogPartition AutoSocketPartition; -#endif socket_ptr mSocket; bool mSecure; @@ -273,5 +267,3 @@ private: }; #endif - -// vim:ts=4 diff --git a/modules/ripple_websocket/autosocket/ripple_LogWebsockets.cpp b/modules/ripple_websocket/autosocket/ripple_LogWebsockets.cpp index f917dfd58f..1e45ee75cf 100644 --- a/modules/ripple_websocket/autosocket/ripple_LogWebsockets.cpp +++ b/modules/ripple_websocket/autosocket/ripple_LogWebsockets.cpp @@ -12,7 +12,6 @@ namespace websocketpp namespace log { -#if RIPPLE_USE_NAMESPACE using namespace ripple; LogPartition websocketPartition ("WebSocket"); @@ -48,37 +47,6 @@ void websocketLog (websocketpp::log::elevel::value v, const std::string& entry) if (websocketPartition.doLog (s)) Log (s, websocketPartition) << entry; } -#else -LogPartition websocketPartition ("WebSocket"); - -void websocketLog (websocketpp::log::alevel::value v, const std::string& entry) -{ - if ((v == websocketpp::log::alevel::DEVEL) || (v == websocketpp::log::alevel::DEBUG_CLOSE)) - { - if (websocketPartition.doLog (lsTRACE)) - Log (lsDEBUG, websocketPartition) << entry; - } - else if (websocketPartition.doLog (lsDEBUG)) - Log (lsDEBUG, websocketPartition) << entry; -} - -void websocketLog (websocketpp::log::elevel::value v, const std::string& entry) -{ - LogSeverity s = lsDEBUG; - - if ((v & websocketpp::log::elevel::INFO) != 0) - s = lsINFO; - else if ((v & websocketpp::log::elevel::FATAL) != 0) - s = lsFATAL; - else if ((v & websocketpp::log::elevel::RERROR) != 0) - s = lsERROR; - else if ((v & websocketpp::log::elevel::WARN) != 0) - s = lsWARNING; - - if (websocketPartition.doLog (s)) - Log (s, websocketPartition) << entry; -} -#endif } } diff --git a/modules/ripple_websocket/ripple_websocket.cpp b/modules/ripple_websocket/ripple_websocket.cpp index 4785cb9650..402f4c81f3 100644 --- a/modules/ripple_websocket/ripple_websocket.cpp +++ b/modules/ripple_websocket/ripple_websocket.cpp @@ -4,6 +4,8 @@ */ //============================================================================== +#include "BeastConfig.h" + #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS 1 #endif diff --git a/modules/ripple_websocket/ripple_websocket.h b/modules/ripple_websocket/ripple_websocket.h index 0e3b7fb1b1..f9d4a2ac47 100644 --- a/modules/ripple_websocket/ripple_websocket.h +++ b/modules/ripple_websocket/ripple_websocket.h @@ -21,6 +21,10 @@ @deprecated */ +// needed before inclusion of stdint.h for INT32_MIN/INT32_MAX macros +#ifndef __STDC_LIMIT_MACROS + #define __STDC_LIMIT_MACROS +#endif // VFALCO NOTE Log dependencies have wormed their way into websocketpp, // which needs the ripple_basic module to compile. @@ -30,19 +34,8 @@ // #include "../modules/ripple_basics/ripple_basics.h" -#include - -#include -#include -#include - //------------------------------------------------------------------------------ -// VFALCO TODO This include is just to prevent a warning about -// redefinition of __STDC_LIMIT_MACROS. Fix it right. -// -#include "websocket/src/rng/boost_rng.hpp" - #include "websocket/src/common.hpp" #include "websocket/src/sockets/socket_base.hpp" diff --git a/package.json b/package.json index 17d814811d..3009ee8227 100644 --- a/package.json +++ b/package.json @@ -10,22 +10,23 @@ }, "dependencies": { - "ripple-lib": "0.7.10", + "ripple-lib": "0.7.18", "async": "~0.1.22", "extend": "~1.1.1", "simple-jsonrpc": "~0.0.1" }, "devDependencies": { - "buster": "~0.6.12" + "buster": "~0.6.12", + "mocha": "~1.12.0" }, "scripts": { - "test": "node_modules/buster/bin/buster test" + "test": "./node_modules/buster/bin/buster-test" }, "repository": { "type": "git", - "url": "git://github.com/jedmccaleb/NewCoin.git" + "url": "git://github.com/ripple/rippled.git" }, "readmeFilename": "README.md" diff --git a/ripple-example.txt b/ripple-example.txt index 32424fe8a6..769d11f745 100644 --- a/ripple-example.txt +++ b/ripple-example.txt @@ -20,6 +20,9 @@ # Undefined sections are reserved. # No escapes are currently defined. # +# IMPORTANT: Please remove these comments before publishing this file. Doing so +# makes the file much more readable to humans trying to find info on your site. +# # [expires] # Number of days after which this file should be considered expire. Clients # are expected to check this file when they have cause to believe it has @@ -102,6 +105,11 @@ # [currencies]: # This section allows a site to declare currencies it currently issues. # +# Examples: (multiple allowed one per line) +# USD +# BTC +# LTC +# [validation_public_key] n9MZTnHe5D5Q2cgE8oV2usFwRqhUvEA8MwP5Mu1XVD6TxmssPRev diff --git a/rippled-example.cfg b/rippled-example.cfg index e56f90f4fd..13ee8cf3ef 100644 --- a/rippled-example.cfg +++ b/rippled-example.cfg @@ -223,10 +223,9 @@ # shfArahZT9Q9ckTf3s1psJ7C7qzVN # # [node_db] -# Sets the database used for hashed nodes. Options are "LevelDB" and -# "SQLite". LevelDB is recommended for optimum performance and SQLite -# support will soon be removed. Databases can be migrated by enabling -# "LevelDB" and starting rippled with the "--import" flag +# Sets the database back end used for hashed nodes and the parameters +# for that back end, separated by pipe characters. For LevelDB, +# use "type=LevelDB|Path=db/hashnode" for MDB, use "type=mdb|path=db". # # [node_size] # Tunes the servers based on the expected load and available memory. Legal @@ -298,6 +297,12 @@ [rpc_allow_remote] 0 +[node_size] +medium + +[node_db] +type=mdb|path=db + [debug_logfile] log/debug.log diff --git a/src/cpp/ripple/AccountSetTransactor.cpp b/src/cpp/ripple/AccountSetTransactor.cpp index a1e72df13b..39697ec988 100644 --- a/src/cpp/ripple/AccountSetTransactor.cpp +++ b/src/cpp/ripple/AccountSetTransactor.cpp @@ -50,7 +50,7 @@ TER AccountSetTransactor::doApply () { WriteLog (lsINFO, AccountSetTransactor) << "AccountSet: Retry: OwnerCount not zero."; - return terOWNERS; + return isSetBit(mParams, tapRETRY) ? terOWNERS : tecOWNERS; } WriteLog (lsINFO, AccountSetTransactor) << "AccountSet: Set RequireAuth."; diff --git a/src/cpp/ripple/CallRPC.cpp b/src/cpp/ripple/CallRPC.cpp index d00594638a..01c3a15782 100644 --- a/src/cpp/ripple/CallRPC.cpp +++ b/src/cpp/ripple/CallRPC.cpp @@ -878,12 +878,13 @@ int commandLineRPC (const std::vector& vCmd) callRPC ( isService, - theConfig.RPC_IP, theConfig.RPC_PORT, - theConfig.RPC_USER, theConfig.RPC_PASSWORD, + theConfig.getRpcIP (), + theConfig.getRpcPort (), + theConfig.RPC_USER, + theConfig.RPC_PASSWORD, "", jvRequest.isMember ("method") // Allow parser to rewrite method. - ? jvRequest["method"].asString () - : vCmd[0], + ? jvRequest["method"].asString () : vCmd[0], jvParams, // Parsed, execute. false, BIND_TYPE (callRPCHandler, &jvOutput, P_1)); @@ -1014,10 +1015,10 @@ void callRPC ( // Connect to localhost if (!theConfig.QUIET) { - std::cerr << "Connecting to: " << strIp << ":" << iPort << std::endl; - // std::cerr << "Username: " << strUsername << ":" << strPassword << std::endl; - // std::cerr << "Path: " << strPath << std::endl; - // std::cerr << "Method: " << strMethod << std::endl; + Log::out() << "Connecting to: " << strIp << ":" << iPort; + // Log::out() << "Username: " << strUsername << ":" << strPassword; + // Log::out() << "Path: " << strPath; + // Log::out() << "Method: " << strMethod; } // HTTP basic authentication diff --git a/src/cpp/ripple/CallRPC.h b/src/cpp/ripple/CallRPC.h index 3bfefe7d03..bb0d2cf1e6 100644 --- a/src/cpp/ripple/CallRPC.h +++ b/src/cpp/ripple/CallRPC.h @@ -66,7 +66,7 @@ extern void callRPC ( const std::string& strUsername, const std::string& strPassword, const std::string& strPath, const std::string& strMethod, const Json::Value& jvParams, const bool bSSL, - FUNCTION_TYPE callbackFuncP = 0); + FUNCTION_TYPE callbackFuncP = FUNCTION_TYPE ()); #endif // vim:ts=4 diff --git a/src/cpp/ripple/ChangeTransactor.cpp b/src/cpp/ripple/ChangeTransactor.cpp index 16514708cd..091de0d5dd 100644 --- a/src/cpp/ripple/ChangeTransactor.cpp +++ b/src/cpp/ripple/ChangeTransactor.cpp @@ -94,10 +94,10 @@ TER ChangeTransactor::applyFeature () featureObject->setFieldV256 (sfFeatures, features); mEngine->entryModify (featureObject); - theApp->getFeatureTable ().enableFeature (feature); + getApp().getFeatureTable ().enableFeature (feature); - if (!theApp->getFeatureTable ().isFeatureSupported (feature)) - theApp->getOPs ().setFeatureBlocked (); + if (!getApp().getFeatureTable ().isFeatureSupported (feature)) + getApp().getOPs ().setFeatureBlocked (); return tesSUCCESS; } diff --git a/src/cpp/ripple/Ledger.cpp b/src/cpp/ripple/Ledger.cpp index 7880829ac2..4b0300a554 100644 --- a/src/cpp/ripple/Ledger.cpp +++ b/src/cpp/ripple/Ledger.cpp @@ -8,7 +8,7 @@ SETUP_LOG (Ledger) Ledger::Ledger (const RippleAddress& masterID, uint64 startAmount) : mTotCoins (startAmount) - , mLedgerSeq (1) + , mLedgerSeq (1) // First Ledger , mCloseTime (0) , mParentCloseTime (0) , mCloseResolution (LEDGER_TIME_ACCURACY) @@ -135,7 +135,7 @@ Ledger::Ledger (bool /* dummy */, if (prevLedger.mCloseTime == 0) { - mCloseTime = roundCloseTime (theApp->getOPs ().getCloseTimeNC (), mCloseResolution); + mCloseTime = roundCloseTime (getApp().getOPs ().getCloseTimeNC (), mCloseResolution); } else { @@ -194,6 +194,7 @@ void Ledger::updateHash () mAccountHash.zero (); } + // VFALCO TODO Fix this hard coded magic number 118 Serializer s (118); s.add32 (HashPrefix::ledgerMaster); addRaw (s); @@ -205,7 +206,8 @@ void Ledger::setRaw (Serializer& s, bool hasPrefix) { SerializerIterator sit (s); - if (hasPrefix) sit.get32 (); + if (hasPrefix) + sit.get32 (); mLedgerSeq = sit.get32 (); mTotCoins = sit.get64 (); @@ -267,8 +269,8 @@ bool Ledger::hasAccount (const RippleAddress& accountID) AccountState::pointer Ledger::getAccountState (const RippleAddress& accountID) { -#ifdef DEBUG - // std::cerr << "Ledger:getAccountState(" << accountID.humanAccountID() << ")" << std::endl; +#ifdef BEAST_DEBUG + // Log::out() << "Ledger:getAccountState(" << accountID.humanAccountID() << ")"; #endif SLE::pointer sle = getSLEi (Ledger::getAccountRootIndex (accountID)); @@ -345,7 +347,7 @@ Transaction::pointer Ledger::getTransaction (uint256 const& transID) const if (!item) return Transaction::pointer (); - Transaction::pointer txn = theApp->getMasterTransaction ().fetch (transID, false); + Transaction::pointer txn = getApp().getMasterTransaction ().fetch (transID, false); if (txn) return txn; @@ -371,7 +373,7 @@ Transaction::pointer Ledger::getTransaction (uint256 const& transID) const if (txn->getStatus () == NEW) txn->setStatus (mClosed ? COMMITTED : INCLUDED, mLedgerSeq); - theApp->getMasterTransaction ().canonicalize (txn, false); + getApp().getMasterTransaction ().canonicalize (txn); return txn; } @@ -425,7 +427,7 @@ bool Ledger::getTransaction (uint256 const& txID, Transaction::pointer& txn, Tra if (type == SHAMapTreeNode::tnTRANSACTION_NM) { // in tree with no metadata - txn = theApp->getMasterTransaction ().fetch (txID, false); + txn = getApp().getMasterTransaction ().fetch (txID, false); meta.reset (); if (!txn) @@ -435,7 +437,7 @@ bool Ledger::getTransaction (uint256 const& txID, Transaction::pointer& txn, Tra { // in tree with metadata SerializerIterator it (item->peekSerializer ()); - txn = theApp->getMasterTransaction ().fetch (txID, false); + txn = getApp().getMasterTransaction ().fetch (txID, false); if (!txn) txn = Transaction::sharedTransaction (it.getVL (), true); @@ -450,7 +452,7 @@ bool Ledger::getTransaction (uint256 const& txID, Transaction::pointer& txn, Tra if (txn->getStatus () == NEW) txn->setStatus (mClosed ? COMMITTED : INCLUDED, mLedgerSeq); - theApp->getMasterTransaction ().canonicalize (txn, false); + getApp().getMasterTransaction ().canonicalize (txn); return true; } @@ -531,18 +533,18 @@ void Ledger::saveAcceptedLedger (Job&, bool fromConsensus) Serializer s (128); s.add32 (HashPrefix::ledgerMaster); addRaw (s); - theApp->getHashedObjectStore ().store (hotLEDGER, mLedgerSeq, s.peekData (), mHash); + getApp().getNodeStore ().store (hotLEDGER, mLedgerSeq, s.peekData (), mHash); AcceptedLedger::pointer aLedger = AcceptedLedger::makeAcceptedLedger (shared_from_this ()); { - ScopedLock sl (theApp->getLedgerDB ()->getDBLock ()); - theApp->getLedgerDB ()->getDB ()->executeSQL (boost::str (deleteLedger % mLedgerSeq)); + ScopedLock sl (getApp().getLedgerDB ()->getDBLock ()); + getApp().getLedgerDB ()->getDB ()->executeSQL (boost::str (deleteLedger % mLedgerSeq)); } { - Database* db = theApp->getTxnDB ()->getDB (); - ScopedLock dbLock (theApp->getTxnDB ()->getDBLock ()); + Database* db = getApp().getTxnDB ()->getDB (); + ScopedLock dbLock (getApp().getTxnDB ()->getDBLock ()); db->executeSQL ("BEGIN TRANSACTION;"); db->executeSQL (boost::str (deleteTrans1 % mLedgerSeq)); @@ -551,7 +553,7 @@ void Ledger::saveAcceptedLedger (Job&, bool fromConsensus) BOOST_FOREACH (const AcceptedLedger::value_type & vt, aLedger->getMap ()) { uint256 txID = vt.second->getTransactionID (); - theApp->getMasterTransaction ().inLedger (txID, mLedgerSeq); + getApp().getMasterTransaction ().inLedger (txID, mLedgerSeq); db->executeSQL (boost::str (deleteAcctTrans % txID.GetHex ())); @@ -596,8 +598,9 @@ void Ledger::saveAcceptedLedger (Job&, bool fromConsensus) } { - ScopedLock sl (theApp->getLedgerDB ()->getDBLock ()); - theApp->getLedgerDB ()->getDB ()->executeSQL (boost::str (addLedger % + ScopedLock sl (getApp().getLedgerDB ()->getDBLock ()); + + getApp().getLedgerDB ()->getDB ()->executeSQL (boost::str (addLedger % getHash ().GetHex () % mLedgerSeq % mParentHash.GetHex () % boost::lexical_cast (mTotCoins) % mCloseTime % mParentCloseTime % mCloseResolution % mCloseFlags % mAccountHash.GetHex () % mTransHash.GetHex ())); @@ -606,8 +609,8 @@ void Ledger::saveAcceptedLedger (Job&, bool fromConsensus) if (!fromConsensus && (theConfig.NODE_SIZE < 2)) // tiny or small dropCache (); - if (theApp->getJobQueue ().getJobCountTotal (jtPUBOLDLEDGER) < 2) - theApp->getLedgerMaster ().resumeAcquiring (); + if (getApp().getJobQueue ().getJobCountTotal (jtPUBOLDLEDGER) < 2) + getApp().getLedgerMaster ().resumeAcquiring (); else WriteLog (lsTRACE, Ledger) << "no resume, too many pending ledger saves"; } @@ -618,8 +621,8 @@ Ledger::pointer Ledger::loadByIndex (uint32 ledgerIndex) { Ledger::pointer ledger; { - Database* db = theApp->getLedgerDB ()->getDB (); - ScopedLock sl (theApp->getLedgerDB ()->getDBLock ()); + Database* db = getApp().getLedgerDB ()->getDB (); + ScopedLock sl (getApp().getLedgerDB ()->getDBLock ()); SqliteStatement pSt (db->getSqliteDB (), "SELECT " "LedgerHash,PrevHash,AccountSetHash,TransSetHash,TotalCoins," @@ -643,8 +646,8 @@ Ledger::pointer Ledger::loadByHash (uint256 const& ledgerHash) { Ledger::pointer ledger; { - Database* db = theApp->getLedgerDB ()->getDB (); - ScopedLock sl (theApp->getLedgerDB ()->getDBLock ()); + Database* db = getApp().getLedgerDB ()->getDB (); + ScopedLock sl (getApp().getLedgerDB ()->getDBLock ()); SqliteStatement pSt (db->getSqliteDB (), "SELECT " "LedgerHash,PrevHash,AccountSetHash,TransSetHash,TotalCoins," @@ -698,8 +701,8 @@ Ledger::pointer Ledger::getSQL (const std::string& sql) std::string hash; { - Database* db = theApp->getLedgerDB ()->getDB (); - ScopedLock sl (theApp->getLedgerDB ()->getDBLock ()); + Database* db = getApp().getLedgerDB ()->getDB (); + ScopedLock sl (getApp().getLedgerDB ()->getDBLock ()); if (!db->executeSQL (sql) || !db->startIterRows ()) return Ledger::pointer (); @@ -731,7 +734,7 @@ Ledger::pointer Ledger::getSQL (const std::string& sql) ret->setClosed (); - if (theApp->getOPs ().haveLedger (ledgerSeq)) + if (getApp().getOPs ().haveLedger (ledgerSeq)) ret->setAccepted (); if (ret->getHash () != ledgerHash) @@ -798,7 +801,7 @@ void Ledger::getSQL2 (Ledger::ref ret) ret->setClosed (); ret->setImmutable (); - if (theApp->getOPs ().haveLedger (ret->getLedgerSeq ())) + if (getApp().getOPs ().haveLedger (ret->getLedgerSeq ())) ret->setAccepted (); WriteLog (lsTRACE, Ledger) << "Loaded ledger: " << ret->getHash ().GetHex (); @@ -814,8 +817,8 @@ uint256 Ledger::getHashByIndex (uint32 ledgerIndex) std::string hash; { - Database* db = theApp->getLedgerDB ()->getDB (); - ScopedLock sl (theApp->getLedgerDB ()->getDBLock ()); + Database* db = getApp().getLedgerDB ()->getDB (); + ScopedLock sl (getApp().getLedgerDB ()->getDBLock ()); if (!db->executeSQL (sql) || !db->startIterRows ()) return ret; @@ -832,7 +835,7 @@ bool Ledger::getHashesByIndex (uint32 ledgerIndex, uint256& ledgerHash, uint256& { #ifndef NO_SQLITE3_PREPARE - DatabaseCon* con = theApp->getLedgerDB (); + DatabaseCon* con = getApp().getLedgerDB (); ScopedLock sl (con->getDBLock ()); SqliteStatement pSt (con->getDB ()->getSqliteDB (), @@ -868,8 +871,8 @@ bool Ledger::getHashesByIndex (uint32 ledgerIndex, uint256& ledgerHash, uint256& std::string hash, prevHash; { - Database* db = theApp->getLedgerDB ()->getDB (); - ScopedLock sl (theApp->getLedgerDB ()->getDBLock ()); + Database* db = getApp().getLedgerDB ()->getDB (); + ScopedLock sl (getApp().getLedgerDB ()->getDBLock ()); if (!db->executeSQL (sql) || !db->startIterRows ()) return false; @@ -891,39 +894,27 @@ bool Ledger::getHashesByIndex (uint32 ledgerIndex, uint256& ledgerHash, uint256& std::map< uint32, std::pair > Ledger::getHashesByIndex (uint32 minSeq, uint32 maxSeq) { -#ifndef NO_SQLITE_PREPARE std::map< uint32, std::pair > ret; - DatabaseCon* con = theApp->getLedgerDB (); + + std::string sql = "SELECT LedgerSeq,LedgerHash,PrevHash FROM Ledgers WHERE LedgerSeq >= "; + sql.append (boost::lexical_cast (minSeq)); + sql.append (" AND LedgerSeq <= "); + sql.append (boost::lexical_cast (maxSeq)); + sql.append (";"); + + DatabaseCon* con = getApp().getLedgerDB (); ScopedLock sl (con->getDBLock ()); - SqliteStatement pSt (con->getDB ()->getSqliteDB (), - "SELECT LedgerSeq,LedgerHash,PrevHash FROM Ledgers INDEXED BY SeqLedger " - "WHERE LedgerSeq >= ? AND LedgerSeq <= ?;"); + SqliteStatement pSt (con->getDB ()->getSqliteDB (), sql); - std::pair hashes; - - pSt.bind (1, minSeq); - pSt.bind (2, maxSeq); - - do + while (pSt.isRow (pSt.step ())) { - int r = pSt.step (); - - if (pSt.isDone (r)) - return ret; - - if (!pSt.isRow (r)) - return ret; - + std::pair& hashes = ret[pSt.getUInt32 (0)]; hashes.first.SetHexExact (pSt.peekString (1)); hashes.second.SetHexExact (pSt.peekString (2)); - ret[pSt.getUInt32 (0)] = hashes; } - while (1); -#else -#error SQLite prepare is required -#endif + return ret; } Ledger::pointer Ledger::getLastFullLedger () @@ -1150,13 +1141,13 @@ SLE::pointer Ledger::getSLEi (uint256 const& uId) if (!node) return SLE::pointer (); - SLE::pointer ret = theApp->getSLECache ().fetch (hash); + SLE::pointer ret = getApp().getSLECache ().fetch (hash); if (!ret) { ret = boost::make_shared (node->peekSerializer (), node->getTag ()); ret->setImmutable (); - theApp->getSLECache ().canonicalize (hash, ret); + getApp().getSLECache ().canonicalize (hash, ret); } return ret; @@ -1488,7 +1479,10 @@ uint256 Ledger::getLedgerHash (uint32 ledgerIndex) WriteLog (lsWARNING, Ledger) << "Ledger " << mLedgerSeq << " missing hash for " << ledgerIndex << " (" << vec.size () << "," << diff << ")"; } - else WriteLog (lsWARNING, Ledger) << "Ledger " << mLedgerSeq << ":" << getHash () << " missing normal list"; + else + { + WriteLog (lsWARNING, Ledger) << "Ledger " << mLedgerSeq << ":" << getHash () << " missing normal list"; + } } if ((ledgerIndex & 0xff) != 0) @@ -1811,12 +1805,12 @@ uint32 Ledger::roundCloseTime (uint32 closeTime, uint32 closeResolution) void Ledger::pendSave (bool fromConsensus) { - if (!fromConsensus && !theApp->getHashRouter ().setFlag (getHash (), SF_SAVED)) + if (!fromConsensus && !getApp().getHashRouter ().setFlag (getHash (), SF_SAVED)) return; assert (isImmutable ()); - theApp->getJobQueue ().addJob ( + getApp().getJobQueue ().addJob ( fromConsensus ? jtPUBLEDGER : jtPUBOLDLEDGER, fromConsensus ? "Ledger::pendSave" : "Ledger::pendOldSave", BIND_TYPE (&Ledger::saveAcceptedLedger, shared_from_this (), P_1, fromConsensus)); @@ -1879,7 +1873,7 @@ uint64 Ledger::scaleFeeBase (uint64 fee) if (!mBaseFee) updateFees (); - return theApp->getFeeTrack ().scaleFeeBase (fee, mBaseFee, mReferenceFeeUnits); + return getApp().getFeeTrack ().scaleFeeBase (fee, mBaseFee, mReferenceFeeUnits); } uint64 Ledger::scaleFeeLoad (uint64 fee, bool bAdmin) @@ -1887,7 +1881,7 @@ uint64 Ledger::scaleFeeLoad (uint64 fee, bool bAdmin) if (!mBaseFee) updateFees (); - return theApp->getFeeTrack ().scaleFeeLoad (fee, mBaseFee, mReferenceFeeUnits, bAdmin); + return getApp().getFeeTrack ().scaleFeeLoad (fee, mBaseFee, mReferenceFeeUnits, bAdmin); } std::vector Ledger::getNeededTransactionHashes (int max, SHAMapSyncFilter* filter) @@ -1919,16 +1913,3 @@ std::vector Ledger::getNeededAccountStateHashes (int max, SHAMapSyncFil return ret; } - -BOOST_AUTO_TEST_SUITE (quality) - -BOOST_AUTO_TEST_CASE ( getquality ) -{ - uint256 uBig ("D2DC44E5DC189318DB36EF87D2104CDF0A0FE3A4B698BEEE55038D7EA4C68000"); - - if (6125895493223874560 != Ledger::getQuality (uBig)) - BOOST_FAIL ("Ledger::getQuality fails."); -} - -BOOST_AUTO_TEST_SUITE_END () -// vim:ts=4 diff --git a/src/cpp/ripple/Ledger.h b/src/cpp/ripple/Ledger.h index f39c03762c..6d0ddc686a 100644 --- a/src/cpp/ripple/Ledger.h +++ b/src/cpp/ripple/Ledger.h @@ -39,8 +39,11 @@ class SqliteStatement; class Ledger : public boost::enable_shared_from_this , public CountedObject + , Uncopyable { public: + static char const* getCountedObjectName () { return "Ledger"; } + typedef boost::shared_ptr pointer; typedef const boost::shared_ptr& ref; @@ -70,6 +73,7 @@ public: uint32 ledgerSeq, bool & loaded); // used for database ledgers Ledger (Blob const & rawLedger, bool hasPrefix); + Ledger (const std::string & rawLedger, bool hasPrefix); Ledger (bool dummy, Ledger & previous); // ledger after this one @@ -229,6 +233,9 @@ public: // next/prev function SLE::pointer getSLE (uint256 const & uHash); // SLE is mutable SLE::pointer getSLEi (uint256 const & uHash); // SLE is immutable + + // VFALCO NOTE These seem to let you walk the list of ledgers + // uint256 getFirstLedgerIndex (); uint256 getLastLedgerIndex (); uint256 getNextLedgerIndex (uint256 const & uHash); // first node >hash @@ -452,10 +459,6 @@ private: SHAMap::pointer mAccountStateMap; mutable boost::recursive_mutex mLock; - - // VFALCO TODO derive this from beast::Uncopyable - Ledger (const Ledger&); // no implementation - Ledger& operator= (const Ledger&); // no implementation }; inline LedgerStateParms operator| (const LedgerStateParms& l1, const LedgerStateParms& l2) diff --git a/src/cpp/ripple/LedgerMaster.cpp b/src/cpp/ripple/LedgerMaster.cpp index 359fa246f2..b38aa654a4 100644 --- a/src/cpp/ripple/LedgerMaster.cpp +++ b/src/cpp/ripple/LedgerMaster.cpp @@ -23,6 +23,23 @@ Ledger::ref LedgerMaster::getCurrentSnapshot () return mCurrentSnapshot; } +int LedgerMaster::getPublishedLedgerAge () +{ + boost::recursive_mutex::scoped_lock ml (mLock); + if (!mPubLedger) + { + WriteLog (lsDEBUG, LedgerMaster) << "No published ledger"; + return 999999; + } + + int64 ret = getApp().getOPs ().getCloseTimeNC (); + ret -= static_cast (mPubLedger->getCloseTimeNC ()); + ret = std::max (0LL, ret); + + WriteLog (lsTRACE, LedgerMaster) << "Published ledger age is " << ret; + return static_cast (ret); +} + int LedgerMaster::getValidatedLedgerAge () { boost::recursive_mutex::scoped_lock ml (mLock); @@ -32,7 +49,7 @@ int LedgerMaster::getValidatedLedgerAge () return 999999; } - int64 ret = theApp->getOPs ().getCloseTimeNC (); + int64 ret = getApp().getOPs ().getCloseTimeNC (); ret -= static_cast (mValidLedger->getCloseTimeNC ()); ret = std::max (0LL, ret); @@ -42,7 +59,7 @@ int LedgerMaster::getValidatedLedgerAge () bool LedgerMaster::isCaughtUp(std::string& reason) { - if (getValidatedLedgerAge() > 180) + if (getPublishedLedgerAge() > 180) { reason = "No recently-validated ledger"; return false; @@ -73,6 +90,7 @@ void LedgerMaster::pushLedger (Ledger::pointer newLedger) // Caller should already have properly assembled this ledger into "ready-to-close" form -- // all candidate transactions must already be applied WriteLog (lsINFO, LedgerMaster) << "PushLedger: " << newLedger->getHash (); + boost::recursive_mutex::scoped_lock ml (mLock); if (!mPubLedger) @@ -154,7 +172,7 @@ Ledger::pointer LedgerMaster::closeLedger (bool recover) { TransactionEngineParams tepFlags = tapOPEN_LEDGER; - if (theApp->getHashRouter ().addSuppressionPeer (it->first.getTXID (), SF_SIGGOOD)) + if (getApp().getHashRouter ().addSuppressionPeer (it->first.getTXID (), SF_SIGGOOD)) tepFlags = static_cast (tepFlags | tapNO_CHECK_SIGN); bool didApply; @@ -192,8 +210,8 @@ TER LedgerMaster::doTransaction (SerializedTransaction::ref txn, TransactionEngi result = mEngine.applyTransaction (*txn, params, didApply); ledger = mEngine.getLedger (); } - // if (didApply) - theApp->getOPs ().pubProposedTransaction (ledger, txn, result); + if (didApply) + getApp().getOPs ().pubProposedTransaction (ledger, txn, result); return result; } @@ -201,7 +219,7 @@ bool LedgerMaster::haveLedgerRange (uint32 from, uint32 to) { boost::recursive_mutex::scoped_lock sl (mLock); uint32 prevMissing = mCompleteLedgers.prevMissing (to + 1); - return (prevMissing == RangeSet::RangeSetAbsent) || (prevMissing < from); + return (prevMissing == RangeSet::absent) || (prevMissing < from); } bool LedgerMaster::haveLedger (uint32 seq) @@ -224,7 +242,7 @@ bool LedgerMaster::getValidatedRange (uint32& minVal, uint32& maxVal) minVal = mCompleteLedgers.prevMissing (maxVal); - if (minVal == RangeSet::RangeSetAbsent) + if (minVal == RangeSet::absent) minVal = maxVal; else ++minVal; @@ -257,7 +275,7 @@ void LedgerMaster::asyncAccept (Ledger::pointer ledger) if (it == ledgerHashes.end ()) { - if (theApp->isShutdown ()) + if (getApp().isShutdown ()) return; { @@ -294,18 +312,18 @@ bool LedgerMaster::acquireMissingLedger (Ledger::ref origLedger, uint256 const& if (ledger && (Ledger::getHashByIndex (ledgerSeq) == ledgerHash)) { WriteLog (lsTRACE, LedgerMaster) << "Ledger hash found in database"; - theApp->getJobQueue ().addJob (jtPUBOLDLEDGER, "LedgerMaster::asyncAccept", + getApp().getJobQueue ().addJob (jtPUBOLDLEDGER, "LedgerMaster::asyncAccept", BIND_TYPE (&LedgerMaster::asyncAccept, this, ledger)); return true; } - if (theApp->getInboundLedgers ().isFailure (ledgerHash)) + if (getApp().getInboundLedgers ().isFailure (ledgerHash)) { WriteLog (lsTRACE, LedgerMaster) << "Already failed to acquire " << ledgerSeq; return false; } - mMissingLedger = theApp->getInboundLedgers ().findCreate (ledgerHash, ledgerSeq); + mMissingLedger = getApp().getInboundLedgers ().findCreate (ledgerHash, ledgerSeq); if (mMissingLedger->isComplete ()) { @@ -328,14 +346,14 @@ bool LedgerMaster::acquireMissingLedger (Ledger::ref origLedger, uint256 const& if (mMissingLedger->setAccept ()) { if (!mMissingLedger->addOnComplete (BIND_TYPE (&LedgerMaster::missingAcquireComplete, this, P_1))) - theApp->getIOService ().post (boost::bind (&LedgerMaster::missingAcquireComplete, this, mMissingLedger)); + getApp().getIOService ().post (BIND_TYPE (&LedgerMaster::missingAcquireComplete, this, mMissingLedger)); } int fetchMax = theConfig.getSize (siLedgerFetch); int timeoutCount; - int fetchCount = theApp->getInboundLedgers ().getFetchCount (timeoutCount); + int fetchCount = getApp().getInboundLedgers ().getFetchCount (timeoutCount); - if ((fetchCount < fetchMax) && theApp->getOPs ().isFull ()) + if ((fetchCount < fetchMax) && getApp().getOPs ().isFull ()) { if (timeoutCount > 2) { @@ -348,9 +366,9 @@ bool LedgerMaster::acquireMissingLedger (Ledger::ref origLedger, uint256 const& BOOST_REVERSE_FOREACH (const u_pair & it, vec) { if ((fetchCount < fetchMax) && (it.first < ledgerSeq) && - !mCompleteLedgers.hasValue (it.first) && !theApp->getInboundLedgers ().find (it.second)) + !mCompleteLedgers.hasValue (it.first) && !getApp().getInboundLedgers ().find (it.second)) { - InboundLedger::pointer acq = theApp->getInboundLedgers ().findCreate (it.second, it.first); + InboundLedger::pointer acq = getApp().getInboundLedgers ().findCreate (it.second, it.first); if (acq && acq->isComplete ()) { @@ -365,7 +383,7 @@ bool LedgerMaster::acquireMissingLedger (Ledger::ref origLedger, uint256 const& } } - if (theApp->getOPs ().shouldFetchPack (ledgerSeq) && (ledgerSeq > 40000)) + if (getApp().getOPs ().shouldFetchPack (ledgerSeq) && (ledgerSeq > 40000)) { // refill our fetch pack Ledger::pointer nextLedger = mLedgerHistory.getLedgerBySeq (ledgerSeq + 1); @@ -377,7 +395,7 @@ bool LedgerMaster::acquireMissingLedger (Ledger::ref origLedger, uint256 const& tmBH.set_query (true); tmBH.set_seq (ledgerSeq); tmBH.set_ledgerhash (ledgerHash.begin (), 32); - std::vector peerList = theApp->getPeers ().getPeerVector (); + std::vector peerList = getApp().getPeers ().getPeerVector (); Peer::pointer target; int count = 0; @@ -446,7 +464,7 @@ void LedgerMaster::resumeAcquiring () // based on a myriad of conditions which short circuit the function // in ways that the caller cannot expect or predict. // - if (!theApp->getOPs ().isFull ()) + if (!getApp().getOPs ().isFull ()) return; boost::recursive_mutex::scoped_lock ml (mLock); @@ -462,7 +480,7 @@ void LedgerMaster::resumeAcquiring () uint32 prevMissing = mCompleteLedgers.prevMissing (mFinalizedLedger->getLedgerSeq ()); - if (prevMissing == RangeSet::RangeSetAbsent) + if (prevMissing == RangeSet::absent) { WriteLog (lsDEBUG, LedgerMaster) << "no prior missing ledger, not resuming"; return; @@ -524,7 +542,7 @@ void LedgerMaster::setFullLedger (Ledger::pointer ledger) WriteLog (lsDEBUG, LedgerMaster) << "Ledger " << ledger->getLedgerSeq () << " accepted :" << ledger->getHash (); assert (ledger->peekAccountStateMap ()->getHash ().isNonZero ()); - if (theApp->getOPs ().isNeedNetworkLedger ()) + if (getApp().getOPs ().isNeedNetworkLedger ()) return; boost::recursive_mutex::scoped_lock ml (mLock); @@ -553,7 +571,7 @@ void LedgerMaster::setFullLedger (Ledger::pointer ledger) if (mMissingLedger && mMissingLedger->isDone ()) { if (mMissingLedger->isFailed ()) - theApp->getInboundLedgers ().dropLedger (mMissingLedger->getHash ()); + getApp().getInboundLedgers ().dropLedger (mMissingLedger->getHash ()); mMissingLedger.reset (); } @@ -564,7 +582,7 @@ void LedgerMaster::setFullLedger (Ledger::pointer ledger) return; } - if (theApp->getJobQueue ().getJobCountTotal (jtPUBOLDLEDGER) > 1) + if (getApp().getJobQueue ().getJobCountTotal (jtPUBOLDLEDGER) > 1) { WriteLog (lsDEBUG, LedgerMaster) << "Too many pending ledger saves"; return; @@ -586,7 +604,7 @@ void LedgerMaster::setFullLedger (Ledger::pointer ledger) { uint32 prevMissing = mCompleteLedgers.prevMissing (ledger->getLedgerSeq ()); - if (prevMissing == RangeSet::RangeSetAbsent) + if (prevMissing == RangeSet::absent) { WriteLog (lsDEBUG, LedgerMaster) << "no prior missing ledger"; return; @@ -631,7 +649,7 @@ void LedgerMaster::checkAccept (uint256 const& hash, uint32 seq) if (mLastValidateHash.isNonZero ()) { - int val = theApp->getValidations ().getTrustedValidationCount (mLastValidateHash); + int val = getApp().getValidations ().getTrustedValidationCount (mLastValidateHash); val *= MIN_VALIDATION_RATIO; val /= 256; @@ -641,10 +659,10 @@ void LedgerMaster::checkAccept (uint256 const& hash, uint32 seq) if (theConfig.RUN_STANDALONE) minVal = 0; - else if (theApp->getOPs ().isNeedNetworkLedger ()) + else if (getApp().getOPs ().isNeedNetworkLedger ()) minVal = 1; - if (theApp->getValidations ().getTrustedValidationCount (hash) < minVal) // nothing we can do + if (getApp().getValidations ().getTrustedValidationCount (hash) < minVal) // nothing we can do return; WriteLog (lsINFO, LedgerMaster) << "Advancing accepted ledger to " << seq << " with >= " << minVal << " validations"; @@ -656,12 +674,22 @@ void LedgerMaster::checkAccept (uint256 const& hash, uint32 seq) if (!ledger) { - theApp->getInboundLedgers ().findCreate (hash, seq); + getApp().getInboundLedgers ().findCreate (hash, seq); return; } mValidLedger = ledger; + uint64 fee, fee2, ref; + ref = getApp().getFeeTrack().getLoadBase(); + int count = getApp().getValidations().getFeeAverage(ledger->getHash(), ref, fee); + int count2 = getApp().getValidations().getFeeAverage(ledger->getParentHash(), ref, fee2); + + if ((count + count2) == 0) + getApp().getFeeTrack().setRemoteFee(ref); + else + getApp().getFeeTrack().setRemoteFee(((fee * count) + (fee2 * count2)) / (count + count2)); + tryPublish (); } @@ -684,6 +712,7 @@ void LedgerMaster::tryPublish () } else if (mValidLedger->getLedgerSeq () > mPubLedger->getLedgerSeq ()) { + int acq = 0; for (uint32 seq = mPubLedger->getLedgerSeq () + 1; seq <= mValidLedger->getLedgerSeq (); ++seq) { WriteLog (lsTRACE, LedgerMaster) << "Trying to publish ledger " << seq; @@ -692,7 +721,7 @@ void LedgerMaster::tryPublish () uint256 hash; if (seq == mValidLedger->getLedgerSeq ()) - { + { // We need to publish the ledger we just fully validated ledger = mValidLedger; hash = ledger->getHash (); } @@ -710,50 +739,50 @@ void LedgerMaster::tryPublish () ledger = mLedgerHistory.getLedgerByHash (hash); } - if (ledger) - { - mPubLedger = ledger; - mPubLedgers.push_back (ledger); - } - else - { - if (theApp->getInboundLedgers ().isFailure (hash)) + if (!ledger && (++acq < 4)) + { // We can try to acquire the ledger we need + InboundLedger::pointer acq = getApp().getInboundLedgers ().findCreate (hash, seq); + + if (!acq->isDone ()) { - WriteLog (lsWARNING, LedgerMaster) << "Unable to acquire a recent validated ledger"; + acq->setAccept (); + } + else if (acq->isComplete () && !acq->isFailed ()) + { + ledger = acq->getLedger(); } else { - InboundLedger::pointer acq = theApp->getInboundLedgers ().findCreate (hash, seq); - - if (!acq->isDone ()) - { - acq->setAccept (); - break; - } - else if (acq->isComplete () && !acq->isFailed ()) - { - mPubLedger = acq->getLedger (); - mPubLedgers.push_back (mPubLedger); - } - else - WriteLog (lsWARNING, LedgerMaster) << "Failed to acquire a published ledger"; + WriteLog (lsWARNING, LedgerMaster) << "Failed to acquire a published ledger"; + getApp().getInboundLedgers().dropLedger(hash); + acq = getApp().getInboundLedgers().findCreate(hash, seq); + acq->setAccept(); + if (acq->isDone()) + ledger = acq->getLedger(); } } + + if (ledger && (ledger->getLedgerSeq() == (mPubLedger->getLedgerSeq() + 1))) + { // We acquired the next ledger we need to publish + mPubLedger = ledger; + mPubLedgers.push_back (mPubLedger); + } + } } if (!mPubLedgers.empty () && !mPubThread) { - theApp->getOPs ().clearNeedNetworkLedger (); + getApp().getOPs ().clearNeedNetworkLedger (); mPubThread = true; - theApp->getJobQueue ().addJob (jtPUBLEDGER, "Ledger::pubThread", + getApp().getJobQueue ().addJob (jtPUBLEDGER, "Ledger::pubThread", BIND_TYPE (&LedgerMaster::pubThread, this)); mPathFindNewLedger = true; if (!mPathFindThread) { mPathFindThread = true; - theApp->getJobQueue ().addJob (jtUPDATE_PF, "updatePaths", + getApp().getJobQueue ().addJob (jtUPDATE_PF, "updatePaths", BIND_TYPE (&LedgerMaster::updatePaths, this)); } } @@ -779,7 +808,7 @@ void LedgerMaster::pubThread () if (published && !mPathFindThread) { mPathFindThread = true; - theApp->getJobQueue ().addJob (jtUPDATE_PF, "updatePaths", + getApp().getJobQueue ().addJob (jtUPDATE_PF, "updatePaths", BIND_TYPE (&LedgerMaster::updatePaths, this)); } @@ -791,7 +820,7 @@ void LedgerMaster::pubThread () { WriteLog (lsDEBUG, LedgerMaster) << "Publishing ledger " << l->getLedgerSeq (); setFullLedger (l); // OPTIMIZEME: This is actually more work than we need to do - theApp->getOPs ().pubLedger (l); + getApp().getOPs ().pubLedger (l); published = true; } } @@ -841,7 +870,7 @@ void LedgerMaster::newPathRequest () if (!mPathFindThread) { mPathFindThread = true; - theApp->getJobQueue ().addJob (jtUPDATE_PF, "updatePaths", + getApp().getJobQueue ().addJob (jtUPDATE_PF, "updatePaths", BIND_TYPE (&LedgerMaster::updatePaths, this)); } } diff --git a/src/cpp/ripple/LedgerMaster.h b/src/cpp/ripple/LedgerMaster.h index 45b7913aa7..d000566ae5 100644 --- a/src/cpp/ripple/LedgerMaster.h +++ b/src/cpp/ripple/LedgerMaster.h @@ -14,7 +14,7 @@ // VFALCO TODO Rename to Ledgers // It sounds like this holds all the ledgers... // -class LedgerMaster +class LedgerMaster : LeakChecked { public: typedef FUNCTION_TYPE callback; @@ -32,6 +32,10 @@ public: { } + ~LedgerMaster () + { + } + uint32 getCurrentLedgerIndex (); ScopedLock getLock () @@ -54,11 +58,19 @@ public: return mFinalizedLedger; } - // The published ledger is the last fully validated ledger + // The validated ledger is the last fully validated ledger Ledger::ref getValidatedLedger () + { + return mValidLedger; + } + + // This is the last ledger we published to clients and can lag the validated ledger + Ledger::ref getPublishedLedger () { return mPubLedger; } + + int getPublishedLedgerAge (); int getValidatedLedgerAge (); bool isCaughtUp(std::string& reason); diff --git a/src/cpp/ripple/LedgerProposal.h b/src/cpp/ripple/LedgerProposal.h index 072c2cc50f..34b51258ff 100644 --- a/src/cpp/ripple/LedgerProposal.h +++ b/src/cpp/ripple/LedgerProposal.h @@ -11,6 +11,8 @@ class LedgerProposal : public CountedObject { public: + static char const* getCountedObjectName () { return "LedgerProposal"; } + static const uint32 seqLeave = 0xffffffff; // leaving the consensus process typedef boost::shared_ptr pointer; diff --git a/src/cpp/ripple/LedgerTiming.cpp b/src/cpp/ripple/LedgerTiming.cpp index d9dd412054..8434d83c00 100644 --- a/src/cpp/ripple/LedgerTiming.cpp +++ b/src/cpp/ripple/LedgerTiming.cpp @@ -5,8 +5,10 @@ //============================================================================== // VFALCO Should rename ContinuousLedgerTiming to LedgerTiming -struct LedgerTimingLog; -SETUP_LOG (LedgerTimingLog) + +struct LedgerTiming; // for Log + +SETUP_LOG (LedgerTiming) // NOTE: First and last times must be repeated int ContinuousLedgerTiming::LedgerTimeResolution[] = { 10, 10, 20, 30, 60, 90, 120, 120 }; @@ -26,7 +28,7 @@ bool ContinuousLedgerTiming::shouldClose ( if ((previousMSeconds < -1000) || (previousMSeconds > 600000) || (currentMSeconds < -1000) || (currentMSeconds > 600000)) { - WriteLog (lsWARNING, LedgerTimingLog) << + WriteLog (lsWARNING, LedgerTiming) << boost::str (boost::format ("CLC::shouldClose range Trans=%s, Prop: %d/%d, Secs: %d (last:%d)") % (anyTransactions ? "yes" : "no") % previousProposers % proposersClosed % currentMSeconds % previousMSeconds); @@ -38,7 +40,7 @@ bool ContinuousLedgerTiming::shouldClose ( // no transactions so far this interval if (proposersClosed > (previousProposers / 4)) // did we miss a transaction? { - WriteLog (lsTRACE, LedgerTimingLog) << "no transactions, many proposers: now (" << proposersClosed << " closed, " + WriteLog (lsTRACE, LedgerTiming) << "no transactions, many proposers: now (" << proposersClosed << " closed, " << previousProposers << " before)"; return true; } @@ -47,7 +49,7 @@ bool ContinuousLedgerTiming::shouldClose ( if (previousMSeconds > (1000 * (LEDGER_IDLE_INTERVAL + 2))) // the last ledger was very slow to close { - WriteLog (lsTRACE, LedgerTimingLog) << "was slow to converge (p=" << (previousMSeconds) << ")"; + WriteLog (lsTRACE, LedgerTiming) << "was slow to converge (p=" << (previousMSeconds) << ")"; if (previousMSeconds < 2000) return previousMSeconds; @@ -61,13 +63,13 @@ bool ContinuousLedgerTiming::shouldClose ( if ((openMSeconds < LEDGER_MIN_CLOSE) && ((proposersClosed + proposersValidated) < (previousProposers / 2 ))) { - WriteLog (lsDEBUG, LedgerTimingLog) << "Must wait minimum time before closing"; + WriteLog (lsDEBUG, LedgerTiming) << "Must wait minimum time before closing"; return false; } if ((currentMSeconds < previousMSeconds) && ((proposersClosed + proposersValidated) < previousProposers)) { - WriteLog (lsDEBUG, LedgerTimingLog) << "We are waiting for more closes/validations"; + WriteLog (lsDEBUG, LedgerTiming) << "We are waiting for more closes/validations"; return false; } @@ -86,7 +88,7 @@ bool ContinuousLedgerTiming::haveConsensus ( bool forReal, // deciding whether to stop consensus process bool& failed) // we can't reach a consensus { - WriteLog (lsTRACE, LedgerTimingLog) << boost::str (boost::format ("CLC::haveConsensus: prop=%d/%d agree=%d validated=%d time=%d/%d%s") % + WriteLog (lsTRACE, LedgerTiming) << boost::str (boost::format ("CLC::haveConsensus: prop=%d/%d agree=%d validated=%d time=%d/%d%s") % currentProposers % previousProposers % currentAgree % currentFinished % currentAgreeTime % previousAgreeTime % (forReal ? "" : "X")); @@ -98,7 +100,7 @@ bool ContinuousLedgerTiming::haveConsensus ( // Less than 3/4 of the last ledger's proposers are present, we may need more time if (currentAgreeTime < (previousAgreeTime + LEDGER_MIN_CONSENSUS)) { - CondLog (forReal, lsTRACE, LedgerTimingLog) << "too fast, not enough proposers"; + CondLog (forReal, lsTRACE, LedgerTiming) << "too fast, not enough proposers"; return false; } } @@ -106,7 +108,7 @@ bool ContinuousLedgerTiming::haveConsensus ( // If 80% of current proposers (plus us) agree on a set, we have consensus if (((currentAgree * 100 + 100) / (currentProposers + 1)) > 80) { - CondLog (forReal, lsINFO, LedgerTimingLog) << "normal consensus"; + CondLog (forReal, lsINFO, LedgerTiming) << "normal consensus"; failed = false; return true; } @@ -114,13 +116,13 @@ bool ContinuousLedgerTiming::haveConsensus ( // If 80% of the nodes on your UNL have moved on, you should declare consensus if (((currentFinished * 100) / (currentProposers + 1)) > 80) { - CondLog (forReal, lsWARNING, LedgerTimingLog) << "We see no consensus, but 80% of nodes have moved on"; + CondLog (forReal, lsWARNING, LedgerTiming) << "We see no consensus, but 80% of nodes have moved on"; failed = true; return true; } // no consensus yet - CondLog (forReal, lsTRACE, LedgerTimingLog) << "no consensus"; + CondLog (forReal, lsTRACE, LedgerTiming) << "no consensus"; return false; } diff --git a/src/cpp/ripple/LedgerUnitTests.cpp b/src/cpp/ripple/LedgerUnitTests.cpp new file mode 100644 index 0000000000..52ed5eb484 --- /dev/null +++ b/src/cpp/ripple/LedgerUnitTests.cpp @@ -0,0 +1,19 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +BOOST_AUTO_TEST_SUITE (quality) + +BOOST_AUTO_TEST_CASE ( getquality ) +{ + using namespace ripple; + + uint256 uBig ("D2DC44E5DC189318DB36EF87D2104CDF0A0FE3A4B698BEEE55038D7EA4C68000"); + + if (6125895493223874560 != Ledger::getQuality (uBig)) + BOOST_FAIL ("Ledger::getQuality fails."); +} + +BOOST_AUTO_TEST_SUITE_END () diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index 9a487fa92a..cba75861dc 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -4,7 +4,6 @@ */ //============================================================================== - SETUP_LOG (NetworkOPs) // This is the primary interface into the "client" portion of the program. @@ -18,19 +17,89 @@ SETUP_LOG (NetworkOPs) // code assumes this node is synched (and will continue to do so until // there's a functional network. -NetworkOPs::NetworkOPs (boost::asio::io_service& io_service, LedgerMaster* pLedgerMaster) : - mMode (omDISCONNECTED), mNeedNetworkLedger (false), mProposing (false), mValidating (false), - mFeatureBlocked (false), - mNetTimer (io_service), mLedgerMaster (pLedgerMaster), mCloseTimeOffset (0), mLastCloseProposers (0), - mLastCloseConvergeTime (1000 * LEDGER_IDLE_INTERVAL), mLastCloseTime (0), mLastValidationTime (0), - mJSONCache (120), mFetchPack ("FetchPack", 2048, 20), mLastFetchPack (0), mFetchSeq (static_cast (-1)), - mLastLoadBase (256), mLastLoadFactor (256) +NetworkOPs::NetworkOPs (LedgerMaster* pLedgerMaster) + : mMode (omDISCONNECTED) + , mNeedNetworkLedger (false) + , mProposing (false) + , mValidating (false) + , mFeatureBlocked (false) + , m_netTimer (this) + , m_clusterTimer (this) + , mLedgerMaster (pLedgerMaster) + , mCloseTimeOffset (0) + , mLastCloseProposers (0) + , mLastCloseConvergeTime (1000 * LEDGER_IDLE_INTERVAL) + , mLastCloseTime (0) + , mLastValidationTime (0) + , mFetchPack ("FetchPack", 2048, 20) + , mLastFetchPack (0) + // VFALCO TODO Give this magic number a name + , mFetchSeq (static_cast (-1)) + , mLastLoadBase (256) + , mLastLoadFactor (256) { } +void NetworkOPs::processNetTimer () +{ + ScopedLock sl (getApp().getMasterLock ()); + + getApp().getLoadManager ().resetDeadlockDetector (); + + std::size_t const numPeers = getApp().getPeers ().getPeerVector ().size (); + + // do we have sufficient peers? If not, we are disconnected. + if (numPeers < theConfig.NETWORK_QUORUM) + { + if (mMode != omDISCONNECTED) + { + setMode (omDISCONNECTED); + WriteLog (lsWARNING, NetworkOPs) + << "Node count (" << numPeers << ") " + << "has fallen below quorum (" << theConfig.NETWORK_QUORUM << ")."; + } + + return; + } + + if (mMode == omDISCONNECTED) + { + setMode (omCONNECTED); + WriteLog (lsINFO, NetworkOPs) << "Node count (" << numPeers << ") is sufficient."; + } + + // Check if the last validated ledger forces a change between these states + if (mMode == omSYNCING) + { + setMode (omSYNCING); + } + else if (mMode == omCONNECTED) + { + setMode (omCONNECTED); + } + + if (!mConsensus) + tryStartConsensus (); + + if (mConsensus) + mConsensus->timerEntry (); +} + +void NetworkOPs::onDeadlineTimer (DeadlineTimer& timer) +{ + if (timer == m_netTimer) + { + processNetTimer (); + } + else if (timer == m_clusterTimer) + { + doClusterReport(); + } +} + std::string NetworkOPs::strOperatingMode () { - static const char* paStatusToken[] = + static const char* paStatusToken [] = { "disconnected", "connected", @@ -54,7 +123,10 @@ std::string NetworkOPs::strOperatingMode () boost::posix_time::ptime NetworkOPs::getNetworkTimePT () { int offset = 0; - theApp->getSystemTimeOffset (offset); + + getApp().getSystemTimeOffset (offset); + + // VFALCO TODO Replace this with a beast call return boost::posix_time::microsec_clock::universal_time () + boost::posix_time::seconds (offset); } @@ -101,20 +173,7 @@ uint32 NetworkOPs::getLedgerID (uint256 const& hash) Ledger::pointer NetworkOPs::getLedgerBySeq (const uint32 seq) { - Ledger::pointer ret; - - ret = mLedgerMaster->getLedgerBySeq (seq); - - if (ret) - return ret; - - if (!haveLedger (seq)) - return ret; - - // We should have this ledger but we don't - WriteLog (lsWARNING, NetworkOPs) << "We should have ledger " << seq; - - return ret; + return mLedgerMaster->getLedgerBySeq (seq); } uint32 NetworkOPs::getCurrentLedgerID () @@ -163,7 +222,7 @@ void NetworkOPs::submitTransaction (Job&, SerializedTransaction::pointer iTrans, uint256 suppress = trans->getTransactionID (); int flags; - if (theApp->getHashRouter ().addSuppressionPeer (suppress, 0, flags) && ((flags & SF_RETRY) != 0)) + if (getApp().getHashRouter ().addSuppressionPeer (suppress, 0, flags) && ((flags & SF_RETRY) != 0)) { WriteLog (lsWARNING, NetworkOPs) << "Redundant transactions submitted"; return; @@ -182,11 +241,11 @@ void NetworkOPs::submitTransaction (Job&, SerializedTransaction::pointer iTrans, if (!trans->checkSign ()) { WriteLog (lsWARNING, NetworkOPs) << "Submitted transaction has bad signature"; - theApp->getHashRouter ().setFlag (suppress, SF_BAD); + getApp().getHashRouter ().setFlag (suppress, SF_BAD); return; } - theApp->getHashRouter ().setFlag (suppress, SF_SIGGOOD); + getApp().getHashRouter ().setFlag (suppress, SF_SIGGOOD); } catch (...) { @@ -196,7 +255,7 @@ void NetworkOPs::submitTransaction (Job&, SerializedTransaction::pointer iTrans, } // FIXME: Should submit to job queue - theApp->getIOService ().post (boost::bind (&NetworkOPs::processTransaction, this, + getApp().getIOService ().post (boost::bind (&NetworkOPs::processTransaction, this, boost::make_shared (trans, false), false, false, callback)); } @@ -239,17 +298,17 @@ void NetworkOPs::runTransactionQueue () for (int i = 0; i < 10; ++i) { - theApp->getTxnQueue ().getJob (txn); + getApp().getTxnQueue ().getJob (txn); if (!txn) return; { - LoadEvent::autoptr ev = theApp->getJobQueue ().getLoadEventAP (jtTXN_PROC, "runTxnQ"); + LoadEvent::autoptr ev = getApp().getJobQueue ().getLoadEventAP (jtTXN_PROC, "runTxnQ"); - boost::recursive_mutex::scoped_lock sl (theApp->getMasterLock ()); + boost::recursive_mutex::scoped_lock sl (getApp().getMasterLock ()); - Transaction::pointer dbtx = theApp->getMasterTransaction ().fetch (txn->getID (), true); + Transaction::pointer dbtx = getApp().getMasterTransaction ().fetch (txn->getID (), true); assert (dbtx); bool didApply; @@ -258,16 +317,17 @@ void NetworkOPs::runTransactionQueue () dbtx->setResult (r); if (isTemMalformed (r)) // malformed, cache bad - theApp->getHashRouter ().setFlag (txn->getID (), SF_BAD); + getApp().getHashRouter ().setFlag (txn->getID (), SF_BAD); // else if (isTelLocal (r) || isTerRetry (r)) // can be retried -// theApp->getHashRouter ().setFlag (txn->getID (), SF_RETRY); +// getApp().getHashRouter ().setFlag (txn->getID (), SF_RETRY); + if (isTerRetry (r)) { // transaction should be held WriteLog (lsDEBUG, NetworkOPs) << "QTransaction should be held: " << r; dbtx->setStatus (HELD); - theApp->getMasterTransaction ().canonicalize (dbtx, true); + getApp().getMasterTransaction ().canonicalize (dbtx); mLedgerMaster->addHeldTransaction (dbtx); } else if (r == tefPAST_SEQ) @@ -280,7 +340,7 @@ void NetworkOPs::runTransactionQueue () { WriteLog (lsINFO, NetworkOPs) << "QTransaction is now included in open ledger"; dbtx->setStatus (INCLUDED); - theApp->getMasterTransaction ().canonicalize (dbtx, true); + getApp().getMasterTransaction ().canonicalize (dbtx); } else { @@ -293,7 +353,7 @@ void NetworkOPs::runTransactionQueue () { std::set peers; - if (theApp->getHashRouter ().swapSet (txn->getID (), peers, SF_RELAYED)) + if (getApp().getHashRouter ().swapSet (txn->getID (), peers, SF_RELAYED)) { WriteLog (lsDEBUG, NetworkOPs) << "relaying"; protocol::TMTransaction tx; @@ -304,7 +364,7 @@ void NetworkOPs::runTransactionQueue () tx.set_receivetimestamp (getNetworkTimeNC ()); // FIXME: This should be when we received it PackedMessage::pointer packet = boost::make_shared (tx, protocol::mtTRANSACTION); - theApp->getPeers ().relayMessageBut (peers, packet); + getApp().getPeers ().relayMessageBut (peers, packet); } else WriteLog(lsDEBUG, NetworkOPs) << "recently relayed"; @@ -314,15 +374,15 @@ void NetworkOPs::runTransactionQueue () } } - if (theApp->getTxnQueue ().stopProcessing (txn)) - theApp->getIOService ().post (boost::bind (&NetworkOPs::runTransactionQueue, this)); + if (getApp().getTxnQueue ().stopProcessing (txn)) + getApp().getIOService ().post (BIND_TYPE (&NetworkOPs::runTransactionQueue, this)); } Transaction::pointer NetworkOPs::processTransaction (Transaction::pointer trans, bool bAdmin, bool bFailHard, stCallback callback) { - LoadEvent::autoptr ev = theApp->getJobQueue ().getLoadEventAP (jtTXN_PROC, "ProcessTXN"); + LoadEvent::autoptr ev = getApp().getJobQueue ().getLoadEventAP (jtTXN_PROC, "ProcessTXN"); - int newFlags = theApp->getHashRouter ().getFlags (trans->getID ()); + int newFlags = getApp().getHashRouter ().getFlags (trans->getID ()); if ((newFlags & SF_BAD) != 0) { @@ -340,25 +400,25 @@ Transaction::pointer NetworkOPs::processTransaction (Transaction::pointer trans, WriteLog (lsINFO, NetworkOPs) << "Transaction has bad signature"; trans->setStatus (INVALID); trans->setResult (temBAD_SIGNATURE); - theApp->getHashRouter ().setFlag (trans->getID (), SF_BAD); + getApp().getHashRouter ().setFlag (trans->getID (), SF_BAD); return trans; } - theApp->getHashRouter ().setFlag (trans->getID (), SF_SIGGOOD); + getApp().getHashRouter ().setFlag (trans->getID (), SF_SIGGOOD); } - boost::recursive_mutex::scoped_lock sl (theApp->getMasterLock ()); + boost::recursive_mutex::scoped_lock sl (getApp().getMasterLock ()); bool didApply; TER r = mLedgerMaster->doTransaction (trans->getSTransaction (), bAdmin ? (tapOPEN_LEDGER | tapNO_CHECK_SIGN | tapADMIN) : (tapOPEN_LEDGER | tapNO_CHECK_SIGN), didApply); trans->setResult (r); if (isTemMalformed (r)) // malformed, cache bad - theApp->getHashRouter ().setFlag (trans->getID (), SF_BAD); + getApp().getHashRouter ().setFlag (trans->getID (), SF_BAD); // else if (isTelLocal (r) || isTerRetry (r)) // can be retried -// theApp->getHashRouter ().setFlag (trans->getID (), SF_RETRY); +// getApp().getHashRouter ().setFlag (trans->getID (), SF_RETRY); -#ifdef DEBUG +#ifdef BEAST_DEBUG if (r != tesSUCCESS) { @@ -378,7 +438,7 @@ Transaction::pointer NetworkOPs::processTransaction (Transaction::pointer trans, { WriteLog (lsINFO, NetworkOPs) << "Transaction is now included in open ledger"; trans->setStatus (INCLUDED); - theApp->getMasterTransaction ().canonicalize (trans, true); + getApp().getMasterTransaction ().canonicalize (trans); } else if (r == tefPAST_SEQ) { @@ -393,7 +453,7 @@ Transaction::pointer NetworkOPs::processTransaction (Transaction::pointer trans, // transaction should be held WriteLog (lsDEBUG, NetworkOPs) << "Transaction should be held: " << r; trans->setStatus (HELD); - theApp->getMasterTransaction ().canonicalize (trans, true); + getApp().getMasterTransaction ().canonicalize (trans); mLedgerMaster->addHeldTransaction (trans); } } @@ -407,7 +467,7 @@ Transaction::pointer NetworkOPs::processTransaction (Transaction::pointer trans, { std::set peers; - if (theApp->getHashRouter ().swapSet (trans->getID (), peers, SF_RELAYED)) + if (getApp().getHashRouter ().swapSet (trans->getID (), peers, SF_RELAYED)) { protocol::TMTransaction tx; Serializer s; @@ -417,7 +477,7 @@ Transaction::pointer NetworkOPs::processTransaction (Transaction::pointer trans, tx.set_receivetimestamp (getNetworkTimeNC ()); // FIXME: This should be when we received it PackedMessage::pointer packet = boost::make_shared (tx, protocol::mtTRANSACTION); - theApp->getPeers ().relayMessageBut (peers, packet); + getApp().getPeers ().relayMessageBut (peers, packet); } } @@ -580,8 +640,9 @@ void NetworkOPs::setFeatureBlocked () void NetworkOPs::setStateTimer () { - mNetTimer.expires_from_now (boost::posix_time::milliseconds (LEDGER_GRANULARITY)); - mNetTimer.async_wait (boost::bind (&NetworkOPs::checkState, this, boost::asio::placeholders::error)); + m_netTimer.setRecurringExpiration (LEDGER_GRANULARITY / 1000.0); + + m_clusterTimer.setRecurringExpiration (10.0); } class ValidationCount @@ -592,19 +653,23 @@ public: ValidationCount () : trustedValidations (0), nodesUsing (0) { - ; } + bool operator> (const ValidationCount& v) { - if (trustedValidations > v.trustedValidations) return true; + if (trustedValidations > v.trustedValidations) + return true; - if (trustedValidations < v.trustedValidations) return false; + if (trustedValidations < v.trustedValidations) + return false; if (trustedValidations == 0) { - if (nodesUsing > v.nodesUsing) return true; + if (nodesUsing > v.nodesUsing) + return true; - if (nodesUsing < v.nodesUsing) return false; + if (nodesUsing < v.nodesUsing) return + false; return highNodeUsing > v.highNodeUsing; } @@ -613,65 +678,10 @@ public: } }; -void NetworkOPs::checkState (const boost::system::error_code& result) -{ - // Network state machine - - if ((result == boost::asio::error::operation_aborted) || theConfig.RUN_STANDALONE) - { - // VFALCO NOTE Should never get here. This is probably dead code. - // If RUN_STANDALONE is set then this function isn't called. - // - WriteLog (lsFATAL, NetworkOPs) << "Network state timer error: " << result; - return; - } - - { - ScopedLock sl (theApp->getMasterLock ()); - - theApp->getLoadManager ().resetDeadlockDetector (); - - std::vector peerList = theApp->getPeers ().getPeerVector (); - - // do we have sufficient peers? If not, we are disconnected. - if (peerList.size () < theConfig.NETWORK_QUORUM) - { - if (mMode != omDISCONNECTED) - { - setMode (omDISCONNECTED); - WriteLog (lsWARNING, NetworkOPs) << "Node count (" << peerList.size () << - ") has fallen below quorum (" << theConfig.NETWORK_QUORUM << ")."; - } - - return; - } - - if (mMode == omDISCONNECTED) - { - setMode (omCONNECTED); - WriteLog (lsINFO, NetworkOPs) << "Node count (" << peerList.size () << ") is sufficient."; - } - - // Check if the last validated ledger forces a change between these states - if (mMode == omSYNCING) - setMode (omSYNCING); - else if (mMode == omCONNECTED) - setMode (omCONNECTED); - - if (!mConsensus) - tryStartConsensus (); - - if (mConsensus) - mConsensus->timerEntry (); - } - - setStateTimer (); -} - void NetworkOPs::tryStartConsensus () { uint256 networkClosed; - bool ledgerChange = checkLastClosedLedger (theApp->getPeers ().getPeerVector (), networkClosed); + bool ledgerChange = checkLastClosedLedger (getApp().getPeers ().getPeerVector (), networkClosed); if (networkClosed.isZero ()) return; @@ -695,7 +705,7 @@ void NetworkOPs::tryStartConsensus () // check if the ledger is good enough to go to omFULL // Note: Do not go to omFULL if we don't have the previous ledger // check if the ledger is bad enough to go to omCONNECTED -- TODO - if (theApp->getOPs ().getNetworkTimeNC () < mLedgerMaster->getCurrentLedger ()->getCloseTimeNC ()) + if (getApp().getOPs ().getNetworkTimeNC () < mLedgerMaster->getCurrentLedger ()->getCloseTimeNC ()) setMode (omFULL); } @@ -725,7 +735,7 @@ bool NetworkOPs::checkLastClosedLedger (const std::vector& peerLi boost::unordered_map ledgers; { boost::unordered_map current = - theApp->getValidations ().getCurrentValidations (closedLedger, prevClosedLedger); + getApp().getValidations ().getCurrentValidations (closedLedger, prevClosedLedger); typedef std::map::value_type u256_cvc_pair; BOOST_FOREACH (const u256_cvc_pair & it, current) { @@ -742,7 +752,7 @@ bool NetworkOPs::checkLastClosedLedger (const std::vector& peerLi if (mMode >= omTRACKING) { ++ourVC.nodesUsing; - uint160 ourAddress = theApp->getLocalCredentials ().getNodePublic ().getNodeID (); + uint160 ourAddress = getApp().getLocalCredentials ().getNodePublic ().getNodeID (); if (ourAddress > ourVC.highNodeUsing) ourVC.highNodeUsing = ourAddress; @@ -806,7 +816,7 @@ bool NetworkOPs::checkLastClosedLedger (const std::vector& peerLi if (mAcquiringLedger) { mAcquiringLedger->abort (); - theApp->getInboundLedgers ().dropLedger (mAcquiringLedger->getHash ()); + getApp().getInboundLedgers ().dropLedger (mAcquiringLedger->getHash ()); mAcquiringLedger.reset (); } @@ -827,11 +837,11 @@ bool NetworkOPs::checkLastClosedLedger (const std::vector& peerLi WriteLog (lsINFO, NetworkOPs) << "Acquiring consensus ledger " << closedLedger; if (!mAcquiringLedger || (mAcquiringLedger->getHash () != closedLedger)) - mAcquiringLedger = theApp->getInboundLedgers ().findCreate (closedLedger, 0); + mAcquiringLedger = getApp().getInboundLedgers ().findCreate (closedLedger, 0); if (!mAcquiringLedger || mAcquiringLedger->isFailed ()) { - theApp->getInboundLedgers ().dropLedger (closedLedger); + getApp().getInboundLedgers ().dropLedger (closedLedger); WriteLog (lsERROR, NetworkOPs) << "Network ledger cannot be acquired"; return true; } @@ -867,13 +877,13 @@ void NetworkOPs::switchLastClosedLedger (Ledger::pointer newLedger, bool duringC protocol::TMStatusChange s; s.set_newevent (protocol::neSWITCHED_LEDGER); s.set_ledgerseq (newLedger->getLedgerSeq ()); - s.set_networktime (theApp->getOPs ().getNetworkTimeNC ()); + s.set_networktime (getApp().getOPs ().getNetworkTimeNC ()); uint256 hash = newLedger->getParentHash (); s.set_ledgerhashprevious (hash.begin (), hash.size ()); hash = newLedger->getHash (); s.set_ledgerhash (hash.begin (), hash.size ()); PackedMessage::pointer packet = boost::make_shared (s, protocol::mtSTATUS_CHANGE); - theApp->getPeers ().relayMessage (NULL, packet); + getApp().getPeers ().relayMessage (NULL, packet); } int NetworkOPs::beginConsensus (uint256 const& networkClosed, Ledger::pointer closingLedger) @@ -910,7 +920,7 @@ int NetworkOPs::beginConsensus (uint256 const& networkClosed, Ledger::pointer cl bool NetworkOPs::haveConsensusObject () { - if (mConsensus) + if (mConsensus != nullptr) return true; if ((mMode == omFULL) || (mMode == omTRACKING)) @@ -921,7 +931,7 @@ bool NetworkOPs::haveConsensusObject () { // we need to get into the consensus process uint256 networkClosed; - std::vector peerList = theApp->getPeers ().getPeerVector (); + std::vector peerList = getApp().getPeers ().getPeerVector (); bool ledgerChange = checkLastClosedLedger (peerList, networkClosed); if (!ledgerChange) @@ -931,7 +941,7 @@ bool NetworkOPs::haveConsensusObject () } } - return mConsensus; + return mConsensus != nullptr; } uint256 NetworkOPs::getConsensusLCL () @@ -945,7 +955,7 @@ uint256 NetworkOPs::getConsensusLCL () void NetworkOPs::processTrustedProposal (LedgerProposal::pointer proposal, boost::shared_ptr set, RippleAddress nodePublic, uint256 checkLedger, bool sigGood) { - boost::recursive_mutex::scoped_lock sl (theApp->getMasterLock ()); + boost::recursive_mutex::scoped_lock sl (getApp().getMasterLock ()); bool relay = true; @@ -982,9 +992,9 @@ void NetworkOPs::processTrustedProposal (LedgerProposal::pointer proposal, if (relay) { std::set peers; - theApp->getHashRouter ().swapSet (proposal->getHashRouter (), peers, SF_RELAYED); + getApp().getHashRouter ().swapSet (proposal->getHashRouter (), peers, SF_RELAYED); PackedMessage::pointer message = boost::make_shared (*set, protocol::mtPROPOSE_LEDGER); - theApp->getPeers ().relayMessageBut (peers, message); + getApp().getPeers ().relayMessageBut (peers, message); } else WriteLog (lsINFO, NetworkOPs) << "Not relaying trusted proposal"; @@ -1030,7 +1040,7 @@ SHAMapAddNode NetworkOPs::gotTXData (const boost::shared_ptr& peer, uint25 boost::shared_ptr consensus; { - ScopedLock mlh(theApp->getMasterLock()); + ScopedLock mlh(getApp().getMasterLock()); consensus = mConsensus; } @@ -1072,7 +1082,7 @@ void NetworkOPs::endConsensus (bool correctLCL) { uint256 deadLedger = mLedgerMaster->getClosedLedger ()->getParentHash (); - std::vector peerList = theApp->getPeers ().getPeerVector (); + std::vector peerList = getApp().getPeers ().getPeerVector (); BOOST_FOREACH (Peer::ref it, peerList) { @@ -1106,10 +1116,10 @@ void NetworkOPs::pubServer () jvObj ["type"] = "serverStatus"; jvObj ["server_status"] = strOperatingMode (); - jvObj ["load_base"] = (mLastLoadBase = theApp->getFeeTrack ().getLoadBase ()); - jvObj ["load_factor"] = (mLastLoadFactor = theApp->getFeeTrack ().getLoadFactor ()); + jvObj ["load_base"] = (mLastLoadBase = getApp().getFeeTrack ().getLoadBase ()); + jvObj ["load_factor"] = (mLastLoadFactor = getApp().getFeeTrack ().getLoadFactor ()); - NetworkOPs::subMapType::const_iterator it = mSubServer.begin (); + NetworkOPs::SubMapType::const_iterator it = mSubServer.begin (); while (it != mSubServer.end ()) { @@ -1136,13 +1146,13 @@ void NetworkOPs::setMode (OperatingMode om) if (om == omCONNECTED) { - if (theApp->getLedgerMaster ().getValidatedLedgerAge () < 60) + if (getApp().getLedgerMaster ().getValidatedLedgerAge () < 60) om = omSYNCING; } if (om == omSYNCING) { - if (theApp->getLedgerMaster ().getValidatedLedgerAge () >= 60) + if (getApp().getLedgerMaster ().getValidatedLedgerAge () >= 60) om = omCONNECTED; } @@ -1220,8 +1230,8 @@ NetworkOPs::getAccountTxs (const RippleAddress& account, int32 minLedger, int32 minLedger, maxLedger, descending, offset, limit, false, false, bAdmin); { - Database* db = theApp->getTxnDB ()->getDB (); - ScopedLock sl (theApp->getTxnDB ()->getDBLock ()); + Database* db = getApp().getTxnDB ()->getDB (); + ScopedLock sl (getApp().getTxnDB ()->getDBLock ()); SQL_FOREACH (db, sql) { @@ -1239,6 +1249,15 @@ NetworkOPs::getAccountTxs (const RippleAddress& account, int32 minLedger, int32 } else rawMeta.resize (metaSize); + if (rawMeta.getLength() == 0) + { // Work around a bug that could leave the metadata missing + uint32 seq = static_cast(db->getBigInt("AccountTransactions.LedgerSeq")); + WriteLog(lsWARNING, NetworkOPs) << "Recovering ledger " << seq << ", txn " << txn->getID(); + Ledger::pointer ledger = getLedgerBySeq(seq); + if (ledger) + ledger->pendSave(false); + } + TransactionMetaSet::pointer meta = boost::make_shared (txn->getID (), txn->getLedger (), rawMeta.getData ()); ret.push_back (std::pair (txn, meta)); } @@ -1257,8 +1276,8 @@ std::vector NetworkOPs::getAccountTxsB ( minLedger, maxLedger, descending, offset, limit, true/*binary*/, false, bAdmin); { - Database* db = theApp->getTxnDB ()->getDB (); - ScopedLock sl (theApp->getTxnDB ()->getDBLock ()); + Database* db = getApp().getTxnDB ()->getDB (); + ScopedLock sl (getApp().getTxnDB ()->getDBLock ()); SQL_FOREACH (db, sql) { @@ -1304,8 +1323,8 @@ NetworkOPs::countAccountTxs (const RippleAddress& account, int32 minLedger, int3 std::string sql = NetworkOPs::transactionsSQL ("COUNT(*) AS 'TransactionCount'", account, minLedger, maxLedger, false, 0, -1, true, true, true); - Database* db = theApp->getTxnDB ()->getDB (); - ScopedLock sl (theApp->getTxnDB ()->getDBLock ()); + Database* db = getApp().getTxnDB ()->getDB (); + ScopedLock sl (getApp().getTxnDB ()->getDBLock ()); SQL_FOREACH (db, sql) { ret = db->getInt ("TransactionCount"); @@ -1324,8 +1343,8 @@ NetworkOPs::getLedgerAffectedAccounts (uint32 ledgerSeq) % ledgerSeq); RippleAddress acct; { - Database* db = theApp->getTxnDB ()->getDB (); - ScopedLock sl (theApp->getTxnDB ()->getDBLock ()); + Database* db = getApp().getTxnDB ()->getDB (); + ScopedLock sl (getApp().getTxnDB ()->getDBLock ()); SQL_FOREACH (db, sql) { if (acct.setAccountID (db->getStrBinary ("Account"))) @@ -1338,7 +1357,7 @@ NetworkOPs::getLedgerAffectedAccounts (uint32 ledgerSeq) bool NetworkOPs::recvValidation (SerializedValidation::ref val, const std::string& source) { WriteLog (lsDEBUG, NetworkOPs) << "recvValidation " << val->getLedgerHash () << " from " << source; - return theApp->getValidations ().addValidation (val, source); + return getApp().getValidations ().addValidation (val, source); } Json::Value NetworkOPs::getConsensusInfo () @@ -1355,6 +1374,9 @@ Json::Value NetworkOPs::getServerInfo (bool human, bool admin) { Json::Value info = Json::objectValue; + info ["build_version"] = BuildVersion::getBuildVersion (); + info ["client_version"] = BuildVersion::getClientVersion (); + if (theConfig.TESTNET) info["testnet"] = theConfig.TESTNET; @@ -1373,10 +1395,10 @@ Json::Value NetworkOPs::getServerInfo (bool human, bool admin) info["pubkey_validator"] = "none"; } - info["pubkey_node"] = theApp->getLocalCredentials ().getNodePublic ().humanNodePublic (); + info["pubkey_node"] = getApp().getLocalCredentials ().getNodePublic ().humanNodePublic (); - info["complete_ledgers"] = theApp->getLedgerMaster ().getCompleteLedgers (); + info["complete_ledgers"] = getApp().getLedgerMaster ().getCompleteLedgers (); if (mFeatureBlocked) info["feature_blocked"] = true; @@ -1386,15 +1408,15 @@ Json::Value NetworkOPs::getServerInfo (bool human, bool admin) if (fp != 0) info["fetch_pack"] = Json::UInt (fp); - info["peers"] = theApp->getPeers ().getPeerCount (); + info["peers"] = getApp().getPeers ().getPeerCount (); Json::Value lastClose = Json::objectValue; - lastClose["proposers"] = theApp->getOPs ().getPreviousProposers (); + lastClose["proposers"] = getApp().getOPs ().getPreviousProposers (); if (human) - lastClose["converge_time_s"] = static_cast (theApp->getOPs ().getPreviousConvergeTime ()) / 1000.0; + lastClose["converge_time_s"] = static_cast (getApp().getOPs ().getPreviousConvergeTime ()) / 1000.0; else - lastClose["converge_time"] = Json::Int (theApp->getOPs ().getPreviousConvergeTime ()); + lastClose["converge_time"] = Json::Int (getApp().getOPs ().getPreviousConvergeTime ()); info["last_close"] = lastClose; @@ -1402,16 +1424,34 @@ Json::Value NetworkOPs::getServerInfo (bool human, bool admin) // info["consensus"] = mConsensus->getJson(); if (admin) - info["load"] = theApp->getJobQueue ().getJson (); + info["load"] = getApp().getJobQueue ().getJson (); if (!human) { - info["load_base"] = theApp->getFeeTrack ().getLoadBase (); - info["load_factor"] = theApp->getFeeTrack ().getLoadFactor (); + info["load_base"] = getApp().getFeeTrack ().getLoadBase (); + info["load_factor"] = getApp().getFeeTrack ().getLoadFactor (); } else + { info["load_factor"] = - static_cast (theApp->getFeeTrack ().getLoadFactor ()) / theApp->getFeeTrack ().getLoadBase (); + static_cast (getApp().getFeeTrack ().getLoadFactor ()) / getApp().getFeeTrack ().getLoadBase (); + if (admin) + { + uint32 base = getApp().getFeeTrack().getLoadBase(); + uint32 fee = getApp().getFeeTrack().getLocalFee(); + if (fee != base) + info["load_factor_local"] = + static_cast (fee) / base; + fee = getApp().getFeeTrack ().getRemoteFee(); + if (fee != base) + info["load_factor_net"] = + static_cast (fee) / base; + fee = getApp().getFeeTrack().getClusterFee(); + if (fee != base) + info["load_factor_cluster"] = + static_cast (fee) / base; + } + } bool valid = false; Ledger::pointer lpClosed = getValidatedLedger (); @@ -1460,6 +1500,12 @@ Json::Value NetworkOPs::getServerInfo (bool human, bool admin) info["validated_ledger"] = l; else info["closed_ledger"] = l; + + Ledger::pointer lpPublished = getPublishedLedger (); + if (!lpPublished) + info["published_ledger"] = "none"; + else if (lpPublished->getLedgerSeq() != lpClosed->getLedgerSeq()) + info["published_ledger"] = lpPublished->getLedgerSeq(); } return info; @@ -1489,7 +1535,7 @@ void NetworkOPs::pubProposedTransaction (Ledger::ref lpCurrent, SerializedTransa { boost::recursive_mutex::scoped_lock sl (mMonitorLock); - NetworkOPs::subMapType::const_iterator it = mSubRTTransactions.begin (); + NetworkOPs::SubMapType::const_iterator it = mSubRTTransactions.begin (); while (it != mSubRTTransactions.end ()) { @@ -1537,9 +1583,9 @@ void NetworkOPs::pubLedger (Ledger::ref accepted) jvObj["txn_count"] = Json::UInt (alpAccepted->getTxnCount ()); if (mMode >= omSYNCING) - jvObj["validated_ledgers"] = theApp->getLedgerMaster ().getCompleteLedgers (); + jvObj["validated_ledgers"] = getApp().getLedgerMaster ().getCompleteLedgers (); - NetworkOPs::subMapType::const_iterator it = mSubLedger.begin (); + NetworkOPs::SubMapType::const_iterator it = mSubLedger.begin (); while (it != mSubLedger.end ()) { @@ -1569,11 +1615,11 @@ void NetworkOPs::pubLedger (Ledger::ref accepted) void NetworkOPs::reportFeeChange () { - if ((theApp->getFeeTrack ().getLoadBase () == mLastLoadBase) && - (theApp->getFeeTrack ().getLoadFactor () == mLastLoadFactor)) + if ((getApp().getFeeTrack ().getLoadBase () == mLastLoadBase) && + (getApp().getFeeTrack ().getLoadFactor () == mLastLoadFactor)) return; - theApp->getJobQueue ().addJob (jtCLIENT, "reportFeeChange->pubServer", BIND_TYPE (&NetworkOPs::pubServer, this)); + getApp().getJobQueue ().addJob (jtCLIENT, "reportFeeChange->pubServer", BIND_TYPE (&NetworkOPs::pubServer, this)); } Json::Value NetworkOPs::transJson (const SerializedTransaction& stTxn, TER terResult, bool bValidated, @@ -1595,6 +1641,9 @@ Json::Value NetworkOPs::transJson (const SerializedTransaction& stTxn, TER terRe jvObj["ledger_hash"] = lpCurrent->getHash ().ToString (); jvObj["transaction"]["date"] = lpCurrent->getCloseTimeNC (); jvObj["validated"] = true; + + // WRITEME: Put the account next seq here + } else { @@ -1618,7 +1667,7 @@ void NetworkOPs::pubValidatedTransaction (Ledger::ref alAccepted, const Accepted { boost::recursive_mutex::scoped_lock sl (mMonitorLock); - NetworkOPs::subMapType::const_iterator it = mSubTransactions.begin (); + NetworkOPs::SubMapType::const_iterator it = mSubTransactions.begin (); while (it != mSubTransactions.end ()) { @@ -1648,7 +1697,7 @@ void NetworkOPs::pubValidatedTransaction (Ledger::ref alAccepted, const Accepted it = mSubRTTransactions.erase (it); } } - theApp->getOrderBookDB ().processTxn (alAccepted, alTx, jvObj); + getApp().getOrderBookDB ().processTxn (alAccepted, alTx, jvObj); pubAccountTransaction (alAccepted, alTx, true); } @@ -1667,11 +1716,11 @@ void NetworkOPs::pubAccountTransaction (Ledger::ref lpCurrent, const AcceptedLed { BOOST_FOREACH (const RippleAddress & affectedAccount, alTx.getAffected ()) { - subInfoMapIterator simiIt = mSubRTAccount.find (affectedAccount.getAccountID ()); + SubInfoMapIterator simiIt = mSubRTAccount.find (affectedAccount.getAccountID ()); if (simiIt != mSubRTAccount.end ()) { - NetworkOPs::subMapType::const_iterator it = simiIt->second.begin (); + NetworkOPs::SubMapType::const_iterator it = simiIt->second.begin (); while (it != simiIt->second.end ()) { @@ -1694,7 +1743,7 @@ void NetworkOPs::pubAccountTransaction (Ledger::ref lpCurrent, const AcceptedLed if (simiIt != mSubAccount.end ()) { - NetworkOPs::subMapType::const_iterator it = simiIt->second.begin (); + NetworkOPs::SubMapType::const_iterator it = simiIt->second.begin (); while (it != simiIt->second.end ()) { @@ -1736,7 +1785,7 @@ void NetworkOPs::pubAccountTransaction (Ledger::ref lpCurrent, const AcceptedLed void NetworkOPs::subAccount (InfoSub::ref isrListener, const boost::unordered_set& vnaAccountIDs, uint32 uLedgerIndex, bool rt) { - subInfoMapType& subMap = rt ? mSubRTAccount : mSubAccount; + SubInfoMapType& subMap = rt ? mSubRTAccount : mSubAccount; // For the connection, monitor each account. BOOST_FOREACH (const RippleAddress & naAccountID, vnaAccountIDs) @@ -1750,12 +1799,12 @@ void NetworkOPs::subAccount (InfoSub::ref isrListener, const boost::unordered_se BOOST_FOREACH (const RippleAddress & naAccountID, vnaAccountIDs) { - subInfoMapType::iterator simIterator = subMap.find (naAccountID.getAccountID ()); + SubInfoMapType::iterator simIterator = subMap.find (naAccountID.getAccountID ()); if (simIterator == subMap.end ()) { // Not found, note that account has a new single listner. - subMapType usisElement; + SubMapType usisElement; usisElement[isrListener->getSeq ()] = isrListener; subMap.insert (simIterator, make_pair (naAccountID.getAccountID (), usisElement)); } @@ -1769,7 +1818,7 @@ void NetworkOPs::subAccount (InfoSub::ref isrListener, const boost::unordered_se void NetworkOPs::unsubAccount (uint64 uSeq, const boost::unordered_set& vnaAccountIDs, bool rt) { - subInfoMapType& subMap = rt ? mSubRTAccount : mSubAccount; + SubInfoMapType& subMap = rt ? mSubRTAccount : mSubAccount; // For the connection, unmonitor each account. // FIXME: Don't we need to unsub? @@ -1782,7 +1831,7 @@ void NetworkOPs::unsubAccount (uint64 uSeq, const boost::unordered_setgetOrderBookDB ().makeBookListeners (currencyPays, currencyGets, issuerPays, issuerGets); + getApp().getOrderBookDB ().makeBookListeners (currencyPays, currencyGets, issuerPays, issuerGets); if (listeners) listeners->addSubscriber (isrListener); @@ -1820,7 +1869,7 @@ bool NetworkOPs::unsubBook (uint64 uSeq, const uint160& currencyPays, const uint160& currencyGets, const uint160& issuerPays, const uint160& issuerGets) { BookListeners::pointer listeners = - theApp->getOrderBookDB ().getBookListeners (currencyPays, currencyGets, issuerPays, issuerGets); + getApp().getOrderBookDB ().getBookListeners (currencyPays, currencyGets, issuerPays, issuerGets); if (listeners) listeners->removeSubscriber (uSeq); @@ -1882,7 +1931,7 @@ bool NetworkOPs::subLedger (InfoSub::ref isrListener, Json::Value& jvResult) } if ((mMode >= omSYNCING) && !isNeedNetworkLedger ()) - jvResult["validated_ledgers"] = theApp->getLedgerMaster ().getCompleteLedgers (); + jvResult["validated_ledgers"] = getApp().getLedgerMaster ().getCompleteLedgers (); boost::recursive_mutex::scoped_lock sl (mMonitorLock); return mSubLedger.emplace (isrListener->getSeq (), isrListener).second; @@ -1909,8 +1958,8 @@ bool NetworkOPs::subServer (InfoSub::ref isrListener, Json::Value& jvResult) RandomNumbers::getInstance ().fillBytes (uRandom.begin (), uRandom.size ()); jvResult["random"] = uRandom.ToString (); jvResult["server_status"] = strOperatingMode (); - jvResult["load_base"] = theApp->getFeeTrack ().getLoadBase (); - jvResult["load_factor"] = theApp->getFeeTrack ().getLoadFactor (); + jvResult["load_base"] = getApp().getFeeTrack ().getLoadBase (); + jvResult["load_factor"] = getApp().getFeeTrack ().getLoadFactor (); boost::recursive_mutex::scoped_lock sl (mMonitorLock); return mSubServer.emplace (isrListener->getSeq (), isrListener).second; @@ -2164,7 +2213,7 @@ void NetworkOPs::makeFetchPack (Job&, boost::weak_ptr wPeer, return; } - if (theApp->getFeeTrack ().isLoaded ()) + if (getApp().getFeeTrack ().isLoadedLocal ()) { WriteLog (lsINFO, NetworkOPs) << "Too busy to make fetch pack"; return; @@ -2226,12 +2275,11 @@ void NetworkOPs::makeFetchPack (Job&, boost::weak_ptr wPeer, void NetworkOPs::sweepFetchPack () { mFetchPack.sweep (); - mJSONCache.sweep (); } void NetworkOPs::addFetchPack (uint256 const& hash, boost::shared_ptr< Blob >& data) { - mFetchPack.canonicalize (hash, data, false); + mFetchPack.canonicalize (hash, data); } bool NetworkOPs::getFetchPack (uint256 const& hash, Blob& data) @@ -2267,11 +2315,18 @@ bool NetworkOPs::shouldFetchPack (uint32 seq) int size = mFetchPack.getCacheSize (); if (size == 0) + { + // VFALCO TODO Give this magic number a name + // mFetchSeq = static_cast (-1); + } else if (mFetchPack.getCacheSize () > 64) + { return false; + } mLastFetchPack = now; + return true; } @@ -2284,17 +2339,45 @@ void NetworkOPs::gotFetchPack (bool progress, uint32 seq) { mLastFetchPack = 0; mFetchSeq = seq; // earliest pack we have data on - theApp->getJobQueue ().addJob (jtLEDGER_DATA, "gotFetchPack", - boost::bind (&InboundLedgers::gotFetchPack, &theApp->getInboundLedgers (), _1)); + getApp().getJobQueue ().addJob (jtLEDGER_DATA, "gotFetchPack", + BIND_TYPE (&InboundLedgers::gotFetchPack, &getApp().getInboundLedgers (), P_1)); } void NetworkOPs::missingNodeInLedger (uint32 seq) { WriteLog (lsWARNING, NetworkOPs) << "We are missing a node in ledger " << seq; - uint256 hash = theApp->getLedgerMaster ().getHashBySeq (seq); + uint256 hash = getApp().getLedgerMaster ().getHashBySeq (seq); if (hash.isNonZero ()) - theApp->getInboundLedgers ().findCreate (hash, seq); + getApp().getInboundLedgers ().findCreate (hash, seq); +} + +void NetworkOPs::doClusterReport () +{ + bool synced = (getApp().getLedgerMaster().getValidatedLedgerAge() <= 240); + ClusterNodeStatus us("", synced ? getApp().getFeeTrack().getLocalFee() : 0, getNetworkTimeNC()); + if (!getApp().getUNL().nodeUpdate(getApp().getLocalCredentials().getNodePublic(), us)) + { + WriteLog (lsDEBUG, NetworkOPs) << "To soon to send cluster update"; + return; + } + + std::map nodes = getApp().getUNL().getClusterStatus(); + + protocol::TMCluster cluster; + for (std::map::iterator it = nodes.begin(), + end = nodes.end(); it != end; ++it) + { + protocol::TMClusterNode& node = *cluster.add_clusternodes(); + node.set_publickey(it->first.humanNodePublic()); + node.set_reporttime(it->second.getReportTime()); + node.set_nodeload(it->second.getLoadFee()); + if (!it->second.getName().empty()) + node.set_nodename(it->second.getName()); + } + + PackedMessage::pointer message = boost::make_shared(cluster, protocol::mtCLUSTER); + getApp().getPeers().relayMessageCluster (NULL, message); } // vim:ts=4 diff --git a/src/cpp/ripple/NetworkOPs.h b/src/cpp/ripple/NetworkOPs.h index 329acbb35a..d2797f7471 100644 --- a/src/cpp/ripple/NetworkOPs.h +++ b/src/cpp/ripple/NetworkOPs.h @@ -14,6 +14,8 @@ class Peer; class LedgerConsensus; class NetworkOPs + : public DeadlineTimer::Listener + , LeakChecked { public: enum Fault @@ -34,6 +36,7 @@ public: }; #if 0 + // VFALCO TODO Make this happen /** Subscription data interface. */ class Subscriber @@ -45,13 +48,15 @@ public: */ virtual void onSubscriberReceiveJSON (Json::Value const& json) { } }; - typedef boost::unordered_map subMapType; + typedef boost::unordered_map SubMapType; #endif - typedef boost::unordered_map subMapType; + typedef boost::unordered_map SubMapType; public: - NetworkOPs (boost::asio::io_service& io_service, LedgerMaster* pLedgerMaster); + // VFALCO TODO Make LedgerMaster a SharedObjectPtr or a reference. + // + explicit NetworkOPs (LedgerMaster* pLedgerMaster); // network information uint32 getNetworkTimeNC (); // Our best estimate of wall time in seconds from 1/1/2000 @@ -75,6 +80,10 @@ public: { return mLedgerMaster->getValidatedLedger (); } + Ledger::ref getPublishedLedger () + { + return mLedgerMaster->getPublishedLedger (); + } Ledger::ref getCurrentLedger () { return mLedgerMaster->getCurrentLedger (); @@ -225,30 +234,10 @@ public: int getFetchSize (); void sweepFetchPack (); - float getJSONHitRate () - { - return mJSONCache.getHitRate (); - } - - // VFALCO TODO Rename this to getNumberOfCachedJSONItems or something similar - int getJSONEntries () - { - return mJSONCache.getNumberOfEntries (); - } - - void storeJSONCache (JSONCache::Kind kind, const uint256& ledger, const uint160& object, - const boost::shared_ptr & data) - { - mJSONCache.storeEntry (kind, ledger, object, data); - } - - boost::shared_ptr getJSONCache (JSONCache::Kind kind, const uint256& ledger, const uint160& object) - { - return mJSONCache.getEntry (kind, ledger, object); - } - // network state machine - void checkState (const boost::system::error_code& result); + + // VFALCO TODO Try to make all these private since they seem to be...private + // void switchLastClosedLedger (Ledger::pointer newLedger, bool duringConsensus); // Used for the "jump" case bool checkLastClosedLedger (const std::vector&, uint256& networkClosed); int beginConsensus (uint256 const& networkClosed, Ledger::pointer closingLedger); @@ -323,6 +312,8 @@ public: uint256 getConsensusLCL (); void reportFeeChange (); + void doClusterReport (); + //Helper function to generate SQL query to get transactions std::string transactionsSQL (std::string selection, const RippleAddress& account, int32 minLedger, int32 maxLedger, bool descending, uint32 offset, int limit, @@ -374,9 +365,24 @@ public: InfoSub::pointer addRpcSub (const std::string& strUrl, InfoSub::ref rspEntry); private: - typedef boost::unordered_map subInfoMapType; - typedef boost::unordered_map::value_type subInfoMapValue; - typedef boost::unordered_map::iterator subInfoMapIterator; + void processNetTimer (); + void onDeadlineTimer (DeadlineTimer& timer); + + void setMode (OperatingMode); + + Json::Value transJson (const SerializedTransaction& stTxn, TER terResult, bool bValidated, Ledger::ref lpCurrent); + bool haveConsensusObject (); + + Json::Value pubBootstrapAccountInfo (Ledger::ref lpAccepted, const RippleAddress& naAccountID); + + void pubValidatedTransaction (Ledger::ref alAccepted, const AcceptedLedgerTx& alTransaction); + void pubAccountTransaction (Ledger::ref lpCurrent, const AcceptedLedgerTx& alTransaction, bool isAccepted); + + void pubServer (); + +private: + typedef boost::unordered_map SubInfoMapType; + typedef boost::unordered_map ::iterator SubInfoMapIterator; typedef boost::unordered_map subRpcMapType; @@ -385,7 +391,8 @@ private: bool mProposing, mValidating; bool mFeatureBlocked; boost::posix_time::ptime mConnectTime; - boost::asio::deadline_timer mNetTimer; + DeadlineTimer m_netTimer; + DeadlineTimer m_clusterTimer; boost::shared_ptr mConsensus; boost::unordered_map < uint160, std::list > mStoredProposals; @@ -407,37 +414,27 @@ private: // XXX Split into more locks. boost::recursive_mutex mMonitorLock; - subInfoMapType mSubAccount; - subInfoMapType mSubRTAccount; + SubInfoMapType mSubAccount; + SubInfoMapType mSubRTAccount; subRpcMapType mRpcSubMap; - subMapType mSubLedger; // accepted ledgers - subMapType mSubServer; // when server changes connectivity state - subMapType mSubTransactions; // all accepted transactions - subMapType mSubRTTransactions; // all proposed and accepted transactions - - JSONCache mJSONCache; + SubMapType mSubLedger; // accepted ledgers + SubMapType mSubServer; // when server changes connectivity state + SubMapType mSubTransactions; // all accepted transactions + SubMapType mSubRTTransactions; // all proposed and accepted transactions TaggedCache< uint256, Blob , UptimeTimerAdapter > mFetchPack; uint32 mLastFetchPack; + + // VFALCO TODO Document the special value uint32(-1) for this member + // and replace uint32(-1) with a constant. It is initialized + // in the ctor-initializer list to this constant. + // uint32 mFetchSeq; uint32 mLastLoadBase; uint32 mLastLoadFactor; - - void setMode (OperatingMode); - - Json::Value transJson (const SerializedTransaction& stTxn, TER terResult, bool bValidated, Ledger::ref lpCurrent); - bool haveConsensusObject (); - - Json::Value pubBootstrapAccountInfo (Ledger::ref lpAccepted, const RippleAddress& naAccountID); - - void pubValidatedTransaction (Ledger::ref alAccepted, const AcceptedLedgerTx& alTransaction); - void pubAccountTransaction (Ledger::ref lpCurrent, const AcceptedLedgerTx& alTransaction, bool isAccepted); - - void pubServer (); }; #endif -// vim:ts=4 diff --git a/src/cpp/ripple/OfferCreateTransactor.cpp b/src/cpp/ripple/OfferCreateTransactor.cpp index 7eb04c0706..2d86a7d075 100644 --- a/src/cpp/ripple/OfferCreateTransactor.cpp +++ b/src/cpp/ripple/OfferCreateTransactor.cpp @@ -19,7 +19,7 @@ bool OfferCreateTransactor::bValidOffer ( boost::unordered_set& usAccountTouched, STAmount& saOfferFunds) // <-- { - bool bValid; + bool bValid = false; if (sleOffer->isFieldPresent (sfExpiration) && sleOffer->getFieldU32 (sfExpiration) <= mEngine->getLedger ()->getParentCloseTimeNC ()) { @@ -46,6 +46,8 @@ bool OfferCreateTransactor::bValidOffer ( % saOfferPays % saOfferGets); usOfferUnfundedFound.insert (uOfferIndex); + + bValid = false; } else { @@ -712,7 +714,7 @@ TER OfferCreateTransactor::doApply () CondLog (tesSUCCESS != terResult, lsINFO, OfferCreateTransactor) << boost::str (boost::format ("OfferCreate: final terResult=%s") % transToken (terResult)); if (isTesSuccess (terResult)) - theApp->getOrderBookDB ().invalidate (); + getApp().getOrderBookDB ().invalidate (); return terResult; } diff --git a/src/cpp/ripple/OrderBookDB.cpp b/src/cpp/ripple/OrderBookDB.cpp index 3f4a4b7ccf..8392ac23e8 100644 --- a/src/cpp/ripple/OrderBookDB.cpp +++ b/src/cpp/ripple/OrderBookDB.cpp @@ -28,7 +28,7 @@ void OrderBookDB::setup (Ledger::ref ledger) mSeq = ledger->getLedgerSeq (); - LoadEvent::autoptr ev = theApp->getJobQueue ().getLoadEventAP (jtOB_SETUP, "OrderBookDB::setup"); + LoadEvent::autoptr ev = getApp().getJobQueue ().getLoadEventAP (jtOB_SETUP, "OrderBookDB::setup"); mDestMap.clear (); mSourceMap.clear (); @@ -224,7 +224,7 @@ void BookListeners::publish (Json::Value& jvObj) std::string sObj = jfwWriter.write (jvObj); boost::recursive_mutex::scoped_lock sl (mLock); - NetworkOPs::subMapType::const_iterator it = mListeners.begin (); + NetworkOPs::SubMapType::const_iterator it = mListeners.begin (); while (it != mListeners.end ()) { diff --git a/src/cpp/ripple/OrderBookDB.h b/src/cpp/ripple/OrderBookDB.h index be4bbcf414..467c80546d 100644 --- a/src/cpp/ripple/OrderBookDB.h +++ b/src/cpp/ripple/OrderBookDB.h @@ -34,7 +34,7 @@ private: boost::recursive_mutex mLock; }; -class OrderBookDB +class OrderBookDB : LeakChecked { public: OrderBookDB (); diff --git a/src/cpp/ripple/Peer.h b/src/cpp/ripple/Peer.h index e7987a9b55..5aef8aafab 100644 --- a/src/cpp/ripple/Peer.h +++ b/src/cpp/ripple/Peer.h @@ -59,6 +59,8 @@ public: virtual bool isConnected () const = 0; + virtual bool isInCluster () const = 0; + virtual bool isInbound () const = 0; virtual bool isOutbound () const = 0; diff --git a/src/cpp/ripple/PeerDoor.cpp b/src/cpp/ripple/PeerDoor.cpp index 808fa08aec..34ca3bf1b0 100644 --- a/src/cpp/ripple/PeerDoor.cpp +++ b/src/cpp/ripple/PeerDoor.cpp @@ -6,147 +6,102 @@ SETUP_LOG (PeerDoor) -using namespace std; -using namespace boost::asio::ip; - -PeerDoor::PeerDoor (boost::asio::io_service& io_service) : - mAcceptor (io_service, - tcp::endpoint (address ().from_string (theConfig.PEER_IP.empty () ? "0.0.0.0" : theConfig.PEER_IP), - theConfig.PEER_PORT)), - mCtx (boost::asio::ssl::context::sslv23), mDelayTimer (io_service) +class PeerDoorImp : public PeerDoor, LeakChecked { - mCtx.set_options ( - boost::asio::ssl::context::default_workarounds - | boost::asio::ssl::context::no_sslv2 - | boost::asio::ssl::context::single_dh_use); - - SSL_CTX_set_tmp_dh_callback (mCtx.native_handle (), handleTmpDh); - - if (1 != SSL_CTX_set_cipher_list (mCtx.native_handle (), theConfig.PEER_SSL_CIPHER_LIST.c_str ())) - std::runtime_error ("Error setting cipher list (no valid ciphers)."); - - - if (!theConfig.PEER_IP.empty () && theConfig.PEER_PORT) +public: + PeerDoorImp (std::string const& ip, + int port, + std::string const& sslCiphers, + boost::asio::io_service& io_service) + : mAcceptor ( + io_service, + boost::asio::ip::tcp::endpoint (boost::asio::ip::address ().from_string (ip.empty () ? "0.0.0.0" : ip), + port)) + , mCtx (boost::asio::ssl::context::sslv23) + , mDelayTimer (io_service) { - Log (lsINFO) << "Peer port: " << theConfig.PEER_IP << " " << theConfig.PEER_PORT; - startListening (); - } -} + mCtx.set_options ( + boost::asio::ssl::context::default_workarounds | + boost::asio::ssl::context::no_sslv2 | + boost::asio::ssl::context::single_dh_use); -void PeerDoor::startListening () -{ - Peer::pointer new_connection = Peer::New ( - mAcceptor.get_io_service (), - mCtx, - theApp->getPeers ().assignPeerId (), - true); + SSL_CTX_set_tmp_dh_callback (mCtx.native_handle (), handleTmpDh); - mAcceptor.async_accept (new_connection->getSocket (), - boost::bind (&PeerDoor::handleConnect, this, new_connection, - boost::asio::placeholders::error)); -} + if (SSL_CTX_set_cipher_list (mCtx.native_handle (), sslCiphers.c_str ()) != 1) + std::runtime_error ("Error setting cipher list (no valid ciphers)."); -void PeerDoor::handleConnect (Peer::pointer new_connection, - const boost::system::error_code& error) -{ - bool delay = false; - - if (!error) - { - new_connection->connected (error); - } - else - { - if (error == boost::system::errc::too_many_files_open) - delay = true; - - WriteLog (lsERROR, PeerDoor) << error; - } - - if (delay) - { - mDelayTimer.expires_from_now (boost::posix_time::milliseconds (500)); - mDelayTimer.async_wait (boost::bind (&PeerDoor::startListening, this)); - } - else - { - startListening (); - } -} - -void initSSLContext (boost::asio::ssl::context& context, - std::string key_file, std::string cert_file, std::string chain_file) -{ - SSL_CTX* sslContext = context.native_handle (); - - context.set_options (boost::asio::ssl::context::default_workarounds | - boost::asio::ssl::context::no_sslv2 | - boost::asio::ssl::context::single_dh_use); - - bool cert_set = false; - - if (!cert_file.empty ()) - { - boost::system::error_code error; - context.use_certificate_file (cert_file, boost::asio::ssl::context::pem, error); - - if (error) - throw std::runtime_error ("Unable to use certificate file"); - - cert_set = true; - } - - if (!chain_file.empty ()) - { - // VFALCO Replace fopen() with RAII - FILE* f = fopen (chain_file.c_str (), "r"); - - if (!f) - throw std::runtime_error ("Unable to open chain file"); - - try + if (! ip.empty () && port != 0) { - while (true) - { - X509* x = PEM_read_X509 (f, NULL, NULL, NULL); - - if (x == NULL) - break; - - if (!cert_set) - { - if (SSL_CTX_use_certificate (sslContext, x) != 1) - throw std::runtime_error ("Unable to get certificate from chain file"); - - cert_set = true; - } - else if (SSL_CTX_add_extra_chain_cert (sslContext, x) != 1) - { - X509_free (x); - throw std::runtime_error ("Unable to add chain certificate"); - } - } - - fclose (f); - } - catch (...) - { - fclose (f); - throw; + Log (lsINFO) << "Peer port: " << ip << " " << port; + startListening (); } } - if (!key_file.empty ()) - { - boost::system::error_code error; - context.use_private_key_file (key_file, boost::asio::ssl::context::pem, error); + //-------------------------------------------------------------------------- - if (error) - throw std::runtime_error ("Unable to use private key file"); + boost::asio::ssl::context& getSSLContext () + { + return mCtx; } - if (SSL_CTX_check_private_key (sslContext) != 1) - throw std::runtime_error ("Private key not valid"); -} + //-------------------------------------------------------------------------- -// vim:ts=4 + void startListening () + { + Peer::pointer new_connection = Peer::New ( + mAcceptor.get_io_service (), + mCtx, + getApp().getPeers ().assignPeerId (), + true); + + mAcceptor.async_accept (new_connection->getSocket (), + boost::bind (&PeerDoorImp::handleConnect, this, new_connection, + boost::asio::placeholders::error)); + } + + //-------------------------------------------------------------------------- + + void handleConnect (Peer::pointer new_connection, + const boost::system::error_code& error) + { + bool delay = false; + + if (!error) + { + new_connection->connected (error); + } + else + { + if (error == boost::system::errc::too_many_files_open) + delay = true; + + WriteLog (lsERROR, PeerDoor) << error; + } + + if (delay) + { + mDelayTimer.expires_from_now (boost::posix_time::milliseconds (500)); + mDelayTimer.async_wait (boost::bind (&PeerDoorImp::startListening, this)); + } + else + { + startListening (); + } + } + +private: + boost::asio::ip::tcp::acceptor mAcceptor; + boost::asio::ssl::context mCtx; + boost::asio::deadline_timer mDelayTimer; +}; + +//------------------------------------------------------------------------------ + +PeerDoor* PeerDoor::New ( + std::string const& ip, + int port, + std::string const& sslCiphers, + boost::asio::io_service& io_service) +{ + return new PeerDoorImp (ip, port, sslCiphers, io_service); +} diff --git a/src/cpp/ripple/PeerDoor.h b/src/cpp/ripple/PeerDoor.h index c2e4d89007..decaa657ea 100644 --- a/src/cpp/ripple/PeerDoor.h +++ b/src/cpp/ripple/PeerDoor.h @@ -4,32 +4,23 @@ */ //============================================================================== -#ifndef __PEERDOOR__ -#define __PEERDOOR__ +#ifndef RIPPLE_PEERDOOR_H_INCLUDED +#define RIPPLE_PEERDOOR_H_INCLUDED -/* -Handles incoming connections from other Peers +/** Handles incoming connections from peers. */ - -class PeerDoor +class PeerDoor : LeakChecked { public: - PeerDoor (boost::asio::io_service& io_service); + virtual ~PeerDoor () { } - boost::asio::ssl::context& getSSLContext () - { - return mCtx; - } + static PeerDoor* New ( + std::string const& ip, + int port, + std::string const& sslCiphers, + boost::asio::io_service& io_service); -private: - boost::asio::ip::tcp::acceptor mAcceptor; - boost::asio::ssl::context mCtx; - boost::asio::deadline_timer mDelayTimer; - - void startListening (); - void handleConnect (Peer::pointer new_connection, const boost::system::error_code& error); + virtual boost::asio::ssl::context& getSSLContext () = 0; }; #endif - -// vim:ts=4 diff --git a/src/cpp/ripple/RPC.h b/src/cpp/ripple/RPC.h index 771141ded4..1b7cedad14 100644 --- a/src/cpp/ripple/RPC.h +++ b/src/cpp/ripple/RPC.h @@ -4,44 +4,33 @@ */ //============================================================================== -#ifndef __RPC_h__ -#define __RPC_h__ - -enum http_status_type -{ - ok = 200, - created = 201, - accepted = 202, - no_content = 204, - multiple_choices = 300, - moved_permanently = 301, - moved_temporarily = 302, - not_modified = 304, - bad_request = 400, - unauthorized = 401, - forbidden = 403, - not_found = 404, - internal_server_error = 500, - not_implemented = 501, - bad_gateway = 502, - service_unavailable = 503 -}; +#ifndef RIPPLE_RPC_H_INCLUDED +#define RIPPLE_RPC_H_INCLUDED +// VFALCO TODO Wrap these up into a class. It looks like they just do some +// convenience packaging of JSON data from the pieces. It looks +// Ripple client protocol-specific. +// extern std::string JSONRPCRequest (const std::string& strMethod, const Json::Value& params, const Json::Value& id); -extern std::string createHTTPPost (const std::string& strHost, const std::string& strPath, const std::string& strMsg, - const std::map& mapRequestHeaders); - -extern int ReadHTTP (std::basic_istream& stream, - std::map& mapHeadersRet, std::string& strMessageRet); - -extern std::string HTTPReply (int nStatus, const std::string& strMsg); - extern std::string JSONRPCReply (const Json::Value& result, const Json::Value& error, const Json::Value& id); extern Json::Value JSONRPCError (int code, const std::string& message); -extern bool HTTPAuthorized (const std::map& mapHeaders); +extern std::string createHTTPPost (const std::string& strHost, const std::string& strPath, const std::string& strMsg, + const std::map& mapRequestHeaders); + +extern std::string HTTPReply (int nStatus, const std::string& strMsg); + +// VFALCO TODO Create a HTTPHeaders class with a nice interface instead of the std::map +// +extern bool HTTPAuthorized (std::map const& mapHeaders); + +// VFALCO NOTE This one looks like it does some sort of stream i/o +// +extern int ReadHTTP (std::basic_istream& stream, + std::map& mapHeadersRet, + std::string& strMessageRet); #endif diff --git a/src/cpp/ripple/RPCDoor.cpp b/src/cpp/ripple/RPCDoor.cpp index f27208ca24..b692d442a0 100644 --- a/src/cpp/ripple/RPCDoor.cpp +++ b/src/cpp/ripple/RPCDoor.cpp @@ -6,32 +6,37 @@ SETUP_LOG (RPCDoor) -using namespace std; -using namespace boost::asio::ip; - -extern void initSSLContext (boost::asio::ssl::context& context, - std::string key_file, std::string cert_file, std::string chain_file); - -RPCDoor::RPCDoor (boost::asio::io_service& io_service) : - mAcceptor (io_service, tcp::endpoint (address::from_string (theConfig.RPC_IP), theConfig.RPC_PORT)), - mDelayTimer (io_service), mSSLContext (boost::asio::ssl::context::sslv23) +RPCDoor::RPCDoor (boost::asio::io_service& io_service, RPCServer::Handler& handler) + : m_rpcServerHandler (handler) + , mAcceptor (io_service, + boost::asio::ip::tcp::endpoint (boost::asio::ip::address::from_string (theConfig.getRpcIP ()), theConfig.getRpcPort ())) + , mDelayTimer (io_service) + , mSSLContext (boost::asio::ssl::context::sslv23) { - WriteLog (lsINFO, RPCDoor) << "RPC port: " << theConfig.RPC_IP << " " << theConfig.RPC_PORT << " allow remote: " << theConfig.RPC_ALLOW_REMOTE; + WriteLog (lsINFO, RPCDoor) << "RPC port: " << theConfig.getRpcAddress().toRawUTF8() << " allow remote: " << theConfig.RPC_ALLOW_REMOTE; if (theConfig.RPC_SECURE != 0) - initSSLContext (mSSLContext, theConfig.RPC_SSL_KEY, theConfig.RPC_SSL_CERT, theConfig.RPC_SSL_CHAIN); + { + // VFALCO TODO This could be a method of theConfig + // + basio::SslContext::initializeFromFile ( + mSSLContext, + theConfig.RPC_SSL_KEY, + theConfig.RPC_SSL_CERT, + theConfig.RPC_SSL_CHAIN); + } startListening (); } RPCDoor::~RPCDoor () { - WriteLog (lsINFO, RPCDoor) << "RPC port: " << theConfig.RPC_IP << " " << theConfig.RPC_PORT << " allow remote: " << theConfig.RPC_ALLOW_REMOTE; + WriteLog (lsINFO, RPCDoor) << "RPC port: " << theConfig.getRpcAddress().toRawUTF8() << " allow remote: " << theConfig.RPC_ALLOW_REMOTE; } void RPCDoor::startListening () { - RPCServer::pointer new_connection = RPCServer::create (mAcceptor.get_io_service (), mSSLContext, &theApp->getOPs ()); + RPCServer::pointer new_connection = RPCServer::New (mAcceptor.get_io_service (), mSSLContext, m_rpcServerHandler); mAcceptor.set_option (boost::asio::ip::tcp::acceptor::reuse_address (true)); mAcceptor.async_accept (new_connection->getRawSocket (), @@ -44,6 +49,8 @@ bool RPCDoor::isClientAllowed (const std::string& ip) if (theConfig.RPC_ALLOW_REMOTE) return true; + // VFALCO TODO Represent ip addresses as a structure. Use isLoopback() member here + // if (ip == "127.0.0.1") return true; @@ -59,7 +66,7 @@ void RPCDoor::handleConnect (RPCServer::pointer new_connection, const boost::sys // Restrict callers by IP try { - if (!isClientAllowed (new_connection->getRawSocket ().remote_endpoint ().address ().to_string ())) + if (! isClientAllowed (new_connection->getRemoteAddressText ())) { startListening (); return; diff --git a/src/cpp/ripple/RPCDoor.h b/src/cpp/ripple/RPCDoor.h index 7a5d4763d1..c9d1562e69 100644 --- a/src/cpp/ripple/RPCDoor.h +++ b/src/cpp/ripple/RPCDoor.h @@ -11,13 +11,16 @@ Handles incoming connections from people making RPC Requests */ -class RPCDoor +class RPCDoor : LeakChecked { public: - explicit RPCDoor (boost::asio::io_service& io_service); + explicit RPCDoor ( + boost::asio::io_service& io_service, + RPCServer::Handler& handler); ~RPCDoor (); private: + RPCServer::Handler& m_rpcServerHandler; boost::asio::ip::tcp::acceptor mAcceptor; boost::asio::deadline_timer mDelayTimer; boost::asio::ssl::context mSSLContext; diff --git a/src/cpp/ripple/RPCErr.cpp b/src/cpp/ripple/RPCErr.cpp index 22fa0ab628..caa262fc7b 100644 --- a/src/cpp/ripple/RPCErr.cpp +++ b/src/cpp/ripple/RPCErr.cpp @@ -4,8 +4,7 @@ */ //============================================================================== -// For logging -struct RPCErr { }; +struct RPCErr; // for Log SETUP_LOG (RPCErr) diff --git a/src/cpp/ripple/RPCHandler.cpp b/src/cpp/ripple/RPCHandler.cpp index b9b2c21854..5abb2b6241 100644 --- a/src/cpp/ripple/RPCHandler.cpp +++ b/src/cpp/ripple/RPCHandler.cpp @@ -60,7 +60,7 @@ RPCHandler::RPCHandler (NetworkOPs* netOps, InfoSub::pointer infoSub) : mNetOps Json::Value RPCHandler::transactionSign (Json::Value params, bool bSubmit, bool bFailHard, ScopedLock& mlh) { - if (theApp->getFeeTrack().isLoaded() && (mRole != ADMIN)) + if (getApp().getFeeTrack().isLoadedCluster() && (mRole != ADMIN)) return rpcError(rpcTOO_BUSY); Json::Value jvResult; @@ -70,7 +70,7 @@ Json::Value RPCHandler::transactionSign (Json::Value params, bool bSubmit, bool WriteLog (lsDEBUG, RPCHandler) << boost::str (boost::format ("transactionSign: %s") % params); - if (!bOffline && (theApp->getLedgerMaster().getValidatedLedgerAge() > 120)) + if (!bOffline && (getApp().getLedgerMaster().getValidatedLedgerAge() > 120)) { return rpcError (rpcNO_CURRENT); } @@ -282,9 +282,9 @@ Json::Value RPCHandler::transactionSign (Json::Value params, bool bSubmit, bool // ... or the master key must have been used. && raSrcAddressID.getAccountID () != naAccountPublic.getAccountID ()) { - // std::cerr << "iIndex: " << iIndex << std::endl; - // std::cerr << "sfAuthorizedKey: " << strHex(asSrc->getAuthorizedKey().getAccountID()) << std::endl; - // std::cerr << "naAccountPublic: " << strHex(naAccountPublic.getAccountID()) << std::endl; + // Log::out() << "iIndex: " << iIndex; + // Log::out() << "sfAuthorizedKey: " << strHex(asSrc->getAuthorizedKey().getAccountID()); + // Log::out() << "naAccountPublic: " << strHex(naAccountPublic.getAccountID()); return rpcError (rpcSRC_ACT_NOT_FOUND); } @@ -500,9 +500,9 @@ Json::Value RPCHandler::authorize (Ledger::ref lrLedger, if (asSrc->haveAuthorizedKey () && (asSrc->getAuthorizedKey ().getAccountID () != naAccountPublic.getAccountID ())) { - // std::cerr << "iIndex: " << iIndex << std::endl; - // std::cerr << "sfAuthorizedKey: " << strHex(asSrc->getAuthorizedKey().getAccountID()) << std::endl; - // std::cerr << "naAccountPublic: " << strHex(naAccountPublic.getAccountID()) << std::endl; + // Log::out() << "iIndex: " << iIndex; + // Log::out() << "sfAuthorizedKey: " << strHex(asSrc->getAuthorizedKey().getAccountID()); + // Log::out() << "naAccountPublic: " << strHex(naAccountPublic.getAccountID()); return rpcError (rpcPASSWD_CHANGED); } @@ -652,7 +652,7 @@ Json::Value RPCHandler::doConnect (Json::Value params, LoadType* loadType, Scope int iPort = params.isMember ("port") ? params["port"].asInt () : -1; // XXX Validate legal IP and port - theApp->getPeers ().connectTo (strIp, iPort); + getApp().getPeers ().connectTo (strIp, iPort); return "connecting"; } @@ -670,7 +670,7 @@ Json::Value RPCHandler::doDataDelete (Json::Value params, LoadType* loadType, Sc Json::Value ret = Json::Value (Json::objectValue); - if (theApp->getLocalCredentials ().dataDelete (strKey)) + if (getApp().getLocalCredentials ().dataDelete (strKey)) { ret["key"] = strKey; } @@ -699,7 +699,7 @@ Json::Value RPCHandler::doDataFetch (Json::Value params, LoadType* loadType, Sco ret["key"] = strKey; - if (theApp->getLocalCredentials ().dataFetch (strKey, strValue)) + if (getApp().getLocalCredentials ().dataFetch (strKey, strValue)) ret["value"] = strValue; return ret; @@ -722,7 +722,7 @@ Json::Value RPCHandler::doDataStore (Json::Value params, LoadType* loadType, Sco Json::Value ret = Json::Value (Json::objectValue); - if (theApp->getLocalCredentials ().dataStore (strKey, strValue)) + if (getApp().getLocalCredentials ().dataStore (strKey, strValue)) { ret["key"] = strKey; ret["value"] = strValue; @@ -801,7 +801,9 @@ Json::Value RPCHandler::doPeers (Json::Value, LoadType* loadType, ScopedLock& Ma { Json::Value jvResult (Json::objectValue); - jvResult["peers"] = theApp->getPeers ().getPeersJson (); + jvResult["peers"] = getApp().getPeers ().getPeersJson (); + + getApp().getUNL().addClusterStatus(jvResult); return jvResult; } @@ -944,7 +946,7 @@ Json::Value RPCHandler::doProofCreate (Json::Value params, LoadType* loadType, S } else { - jvResult["token"] = theApp->getProofOfWorkFactory ().getProof ().getToken (); + jvResult["token"] = getApp().getProofOfWorkFactory ().getProof ().getToken (); } return jvResult; @@ -1032,7 +1034,7 @@ Json::Value RPCHandler::doProofVerify (Json::Value params, LoadType* loadType, S else { // XXX Proof should not be marked as used from this - prResult = theApp->getProofOfWorkFactory ().checkProof (strToken, uSolution); + prResult = getApp().getProofOfWorkFactory ().checkProof (strToken, uSolution); } std::string sToken; @@ -1223,7 +1225,7 @@ Json::Value RPCHandler::doAccountOffers (Json::Value params, LoadType* loadType, // } Json::Value RPCHandler::doBookOffers (Json::Value params, LoadType* loadType, ScopedLock& MasterLockHolder) { - if (theApp->getJobQueue ().getJobCountGE (jtCLIENT) > 200) + if (getApp().getJobQueue ().getJobCountGE (jtCLIENT) > 200) { return rpcError (rpcTOO_BUSY); } @@ -1362,7 +1364,7 @@ Json::Value RPCHandler::doPathFind (Json::Value params, LoadType* loadType, Scop if (request->isValid ()) { mInfoSub->setPathRequest (request); - theApp->getLedgerMaster ().newPathRequest (); + getApp().getLedgerMaster ().newPathRequest (); } return result; @@ -1400,7 +1402,7 @@ Json::Value RPCHandler::doPathFind (Json::Value params, LoadType* loadType, Scop // - From a trusted server, allows clients to use path without manipulation. Json::Value RPCHandler::doRipplePathFind (Json::Value params, LoadType* loadType, ScopedLock& MasterLockHolder) { - int jc = theApp->getJobQueue ().getJobCountGE (jtCLIENT); + int jc = getApp().getJobQueue ().getJobCountGE (jtCLIENT); if (jc > 200) { @@ -1782,8 +1784,8 @@ Json::Value RPCHandler::doTxHistory (Json::Value params, LoadType* loadType, Sco % startIndex); { - Database* db = theApp->getTxnDB ()->getDB (); - ScopedLock sl (theApp->getTxnDB ()->getDBLock ()); + Database* db = getApp().getTxnDB ()->getDB (); + ScopedLock sl (getApp().getTxnDB ()->getDBLock ()); SQL_FOREACH (db, sql) { @@ -1815,7 +1817,7 @@ Json::Value RPCHandler::doTx (Json::Value params, LoadType* loadType, ScopedLock // transaction by ID uint256 txid (strTransaction); - Transaction::pointer txn = theApp->getMasterTransaction ().fetch (txid, true); + Transaction::pointer txn = getApp().getMasterTransaction ().fetch (txid, true); if (!txn) return rpcError (rpcTXN_NOT_FOUND); @@ -1900,8 +1902,8 @@ Json::Value RPCHandler::doLedger (Json::Value params, LoadType* loadType, Scoped { Json::Value ret (Json::objectValue), current (Json::objectValue), closed (Json::objectValue); - theApp->getLedgerMaster ().getCurrentLedger ()->addJson (current, 0); - theApp->getLedgerMaster ().getClosedLedger ()->addJson (closed, 0); + getApp().getLedgerMaster ().getCurrentLedger ()->addJson (current, 0); + getApp().getLedgerMaster ().getClosedLedger ()->addJson (closed, 0); ret["open"] = current; ret["closed"] = closed; @@ -2007,7 +2009,7 @@ Json::Value RPCHandler::doAccountTransactions (Json::Value params, LoadType* loa uLedgerMin = uLedgerMax = l->getLedgerSeq (); } -#ifndef DEBUG +#ifndef BEAST_DEBUG try { @@ -2073,7 +2075,7 @@ Json::Value RPCHandler::doAccountTransactions (Json::Value params, LoadType* loa return ret; -#ifndef DEBUG +#ifndef BEAST_DEBUG } catch (...) { @@ -2120,7 +2122,7 @@ Json::Value RPCHandler::doValidationSeed (Json::Value params, LoadType* loadType if (!params.isMember ("secret")) { - std::cerr << "Unset validation seed." << std::endl; + Log::out() << "Unset validation seed."; theConfig.VALIDATION_SEED.clear (); theConfig.VALIDATION_PUB.clear (); @@ -2350,11 +2352,11 @@ Json::Value RPCHandler::doFeature (Json::Value params, LoadType* loadType, Scope if (!params.isMember ("feature")) { Json::Value jvReply = Json::objectValue; - jvReply["features"] = theApp->getFeatureTable ().getJson (0); + jvReply["features"] = getApp().getFeatureTable ().getJson (0); return jvReply; } - uint256 uFeature = theApp->getFeatureTable ().getFeature (params["feature"].asString ()); + uint256 uFeature = getApp().getFeatureTable ().getFeature (params["feature"].asString ()); if (uFeature.isZero ()) { @@ -2365,7 +2367,7 @@ Json::Value RPCHandler::doFeature (Json::Value params, LoadType* loadType, Scope } if (!params.isMember ("vote")) - return theApp->getFeatureTable ().getJson (uFeature); + return getApp().getFeatureTable ().getJson (uFeature); // WRITEME return rpcError (rpcNOT_SUPPORTED); @@ -2390,37 +2392,27 @@ Json::Value RPCHandler::doGetCounts (Json::Value params, LoadType* loadType, Sco ret [it.first] = it.second; } - int dbKB = theApp->getLedgerDB ()->getDB ()->getKBUsedAll (); + int dbKB = getApp().getLedgerDB ()->getDB ()->getKBUsedAll (); if (dbKB > 0) ret["dbKBTotal"] = dbKB; - dbKB = theApp->getLedgerDB ()->getDB ()->getKBUsedDB (); + dbKB = getApp().getLedgerDB ()->getDB ()->getKBUsedDB (); if (dbKB > 0) ret["dbKBLedger"] = dbKB; - if (!theApp->getHashedObjectStore ().isLevelDB ()) - { - dbKB = theApp->getHashNodeDB ()->getDB ()->getKBUsedDB (); - - if (dbKB > 0) - ret["dbKBHashNode"] = dbKB; - } - - dbKB = theApp->getTxnDB ()->getDB ()->getKBUsedDB (); + dbKB = getApp().getTxnDB ()->getDB ()->getKBUsedDB (); if (dbKB > 0) ret["dbKBTransaction"] = dbKB; - ret["write_load"] = theApp->getHashedObjectStore ().getWriteLoad (); + ret["write_load"] = getApp().getNodeStore ().getWriteLoad (); - ret["SLE_hit_rate"] = theApp->getSLECache ().getHitRate (); - ret["node_hit_rate"] = theApp->getHashedObjectStore ().getCacheHitRate (); - ret["ledger_hit_rate"] = theApp->getLedgerMaster ().getCacheHitRate (); + ret["SLE_hit_rate"] = getApp().getSLECache ().getHitRate (); + ret["node_hit_rate"] = getApp().getNodeStore ().getCacheHitRate (); + ret["ledger_hit_rate"] = getApp().getLedgerMaster ().getCacheHitRate (); ret["AL_hit_rate"] = AcceptedLedger::getCacheHitRate (); - ret["JC_hit_rate"] = theApp->getOPs ().getJSONHitRate (); - ret["JC_size"] = theApp->getOPs ().getJSONEntries (); ret["fullbelow_size"] = SHAMap::getFullBelowSize (); @@ -2498,13 +2490,13 @@ Json::Value RPCHandler::doUnlAdd (Json::Value params, LoadType* loadType, Scoped if (raNodePublic.setNodePublic (strNode)) { - theApp->getUNL ().nodeAddPublic (raNodePublic, IUniqueNodeList::vsManual, strComment); + getApp().getUNL ().nodeAddPublic (raNodePublic, UniqueNodeList::vsManual, strComment); return "adding node by public key"; } else { - theApp->getUNL ().nodeAddDomain (strNode, IUniqueNodeList::vsManual, strComment); + getApp().getUNL ().nodeAddDomain (strNode, UniqueNodeList::vsManual, strComment); return "adding node by domain"; } @@ -2524,13 +2516,13 @@ Json::Value RPCHandler::doUnlDelete (Json::Value params, LoadType* loadType, Sco if (raNodePublic.setNodePublic (strNode)) { - theApp->getUNL ().nodeRemovePublic (raNodePublic); + getApp().getUNL ().nodeRemovePublic (raNodePublic); return "removing node by public key"; } else { - theApp->getUNL ().nodeRemoveDomain (strNode); + getApp().getUNL ().nodeRemoveDomain (strNode); return "removing node by domain"; } @@ -2540,7 +2532,7 @@ Json::Value RPCHandler::doUnlList (Json::Value, LoadType* loadType, ScopedLock& { Json::Value obj (Json::objectValue); - obj["unl"] = theApp->getUNL ().getUnlJson (); + obj["unl"] = getApp().getUNL ().getUnlJson (); return obj; } @@ -2548,7 +2540,7 @@ Json::Value RPCHandler::doUnlList (Json::Value, LoadType* loadType, ScopedLock& // Populate the UNL from a local validators.txt file. Json::Value RPCHandler::doUnlLoad (Json::Value, LoadType* loadType, ScopedLock& MasterLockHolder) { - if (theConfig.VALIDATORS_FILE.empty () || !theApp->getUNL ().nodeLoad (theConfig.VALIDATORS_FILE)) + if (theConfig.VALIDATORS_FILE.empty () || !getApp().getUNL ().nodeLoad (theConfig.VALIDATORS_FILE)) { return rpcError (rpcLOAD_FAILED); } @@ -2560,7 +2552,7 @@ Json::Value RPCHandler::doUnlLoad (Json::Value, LoadType* loadType, ScopedLock& // Populate the UNL from ripple.com's validators.txt file. Json::Value RPCHandler::doUnlNetwork (Json::Value params, LoadType* loadType, ScopedLock& MasterLockHolder) { - theApp->getUNL ().nodeNetwork (); + getApp().getUNL ().nodeNetwork (); return "fetching"; } @@ -2568,7 +2560,7 @@ Json::Value RPCHandler::doUnlNetwork (Json::Value params, LoadType* loadType, Sc // unl_reset Json::Value RPCHandler::doUnlReset (Json::Value params, LoadType* loadType, ScopedLock& MasterLockHolder) { - theApp->getUNL ().nodeReset (); + getApp().getUNL ().nodeReset (); return "removing nodes"; } @@ -2576,7 +2568,7 @@ Json::Value RPCHandler::doUnlReset (Json::Value params, LoadType* loadType, Scop // unl_score Json::Value RPCHandler::doUnlScore (Json::Value, LoadType* loadType, ScopedLock& MasterLockHolder) { - theApp->getUNL ().nodeScore (); + getApp().getUNL ().nodeScore (); return "scoring requested"; } @@ -2586,13 +2578,13 @@ Json::Value RPCHandler::doSMS (Json::Value params, LoadType* loadType, ScopedLoc if (!params.isMember ("text")) return rpcError (rpcINVALID_PARAMS); - HttpsClient::sendSMS (theApp->getIOService (), params["text"].asString ()); + HttpsClient::sendSMS (getApp().getIOService (), params["text"].asString ()); return "sms dispatched"; } Json::Value RPCHandler::doStop (Json::Value, LoadType* loadType, ScopedLock& MasterLockHolder) { - theApp->stop (); + getApp().stop (); return SYSTEM_NAME " server stopping"; } @@ -2738,7 +2730,7 @@ Json::Value RPCHandler::lookupLedger (Json::Value params, Ledger::pointer& lpLed break; case LEDGER_CLOSED: - lpLedger = theApp->getLedgerMaster ().getClosedLedger (); + lpLedger = getApp().getLedgerMaster ().getClosedLedger (); iLedgerIndex = lpLedger->getLedgerSeq (); assert (lpLedger->isImmutable () && lpLedger->isClosed ()); break; @@ -3308,7 +3300,7 @@ Json::Value RPCHandler::doSubscribe (Json::Value params, LoadType* loadType, Sco if (bSnapshot) { - Ledger::pointer lpLedger = theApp->getLedgerMaster ().getClosedLedger (); + Ledger::pointer lpLedger = getApp().getLedgerMaster ().getPublishedLedger (); const Json::Value jvMarker = Json::Value (Json::nullValue); if (bBoth) @@ -3526,7 +3518,7 @@ Json::Value RPCHandler::doUnsubscribe (Json::Value params, LoadType* loadType, S // // JSON-RPC provides a method and an array of params. JSON-RPC is used as a transport for a command and a request object. The // command is the method. The request object is supplied as the first element of the params. -Json::Value RPCHandler::doRpcCommand (const std::string& strMethod, Json::Value& jvParams, int iRole, LoadType* loadType) +Json::Value RPCHandler::doRpcCommand (const std::string& strMethod, Json::Value const& jvParams, int iRole, LoadType* loadType) { WriteLog (lsTRACE, RPCHandler) << "doRpcCommand:" << strMethod << ":" << jvParams; @@ -3571,7 +3563,7 @@ Json::Value RPCHandler::doCommand (const Json::Value& params, int iRole, LoadTyp { if (iRole != ADMIN) { - int jc = theApp->getJobQueue ().getJobCountGE (jtCLIENT); + int jc = getApp().getJobQueue ().getJobCountGE (jtCLIENT); if (jc > 500) { @@ -3680,7 +3672,7 @@ Json::Value RPCHandler::doCommand (const Json::Value& params, int iRole, LoadTyp return rpcError (rpcNO_PERMISSION); } - ScopedLock MasterLockHolder (theApp->getMasterLock ()); + ScopedLock MasterLockHolder (getApp().getMasterLock ()); if ((commandsA[i].iOptions & optNetwork) && (mNetOps->getOperatingMode () < NetworkOPs::omSYNCING)) { @@ -3689,7 +3681,7 @@ Json::Value RPCHandler::doCommand (const Json::Value& params, int iRole, LoadTyp return rpcError (rpcNO_NETWORK); } - if ((commandsA[i].iOptions & optCurrent) && (theApp->getLedgerMaster().getValidatedLedgerAge() > 120)) + if ((commandsA[i].iOptions & optCurrent) && (getApp().getLedgerMaster().getValidatedLedgerAge() > 120)) { return rpcError (rpcNO_CURRENT); } diff --git a/src/cpp/ripple/RPCHandler.h b/src/cpp/ripple/RPCHandler.h index c9298e07b2..f2931c6cad 100644 --- a/src/cpp/ripple/RPCHandler.h +++ b/src/cpp/ripple/RPCHandler.h @@ -34,7 +34,7 @@ public: Json::Value doCommand (const Json::Value& jvRequest, int role, LoadType* loadType); - Json::Value doRpcCommand (const std::string& strCommand, Json::Value& jvParams, int iRole, LoadType* loadType); + Json::Value doRpcCommand (const std::string& strCommand, Json::Value const& jvParams, int iRole, LoadType* loadType); private: typedef Json::Value (RPCHandler::*doFuncPtr) ( diff --git a/src/cpp/ripple/RPCServer.cpp b/src/cpp/ripple/RPCServer.cpp deleted file mode 100644 index b5671a665c..0000000000 --- a/src/cpp/ripple/RPCServer.cpp +++ /dev/null @@ -1,207 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RPC_MAXIMUM_QUERY -#define RPC_MAXIMUM_QUERY (1024*1024) -#endif - -SETUP_LOG (RPCServer) - -RPCServer::RPCServer (boost::asio::io_service& io_service, boost::asio::ssl::context& context, NetworkOPs* nopNetwork) - : mNetOps (nopNetwork), mSocket (io_service, context), mStrand(io_service) -{ - mRole = RPCHandler::GUEST; -} - -void RPCServer::connected () -{ - //std::cerr << "RPC request" << std::endl; - boost::asio::async_read_until (mSocket, mLineBuffer, "\r\n", - mStrand.wrap (boost::bind (&RPCServer::handle_read_line, shared_from_this (), boost::asio::placeholders::error))); -} - -void RPCServer::handle_read_req (const boost::system::error_code& e) -{ - std::string req; - - if (mLineBuffer.size ()) - { - req.assign (boost::asio::buffer_cast (mLineBuffer.data ()), mLineBuffer.size ()); - mLineBuffer.consume (mLineBuffer.size ()); - } - - req += strCopy (mQueryVec); - - if (!HTTPAuthorized (mHTTPRequest.peekHeaders ())) - mReplyStr = HTTPReply (403, "Forbidden"); - else - mReplyStr = handleRequest (req); - - boost::asio::async_write (mSocket, boost::asio::buffer (mReplyStr), - mStrand.wrap (boost::bind (&RPCServer::handle_write, shared_from_this (), boost::asio::placeholders::error))); -} - -void RPCServer::handle_read_line (const boost::system::error_code& e) -{ - if (e) - return; - - HTTPRequestAction action = mHTTPRequest.consume (mLineBuffer); - - if (action == haDO_REQUEST) - { - // request with no body - WriteLog (lsWARNING, RPCServer) << "RPC HTTP request with no body"; - mSocket.async_shutdown (mStrand.wrap (boost::bind (&RPCServer::handle_shutdown, shared_from_this(), boost::asio::placeholders::error))); - return; - } - else if (action == haREAD_LINE) - { - boost::asio::async_read_until (mSocket, mLineBuffer, "\r\n", - mStrand.wrap (boost::bind (&RPCServer::handle_read_line, shared_from_this (), - boost::asio::placeholders::error))); - } - else if (action == haREAD_RAW) - { - int rLen = mHTTPRequest.getDataSize (); - - if ((rLen < 0) || (rLen > RPC_MAXIMUM_QUERY)) - { - WriteLog (lsWARNING, RPCServer) << "Illegal RPC request length " << rLen; - mSocket.async_shutdown (mStrand.wrap (boost::bind (&RPCServer::handle_shutdown, shared_from_this(), boost::asio::placeholders::error))); - return; - } - - int alreadyHave = mLineBuffer.size (); - - if (alreadyHave < rLen) - { - mQueryVec.resize (rLen - alreadyHave); - boost::asio::async_read (mSocket, boost::asio::buffer (mQueryVec), - mStrand.wrap (boost::bind (&RPCServer::handle_read_req, shared_from_this (), boost::asio::placeholders::error))); - WriteLog (lsTRACE, RPCServer) << "Waiting for completed request: " << rLen; - } - else - { - // we have the whole thing - mQueryVec.resize (0); - handle_read_req (e); - } - } - else - mSocket.async_shutdown (mStrand.wrap (boost::bind (&RPCServer::handle_shutdown, shared_from_this(), boost::asio::placeholders::error))); -} - -std::string RPCServer::handleRequest (const std::string& requestStr) -{ - WriteLog (lsTRACE, RPCServer) << "handleRequest " << requestStr; - - Json::Value id; - - // Parse request - Json::Value jvRequest; - Json::Reader reader; - - if (!reader.parse (requestStr, jvRequest) || jvRequest.isNull () || !jvRequest.isObject ()) - return (HTTPReply (400, "unable to parse request")); - - // Parse id now so errors from here on will have the id - id = jvRequest["id"]; - - // Parse method - Json::Value valMethod = jvRequest["method"]; - - if (valMethod.isNull ()) - return (HTTPReply (400, "null method")); - - if (!valMethod.isString ()) - return (HTTPReply (400, "method is not string")); - - std::string strMethod = valMethod.asString (); - - // Parse params - Json::Value valParams = jvRequest["params"]; - - if (valParams.isNull ()) - { - valParams = Json::Value (Json::arrayValue); - } - else if (!valParams.isArray ()) - { - return HTTPReply (400, "params unparseable"); - } - - try - { - mRole = iAdminGet (jvRequest, mSocket.PlainSocket ().remote_endpoint ().address ().to_string ()); - } - catch (...) - { - // endpoint already disconnected - return ""; - } - - if (RPCHandler::FORBID == mRole) - { - // XXX This needs rate limiting to prevent brute forcing password. - return HTTPReply (403, "Forbidden"); - } - - RPCHandler mRPCHandler (mNetOps); - - WriteLog (lsINFO, RPCServer) << valParams; - LoadType loadType = LT_RPCReference; - Json::Value result = mRPCHandler.doRpcCommand (strMethod, valParams, mRole, &loadType); - - // VFALCO NOTE We discard loadType since there is no endpoint to punish - - WriteLog (lsINFO, RPCServer) << result; - - std::string strReply = JSONRPCReply (result, Json::Value (), id); - return HTTPReply (200, strReply); -} - -#if 0 -// now, expire, n -bool RPCServer::parseAcceptRate (const std::string& sAcceptRate) -{ - if (!sAcceptRate.compare ("expire")) - 0; - - return true; -} -#endif - - -void RPCServer::handle_write (const boost::system::error_code& e) -{ - //std::cerr << "async_write complete " << e << std::endl; - - if (!e) - { - HTTPRequestAction action = mHTTPRequest.requestDone (false); - - if (action == haCLOSE_CONN) - mSocket.async_shutdown (mStrand.wrap (boost::bind (&RPCServer::handle_shutdown, shared_from_this(), boost::asio::placeholders::error))); - else - { - boost::asio::async_read_until (mSocket, mLineBuffer, "\r\n", - mStrand.wrap (boost::bind (&RPCServer::handle_read_line, shared_from_this (), boost::asio::placeholders::error))); - } - } - - if (e != boost::asio::error::operation_aborted) - { - //connection_manager_.stop(shared_from_this()); - } -} - -// nothing to do, we just keep the object alive -void RPCServer::handle_shutdown (const boost::system::error_code& e) -{ -} - -// vim:ts=4 diff --git a/src/cpp/ripple/RPCServer.h b/src/cpp/ripple/RPCServer.h deleted file mode 100644 index d7cd1351a8..0000000000 --- a/src/cpp/ripple/RPCServer.h +++ /dev/null @@ -1,65 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef __RPCSERVER__ -#define __RPCSERVER__ - -class RPCServer : public boost::enable_shared_from_this -{ -public: - - typedef boost::shared_ptr pointer; - -private: - - NetworkOPs* mNetOps; - - AutoSocket mSocket; - boost::asio::io_service::strand mStrand; - - boost::asio::streambuf mLineBuffer; - Blob mQueryVec; - std::string mReplyStr; - - HTTPRequest mHTTPRequest; - - - int mRole; - - RPCServer (boost::asio::io_service& io_service, boost::asio::ssl::context& ssl_context, NetworkOPs* nopNetwork); - - RPCServer (const RPCServer&); // no implementation - RPCServer& operator= (const RPCServer&); // no implementation - - void handle_write (const boost::system::error_code& ec); - void handle_read_line (const boost::system::error_code& ec); - void handle_read_req (const boost::system::error_code& ec); - void handle_shutdown (const boost::system::error_code& ec); - - std::string handleRequest (const std::string& requestStr); - -public: - static pointer create (boost::asio::io_service& io_service, boost::asio::ssl::context& context, NetworkOPs* mNetOps) - { - return pointer (new RPCServer (io_service, context, mNetOps)); - } - - AutoSocket& getSocket () - { - return mSocket; - } - - boost::asio::ip::tcp::socket& getRawSocket () - { - return mSocket.PlainSocket (); - } - - void connected (); -}; - -#endif - -// vim:ts=4 diff --git a/src/cpp/ripple/RPCSub.cpp b/src/cpp/ripple/RPCSub.cpp index 9a1b8781c7..3f7e771a9c 100644 --- a/src/cpp/ripple/RPCSub.cpp +++ b/src/cpp/ripple/RPCSub.cpp @@ -75,7 +75,7 @@ void RPCSub::sendThread () WriteLog (lsINFO, RPCSub) << boost::str (boost::format ("callRPC calling: %s") % mIp); callRPC ( - theApp->getIOService (), + getApp().getIOService (), mIp, mPort, mUsername, mPassword, mPath, "event", @@ -112,7 +112,7 @@ void RPCSub::send (const Json::Value& jvObj, bool broadcast) mSending = true; WriteLog (lsINFO, RPCSub) << boost::str (boost::format ("callRPC start")); - boost::thread (boost::bind (&RPCSub::sendThread, this)).detach (); + boost::thread (BIND_TYPE (&RPCSub::sendThread, this)).detach (); } } diff --git a/src/cpp/ripple/RPCSub.h b/src/cpp/ripple/RPCSub.h index 0b941a305c..4f822ed12e 100644 --- a/src/cpp/ripple/RPCSub.h +++ b/src/cpp/ripple/RPCSub.h @@ -10,7 +10,9 @@ #define RPC_EVENT_QUEUE_MAX 32 // Subscription object for JSON-RPC -class RPCSub : public InfoSub +class RPCSub + : public InfoSub + , LeakChecked { public: typedef boost::shared_ptr pointer; diff --git a/src/cpp/ripple/RegularKeySetTransactor.cpp b/src/cpp/ripple/RegularKeySetTransactor.cpp index fae42620f1..e97a85cf30 100644 --- a/src/cpp/ripple/RegularKeySetTransactor.cpp +++ b/src/cpp/ripple/RegularKeySetTransactor.cpp @@ -22,7 +22,7 @@ uint64 RegularKeySetTransactor::calculateBaseFee () TER RegularKeySetTransactor::doApply () { - std::cerr << "RegularKeySet>" << std::endl; + Log::out() << "RegularKeySet>"; const uint32 uTxFlags = mTxn.getFlags (); @@ -50,7 +50,7 @@ TER RegularKeySetTransactor::doApply () mTxnAccount->makeFieldAbsent (sfRegularKey); } - std::cerr << "RegularKeySet<" << std::endl; + Log::out() << "RegularKeySet<"; return tesSUCCESS; } diff --git a/src/cpp/ripple/SerializedValidation.h b/src/cpp/ripple/SerializedValidation.h index a04fd16df4..e4aaed02e0 100644 --- a/src/cpp/ripple/SerializedValidation.h +++ b/src/cpp/ripple/SerializedValidation.h @@ -12,6 +12,8 @@ class SerializedValidation , public CountedObject { public: + static char const* getCountedObjectName () { return "SerializedValidation"; } + typedef boost::shared_ptr pointer; typedef const boost::shared_ptr& ref; diff --git a/src/cpp/ripple/Transaction.cpp b/src/cpp/ripple/Transaction.cpp index e3cc1e7c35..2ac415ec8d 100644 --- a/src/cpp/ripple/Transaction.cpp +++ b/src/cpp/ripple/Transaction.cpp @@ -45,7 +45,7 @@ Transaction::pointer Transaction::sharedTransaction (Blob const& vucTransaction, // Transaction::Transaction ( - TransactionType ttKind, + TxType ttKind, const RippleAddress& naPublicKey, const RippleAddress& naSourceAccount, uint32 uSeq, @@ -121,11 +121,6 @@ void Transaction::setStatus (TransStatus ts, uint32 lseq) mInLedger = lseq; } -void Transaction::save () -{ // This can destroy metadata, so don't do it - return; -} - Transaction::pointer Transaction::transactionFromSQL (Database* db, bool bValidate) { Serializer rawTxn; @@ -198,8 +193,8 @@ Transaction::pointer Transaction::transactionFromSQL (const std::string& sql) rawTxn.resize (txSize); { - ScopedLock sl (theApp->getTxnDB ()->getDBLock ()); - Database* db = theApp->getTxnDB ()->getDB (); + ScopedLock sl (getApp().getTxnDB ()->getDBLock ()); + Database* db = getApp().getTxnDB ()->getDB (); if (!db->executeSQL (sql, true) || !db->startIterRows ()) return Transaction::pointer (); @@ -332,7 +327,7 @@ Json::Value Transaction::getJson (int options, bool binary) const if (options == 1) { - Ledger::pointer ledger = theApp->getLedgerMaster ().getLedgerBySeq (mInLedger); + Ledger::pointer ledger = getApp().getLedgerMaster ().getLedgerBySeq (mInLedger); if (ledger) ret["date"] = ledger->getCloseTimeNC (); diff --git a/src/cpp/ripple/Transaction.h b/src/cpp/ripple/Transaction.h index 9543b177b4..b9949bb453 100644 --- a/src/cpp/ripple/Transaction.h +++ b/src/cpp/ripple/Transaction.h @@ -32,6 +32,8 @@ class Transaction , public CountedObject { public: + static char const* getCountedObjectName () { return "Transaction"; } + typedef boost::shared_ptr pointer; typedef const pointer& ref; @@ -42,7 +44,7 @@ public: static Transaction::pointer transactionFromSQL (Database * db, bool bValidate); Transaction ( - TransactionType ttKind, + TxType ttKind, const RippleAddress & naPublicKey, // To prove transaction is consistent and authorized. const RippleAddress & naSourceAccount, // To identify the paying account. uint32 uSeq, // To order transactions. @@ -132,9 +134,6 @@ public: mInLedger = ledger; } - // database functions - void save (); - bool operator< (const Transaction&) const; bool operator> (const Transaction&) const; bool operator== (const Transaction&) const; diff --git a/src/cpp/ripple/TransactionEngine.cpp b/src/cpp/ripple/TransactionEngine.cpp index 2db19eb2a1..687c220a54 100644 --- a/src/cpp/ripple/TransactionEngine.cpp +++ b/src/cpp/ripple/TransactionEngine.cpp @@ -65,7 +65,7 @@ TER TransactionEngine::applyTransaction (const SerializedTransaction& txn, Trans assert (mLedger); mNodes.init (mLedger, txn.getTransactionID (), mLedger->getLedgerSeq (), params); -#ifdef DEBUG +#ifdef BEAST_DEBUG if (1) { diff --git a/src/cpp/ripple/TransactionEngine.h b/src/cpp/ripple/TransactionEngine.h index 6d45bbb14f..d643811c63 100644 --- a/src/cpp/ripple/TransactionEngine.h +++ b/src/cpp/ripple/TransactionEngine.h @@ -15,6 +15,9 @@ class TransactionEngine : public CountedObject { +public: + static char const* getCountedObjectName () { return "TransactionEngine"; } + private: LedgerEntrySet mNodes; diff --git a/src/cpp/ripple/TransactionMaster.cpp b/src/cpp/ripple/TransactionMaster.cpp index 57a2d241d2..133fac7cab 100644 --- a/src/cpp/ripple/TransactionMaster.cpp +++ b/src/cpp/ripple/TransactionMaster.cpp @@ -49,7 +49,7 @@ SerializedTransaction::pointer TransactionMaster::fetch (SHAMapItem::ref item, S bool checkDisk, uint32 uCommitLedger) { SerializedTransaction::pointer txn; - Transaction::pointer iTx = theApp->getMasterTransaction ().fetch (item->getTag (), false); + Transaction::pointer iTx = getApp().getMasterTransaction ().fetch (item->getTag (), false); if (!iTx) { @@ -80,7 +80,7 @@ SerializedTransaction::pointer TransactionMaster::fetch (SHAMapItem::ref item, S return txn; } -bool TransactionMaster::canonicalize (Transaction::pointer& txn, bool may_be_new) +bool TransactionMaster::canonicalize (Transaction::pointer& txn) { uint256 tid = txn->getID (); @@ -90,9 +90,6 @@ bool TransactionMaster::canonicalize (Transaction::pointer& txn, bool may_be_new if (mCache.canonicalize (tid, txn)) return true; - if (may_be_new) - txn->save (); - return false; } // vim:ts=4 diff --git a/src/cpp/ripple/TransactionMaster.h b/src/cpp/ripple/TransactionMaster.h index c46539d1f8..4bfce624bc 100644 --- a/src/cpp/ripple/TransactionMaster.h +++ b/src/cpp/ripple/TransactionMaster.h @@ -9,7 +9,7 @@ // Tracks all transactions in memory -class TransactionMaster +class TransactionMaster : LeakChecked { public: TransactionMaster (); @@ -20,7 +20,7 @@ public: // return value: true = we had the transaction already bool inLedger (uint256 const& hash, uint32 ledger); - bool canonicalize (Transaction::pointer& txn, bool maybeNew); + bool canonicalize (Transaction::pointer& txn); void sweep (void) { mCache.sweep (); diff --git a/src/cpp/ripple/TransactionMeta.h b/src/cpp/ripple/TransactionMeta.h index ba7b287717..482c7b898d 100644 --- a/src/cpp/ripple/TransactionMeta.h +++ b/src/cpp/ripple/TransactionMeta.h @@ -7,7 +7,7 @@ #ifndef RIPPLE_TRANSACTIONMETA_H #define RIPPLE_TRANSACTIONMETA_H -class TransactionMetaSet +class TransactionMetaSet : LeakChecked { public: typedef boost::shared_ptr pointer; diff --git a/src/cpp/ripple/TransactionQueue.h b/src/cpp/ripple/TransactionQueue.h index ed4b400e06..15da038261 100644 --- a/src/cpp/ripple/TransactionQueue.h +++ b/src/cpp/ripple/TransactionQueue.h @@ -53,7 +53,7 @@ private: void addCallbacks (const TXQEntry& otherEntry); }; -class TXQueue +class TXQueue : LeakChecked { public: TXQueue () : mRunning (false) diff --git a/src/cpp/ripple/WSConnection.cpp b/src/cpp/ripple/WSConnection.cpp index 75799eb9b1..7da13ba995 100644 --- a/src/cpp/ripple/WSConnection.cpp +++ b/src/cpp/ripple/WSConnection.cpp @@ -4,5 +4,4 @@ */ //============================================================================== - -SETUP_LOG (WSConnectionLog) +SETUP_LOGN (WSConnectionLog,"WSConnection") diff --git a/src/cpp/ripple/WSConnection.h b/src/cpp/ripple/WSConnection.h index fbfa63aa9b..2056dc26a4 100644 --- a/src/cpp/ripple/WSConnection.h +++ b/src/cpp/ripple/WSConnection.h @@ -24,6 +24,8 @@ class WSConnection , public CountedObject > { public: + static char const* getCountedObjectName () { return "WSConnection"; } + typedef typename endpoint_type::connection_type connection; typedef typename boost::shared_ptr connection_ptr; typedef typename boost::weak_ptr weak_connection_ptr; @@ -35,7 +37,7 @@ public: // mConnection(connection_ptr()) { ; } WSConnection (WSServerHandler* wshpHandler, const connection_ptr& cpConnection) - : mHandler (wshpHandler), mConnection (cpConnection), mNetwork (theApp->getOPs ()), + : mHandler (wshpHandler), mConnection (cpConnection), mNetwork (getApp().getOPs ()), mRemoteIP (cpConnection->get_socket ().lowest_layer ().remote_endpoint ().address ().to_string ()), mLoadSource (mRemoteIP), mPingTimer (cpConnection->get_io_service ()), mPinged (false), mRcvQueueRunning (false), mDead (false) @@ -84,7 +86,7 @@ public: // Utilities Json::Value invokeCommand (Json::Value& jvRequest) { - if (theApp->getLoadManager ().shouldCutoff (mLoadSource)) + if (getApp().getLoadManager ().shouldCutoff (mLoadSource)) { // VFALCO TODO This must be implemented before open sourcing @@ -115,7 +117,7 @@ public: jvResult["id"] = jvRequest["id"]; } - theApp->getLoadManager ().applyLoadCharge (mLoadSource, LT_RPCInvalid); + getApp().getLoadManager ().applyLoadCharge (mLoadSource, LT_RPCInvalid); return jvResult; } @@ -139,8 +141,8 @@ public: // Debit/credit the load and see if we should include a warning. // - if (theApp->getLoadManager ().applyLoadCharge (mLoadSource, loadType) && - theApp->getLoadManager ().shouldWarn (mLoadSource)) + if (getApp().getLoadManager ().applyLoadCharge (mLoadSource, loadType) && + getApp().getLoadManager ().shouldWarn (mLoadSource)) { jvResult["warning"] = "load"; } diff --git a/src/cpp/ripple/WSDoor.cpp b/src/cpp/ripple/WSDoor.cpp index feb2667c87..ba6a3423e6 100644 --- a/src/cpp/ripple/WSDoor.cpp +++ b/src/cpp/ripple/WSDoor.cpp @@ -19,7 +19,8 @@ SETUP_LOG (WSDoor) // - We only talk to NetworkOPs (so we will work even in thin mode) // - NetworkOPs is smart enough to subscribe and or pass back messages // - +// VFALCO NOTE NetworkOPs isn't used here... +// void WSDoor::startListening () { setCallingThreadName ("websocket"); @@ -82,7 +83,7 @@ WSDoor* WSDoor::createWSDoor (const std::string& strIp, const int iPort, bool bP % strIp % iPort); - wdpResult->mThread = new boost::thread (boost::bind (&WSDoor::startListening, wdpResult)); + wdpResult->mThread = new boost::thread (BIND_TYPE (&WSDoor::startListening, wdpResult)); return wdpResult; } @@ -98,5 +99,3 @@ void WSDoor::stop () mThread->join (); } } - -// vim:ts=4 diff --git a/src/cpp/ripple/WSDoor.h b/src/cpp/ripple/WSDoor.h index f5df7dcf82..068cd61ce3 100644 --- a/src/cpp/ripple/WSDoor.h +++ b/src/cpp/ripple/WSDoor.h @@ -7,7 +7,7 @@ #ifndef RIPPLE_WSDOOR_RIPPLEHEADER #define RIPPLE_WSDOOR_RIPPLEHEADER -class WSDoor +class WSDoor : LeakChecked { private: websocketpp::server_autotls* mSEndpoint; diff --git a/src/cpp/ripple/WalletAddTransactor.cpp b/src/cpp/ripple/WalletAddTransactor.cpp index 91aeea9b4a..cd0d3bf3da 100644 --- a/src/cpp/ripple/WalletAddTransactor.cpp +++ b/src/cpp/ripple/WalletAddTransactor.cpp @@ -8,7 +8,7 @@ SETUP_LOG (WalletAddTransactor) TER WalletAddTransactor::doApply () { - std::cerr << "WalletAdd>" << std::endl; + Log::out() << "WalletAdd>"; Blob const vucPubKey = mTxn.getFieldVL (sfPublicKey); Blob const vucSignature = mTxn.getFieldVL (sfSignature); @@ -28,7 +28,7 @@ TER WalletAddTransactor::doApply () // FIXME: This should be moved to the transaction's signature check logic and cached if (!naMasterPubKey.accountPublicVerify (Serializer::getSHA512Half (uAuthKeyID.begin (), uAuthKeyID.size ()), vucSignature)) { - std::cerr << "WalletAdd: unauthorized: bad signature " << std::endl; + Log::out() << "WalletAdd: unauthorized: bad signature "; return tefBAD_ADD_AUTH; } @@ -37,7 +37,7 @@ TER WalletAddTransactor::doApply () if (sleDst) { - std::cerr << "WalletAdd: account already created" << std::endl; + Log::out() << "WalletAdd: account already created"; return tefCREATED; } @@ -71,7 +71,7 @@ TER WalletAddTransactor::doApply () sleDst->setFieldAmount (sfBalance, saDstAmount); sleDst->setFieldAccount (sfRegularKey, uAuthKeyID); - std::cerr << "WalletAdd<" << std::endl; + Log::out() << "WalletAdd<"; return tesSUCCESS; } diff --git a/src/cpp/ripple/ripple.proto b/src/cpp/ripple/ripple.proto index 920654c39c..de65a5ba5a 100644 --- a/src/cpp/ripple/ripple.proto +++ b/src/cpp/ripple/ripple.proto @@ -7,6 +7,7 @@ enum MessageType mtERROR_MSG = 2; mtPING = 3; mtPROOFOFWORK = 4; + mtCLUSTER = 5; // network presence detection mtGET_CONTACTS = 10; @@ -76,6 +77,22 @@ message TMHello optional bool testNet = 13; // Running as testnet. } +// The status of a node in our cluster +message TMClusterNode +{ + required string publicKey = 1; + required uint32 reportTime = 2; + required uint32 nodeLoad = 3; + optional string nodeName = 4; + optional string address = 5; +} + +// The status of all nodes in the cluster +message TMCluster +{ + repeated TMClusterNode clusterNodes = 1; +} + // A transaction can have only one input and one output. // If you want to send an amount that is greater than any single address of yours diff --git a/src/cpp/ripple/ripple_AcceptedLedgerTx.cpp b/src/cpp/ripple/ripple_AcceptedLedgerTx.cpp index 23346f01d9..383cc8a1bc 100644 --- a/src/cpp/ripple/ripple_AcceptedLedgerTx.cpp +++ b/src/cpp/ripple/ripple_AcceptedLedgerTx.cpp @@ -10,7 +10,7 @@ AcceptedLedgerTx::AcceptedLedgerTx (uint32 seq, SerializerIterator& sit) SerializerIterator txnIt (txnSer); mTxn = boost::make_shared (boost::ref (txnIt)); - mRawMeta = sit.getVL (); + mRawMeta = sit.getVL (); mMeta = boost::make_shared (mTxn->getTransactionID (), seq, mRawMeta); mAffected = mMeta->getAffectedAccounts (); mResult = mMeta->getResultTER (); diff --git a/src/cpp/ripple/ripple_AcceptedLedgerTx.h b/src/cpp/ripple/ripple_AcceptedLedgerTx.h index 2b14f84547..f9a5137b20 100644 --- a/src/cpp/ripple/ripple_AcceptedLedgerTx.h +++ b/src/cpp/ripple/ripple_AcceptedLedgerTx.h @@ -56,7 +56,7 @@ public: { return mTxn->getTransactionID (); } - TransactionType getTxnType () const + TxType getTxnType () const { return mTxn->getTxnType (); } diff --git a/src/cpp/ripple/ripple_AccountItems.h b/src/cpp/ripple/ripple_AccountItems.h index c94a9dd167..0c1f32427f 100644 --- a/src/cpp/ripple/ripple_AccountItems.h +++ b/src/cpp/ripple/ripple_AccountItems.h @@ -9,7 +9,7 @@ /** A set of AccountItem objects. */ -class AccountItems +class AccountItems : LeakChecked { public: typedef boost::shared_ptr pointer; diff --git a/src/cpp/ripple/ripple_AccountState.h b/src/cpp/ripple/ripple_AccountState.h index bd98a275aa..3a52d83853 100644 --- a/src/cpp/ripple/ripple_AccountState.h +++ b/src/cpp/ripple/ripple_AccountState.h @@ -11,7 +11,7 @@ // Provide abstract access to an account's state, such that access to the serialized format is hidden. // -class AccountState +class AccountState : LeakChecked { public: typedef boost::shared_ptr pointer; diff --git a/src/cpp/ripple/ripple_Application.cpp b/src/cpp/ripple/ripple_Application.cpp index dbce6a46c8..642fd8fe70 100644 --- a/src/cpp/ripple/ripple_Application.cpp +++ b/src/cpp/ripple/ripple_Application.cpp @@ -7,18 +7,83 @@ // VFALCO TODO Clean this global up volatile bool doShutdown = false; -// VFALCO TODO Wrap this up in something neater. -IApplication* theApp = nullptr; - class Application; SETUP_LOG (Application) // VFALCO TODO Move the function definitions into the class declaration -class Application : public IApplication +class Application + : public IApplication + , public SharedSingleton + , LeakChecked { public: - Application (); + static Application* createInstance () + { + return new Application; + } + + class Holder; + + Application () + // + // VFALCO NOTE Change this to control whether or not the Application + // object is destroyed on exit + // + #if 1 + // Application object will be deleted on exit. If the code doesn't exit + // cleanly this could cause hangs or crashes on exit. + // + : SharedSingleton (SingletonLifetime::persistAfterCreation) + #else + // This will make it so that the Application object is not deleted on exit. + // + : SharedSingleton (SingletonLifetime::neverDestroyed) + #endif + , mIOService ((theConfig.NODE_SIZE >= 2) ? 2 : 1) + , mIOWork (mIOService) + , mNetOps (&mLedgerMaster) + , m_rpcServerHandler (mNetOps) + , mTempNodeCache ("NodeCache", 16384, 90) + , m_nodeStore ( + theConfig.NODE_DB, + theConfig.FASTNODE_DB, + 16384, 300) + , mSLECache ("LedgerEntryCache", 4096, 120) + , mSNTPClient (mAuxService) + , mJobQueue (mIOService) + // VFALCO New stuff + , mFeatures (IFeatures::New (2 * 7 * 24 * 60 * 60, 200)) // two weeks, 200/256 + , mFeeVote (IFeeVote::New (10, 50 * SYSTEM_CURRENCY_PARTS, 12.5 * SYSTEM_CURRENCY_PARTS)) + , mFeeTrack (ILoadFeeTrack::New ()) + , mHashRouter (IHashRouter::New (IHashRouter::getDefaultHoldTime ())) + , mValidations (IValidations::New ()) + , mUNL (UniqueNodeList::New ()) + , mProofOfWorkFactory (IProofOfWorkFactory::New ()) + , mPeers (IPeers::New (mIOService)) + , m_loadManager (ILoadManager::New ()) + // VFALCO End new stuff + // VFALCO TODO replace all NULL with nullptr + , mRpcDB (NULL) + , mTxnDB (NULL) + , mLedgerDB (NULL) + , mWalletDB (NULL) // VFALCO NOTE are all these 'NULL' ctor params necessary? + , mNetNodeDB (NULL) + , mPathFindDB (NULL) + , mHashNodeDB (NULL) + , mHashNodeLDB (NULL) + , mEphemeralLDB (NULL) + , mPeerDoor (NULL) + , mRPCDoor (NULL) + , mWSPublicDoor (NULL) + , mWSPrivateDoor (NULL) + , mSweepTimer (mAuxService) + , mShutdown (false) + { + // VFALCO TODO remove these once the call is thread safe. + HashMaps::getInstance ().initializeNonce (); + } + ~Application (); LocalCredentials& getLocalCredentials () @@ -36,11 +101,6 @@ public: return mIOService; } - boost::asio::io_service& getAuxService () - { - return mAuxService; - } - LedgerMaster& getLedgerMaster () { return mLedgerMaster; @@ -61,9 +121,9 @@ public: return mTempNodeCache; } - HashedObjectStore& getHashedObjectStore () + NodeStore& getNodeStore () { - return mHashedObjectStore; + return m_nodeStore; } JobQueue& getJobQueue () @@ -126,7 +186,7 @@ public: return *mValidations; } - IUniqueNodeList& getUNL () + UniqueNodeList& getUNL () { return *mUNL; } @@ -199,15 +259,18 @@ public: void sweep (); private: - void updateTables (bool); + void updateTables (); void startNewLedger (); bool loadOldLedger (const std::string&); private: boost::asio::io_service mIOService; boost::asio::io_service mAuxService; + // The lifetime of the io_service::work object informs the io_service + // of when the work starts and finishes. io_service::run() will not exit + // while the work object exists. + // boost::asio::io_service::work mIOWork; - boost::asio::io_service::work mAuxWork; boost::recursive_mutex mMasterLock; @@ -216,8 +279,9 @@ private: InboundLedgers m_inboundLedgers; TransactionMaster mMasterTransaction; NetworkOPs mNetOps; + RPCServerHandler m_rpcServerHandler; NodeCache mTempNodeCache; - HashedObjectStore mHashedObjectStore; + NodeStore m_nodeStore; SLECache mSLECache; SNTPClient mSNTPClient; JobQueue mJobQueue; @@ -230,7 +294,7 @@ private: beast::ScopedPointer mFeeTrack; beast::ScopedPointer mHashRouter; beast::ScopedPointer mValidations; - beast::ScopedPointer mUNL; + beast::ScopedPointer mUNL; beast::ScopedPointer mProofOfWorkFactory; beast::ScopedPointer mPeers; beast::ScopedPointer m_loadManager; @@ -248,64 +312,30 @@ private: leveldb::DB* mHashNodeLDB; leveldb::DB* mEphemeralLDB; - PeerDoor* mPeerDoor; - RPCDoor* mRPCDoor; - WSDoor* mWSPublicDoor; - WSDoor* mWSPrivateDoor; + ScopedPointer mPeerDoor; + ScopedPointer mRPCDoor; + ScopedPointer mWSPublicDoor; + ScopedPointer mWSPrivateDoor; boost::asio::deadline_timer mSweepTimer; bool volatile mShutdown; }; -Application::Application () - : mIOService ((theConfig.NODE_SIZE >= 2) ? 2 : 1) - , mIOWork (mIOService) - , mAuxWork (mAuxService) - , mNetOps (mIOService, &mLedgerMaster) - , mTempNodeCache ("NodeCache", 16384, 90) - , mHashedObjectStore (16384, 300) - , mSLECache ("LedgerEntryCache", 4096, 120) - , mSNTPClient (mAuxService) - , mJobQueue (mIOService) - // VFALCO New stuff - , mFeatures (IFeatures::New (2 * 7 * 24 * 60 * 60, 200)) // two weeks, 200/256 - , mFeeVote (IFeeVote::New (10, 50 * SYSTEM_CURRENCY_PARTS, 12.5 * SYSTEM_CURRENCY_PARTS)) - , mFeeTrack (ILoadFeeTrack::New ()) - , mHashRouter (IHashRouter::New (IHashRouter::getDefaultHoldTime ())) - , mValidations (IValidations::New ()) - , mUNL (IUniqueNodeList::New (mIOService)) - , mProofOfWorkFactory (IProofOfWorkFactory::New ()) - , mPeers (IPeers::New (mIOService)) - , m_loadManager (ILoadManager::New ()) - // VFALCO End new stuff - // VFALCO TODO replace all NULL with nullptr - , mRpcDB (NULL) - , mTxnDB (NULL) - , mLedgerDB (NULL) - , mWalletDB (NULL) // VFALCO NOTE are all these 'NULL' ctor params necessary? - , mNetNodeDB (NULL) - , mPathFindDB (NULL) - , mHashNodeDB (NULL) - , mHashNodeLDB (NULL) - , mEphemeralLDB (NULL) - , mPeerDoor (NULL) - , mRPCDoor (NULL) - , mWSPublicDoor (NULL) - , mWSPrivateDoor (NULL) - , mSweepTimer (mAuxService) - , mShutdown (false) +Application::~Application () { - // VFALCO TODO remove these once the call is thread safe. - HashMaps::getInstance ().initializeNonce (); -} + // VFALCO TODO Wrap these in ScopedPointer + delete mTxnDB; + delete mLedgerDB; + delete mWalletDB; + delete mHashNodeDB; + delete mNetNodeDB; + delete mPathFindDB; + delete mHashNodeLDB; -// VFALCO TODO Tidy these up into some class with accessors. -// -extern const char* RpcDBInit[], *TxnDBInit[], *LedgerDBInit[], *WalletDBInit[], *HashNodeDBInit[], - *NetNodeDBInit[], *PathFindDBInit[]; -extern int RpcDBCount, TxnDBCount, LedgerDBCount, WalletDBCount, HashNodeDBCount, - NetNodeDBCount, PathFindDBCount; + if (mEphemeralLDB != nullptr) + delete mEphemeralLDB; +} void Application::stop () { @@ -313,7 +343,7 @@ void Application::stop () StopSustain (); mShutdown = true; mIOService.stop (); - mHashedObjectStore.waitWrite (); + m_nodeStore.waitWrite (); mValidations->flush (); mAuxService.stop (); mJobQueue.shutdown (); @@ -325,6 +355,7 @@ void Application::stop () mEphemeralLDB = NULL; WriteLog (lsINFO, Application) << "Stopped: " << mIOService.stopped (); + mShutdown = false; } static void InitDB (DatabaseCon** dbCon, const char* fileName, const char* dbInit[], int dbCount) @@ -362,11 +393,11 @@ void Application::setup () mJobQueue.setThreadCount (0, theConfig.RUN_STANDALONE); mSweepTimer.expires_from_now (boost::posix_time::seconds (10)); - mSweepTimer.async_wait (boost::bind (&Application::sweep, this)); + mSweepTimer.async_wait (BIND_TYPE (&Application::sweep, this)); m_loadManager->startThread (); -#ifndef WIN32 +#if ! BEAST_WIN32 #ifdef SIGINT if (!theConfig.RUN_STANDALONE) @@ -384,14 +415,14 @@ void Application::setup () if (!theConfig.DEBUG_LOGFILE.empty ()) { - // Let DEBUG messages go to the file but only WARNING or higher to regular output (unless verbose) + // Let BEAST_DEBUG messages go to the file but only WARNING or higher to regular output (unless verbose) Log::setLogFile (theConfig.DEBUG_LOGFILE); if (Log::getMinSeverity () > lsDEBUG) LogPartition::setSeverity (lsDEBUG); } - boost::thread (boost::bind (runAux, boost::ref (mAuxService))).detach (); + boost::thread (BIND_TYPE (runAux, boost::ref (mAuxService))).detach (); if (!theConfig.RUN_STANDALONE) mSNTPClient.init (theConfig.SNTP_SERVERS); @@ -399,16 +430,16 @@ void Application::setup () // // Construct databases. // - boost::thread t1 (boost::bind (&InitDB, &mRpcDB, "rpc.db", RpcDBInit, RpcDBCount)); - boost::thread t2 (boost::bind (&InitDB, &mTxnDB, "transaction.db", TxnDBInit, TxnDBCount)); - boost::thread t3 (boost::bind (&InitDB, &mLedgerDB, "ledger.db", LedgerDBInit, LedgerDBCount)); + boost::thread t1 (BIND_TYPE (&InitDB, &mRpcDB, "rpc.db", RpcDBInit, RpcDBCount)); + boost::thread t2 (BIND_TYPE (&InitDB, &mTxnDB, "transaction.db", TxnDBInit, TxnDBCount)); + boost::thread t3 (BIND_TYPE (&InitDB, &mLedgerDB, "ledger.db", LedgerDBInit, LedgerDBCount)); t1.join (); t2.join (); t3.join (); - boost::thread t4 (boost::bind (&InitDB, &mWalletDB, "wallet.db", WalletDBInit, WalletDBCount)); - boost::thread t6 (boost::bind (&InitDB, &mNetNodeDB, "netnode.db", NetNodeDBInit, NetNodeDBCount)); - boost::thread t7 (boost::bind (&InitDB, &mPathFindDB, "pathfind.db", PathFindDBInit, PathFindDBCount)); + boost::thread t4 (BIND_TYPE (&InitDB, &mWalletDB, "wallet.db", WalletDBInit, WalletDBCount)); + boost::thread t6 (BIND_TYPE (&InitDB, &mNetNodeDB, "netnode.db", NetNodeDBInit, NetNodeDBCount)); + boost::thread t7 (BIND_TYPE (&InitDB, &mPathFindDB, "pathfind.db", PathFindDBInit, PathFindDBCount)); t4.join (); t6.join (); t7.join (); @@ -417,51 +448,16 @@ void Application::setup () options.create_if_missing = true; options.block_cache = leveldb::NewLRUCache (theConfig.getSize (siHashNodeDBCache) * 1024 * 1024); - if (theConfig.NODE_SIZE >= 2) - options.filter_policy = leveldb::NewBloomFilterPolicy (10); - - if (theConfig.LDB_IMPORT) - options.write_buffer_size = 32 << 20; - - if (mHashedObjectStore.isLevelDB ()) - { - WriteLog (lsINFO, Application) << "LevelDB used for nodes"; - leveldb::Status status = leveldb::DB::Open (options, (theConfig.DATA_DIR / "hashnode").string (), &mHashNodeLDB); - - if (!status.ok () || !mHashNodeLDB) - { - WriteLog (lsFATAL, Application) << "Unable to open/create hash node db: " - << (theConfig.DATA_DIR / "hashnode").string () - << " " << status.ToString (); - StopSustain (); - exit (3); - } - } - else - { - WriteLog (lsINFO, Application) << "SQLite used for nodes"; - boost::thread t5 (boost::bind (&InitDB, &mHashNodeDB, "hashnode.db", HashNodeDBInit, HashNodeDBCount)); - t5.join (); - } - - if (!theConfig.LDB_EPHEMERAL.empty ()) - { - leveldb::Status status = leveldb::DB::Open (options, theConfig.LDB_EPHEMERAL, &mEphemeralLDB); - - if (!status.ok () || !mEphemeralLDB) - { - WriteLog (lsFATAL, Application) << "Unable to open/create epehemeral db: " - << theConfig.LDB_EPHEMERAL << " " << status.ToString (); - StopSustain (); - exit (3); - } - } + getApp().getLedgerDB ()->getDB ()->executeSQL (boost::str (boost::format ("PRAGMA cache_size=-%d;") % + (theConfig.getSize (siLgrDBCache) * 1024))); + getApp().getTxnDB ()->getDB ()->executeSQL (boost::str (boost::format ("PRAGMA cache_size=-%d;") % + (theConfig.getSize (siTxnDBCache) * 1024))); mTxnDB->getDB ()->setupCheckpointing (&mJobQueue); mLedgerDB->getDB ()->setupCheckpointing (&mJobQueue); if (!theConfig.RUN_STANDALONE) - updateTables (theConfig.LDB_IMPORT); + updateTables (); mFeatures->addInitialFeatures (); @@ -477,7 +473,7 @@ void Application::setup () if (!loadOldLedger (theConfig.START_LEDGER)) { - theApp->stop (); + getApp().stop (); exit (-1); } } @@ -492,7 +488,7 @@ void Application::setup () else startNewLedger (); - mOrderBookDB.setup (theApp->getLedgerMaster ().getCurrentLedger ()); + mOrderBookDB.setup (getApp().getLedgerMaster ().getCurrentLedger ()); // // Begin validation and ip maintenance. @@ -507,22 +503,13 @@ void Application::setup () getUNL ().nodeBootstrap (); mValidations->tune (theConfig.getSize (siValidationsSize), theConfig.getSize (siValidationsAge)); - mHashedObjectStore.tune (theConfig.getSize (siNodeCacheSize), theConfig.getSize (siNodeCacheAge)); + m_nodeStore.tune (theConfig.getSize (siNodeCacheSize), theConfig.getSize (siNodeCacheAge)); mLedgerMaster.tune (theConfig.getSize (siLedgerSize), theConfig.getSize (siLedgerAge)); mSLECache.setTargetSize (theConfig.getSize (siSLECacheSize)); mSLECache.setTargetAge (theConfig.getSize (siSLECacheAge)); mLedgerMaster.setMinValidations (theConfig.VALIDATION_QUORUM); - if (!mHashedObjectStore.isLevelDB ()) - theApp->getHashNodeDB ()->getDB ()->executeSQL (boost::str (boost::format ("PRAGMA cache_size=-%d;") % - (theConfig.getSize (siHashNodeDBCache) * 1024))); - - theApp->getLedgerDB ()->getDB ()->executeSQL (boost::str (boost::format ("PRAGMA cache_size=-%d;") % - (theConfig.getSize (siLgrDBCache) * 1024))); - theApp->getTxnDB ()->getDB ()->executeSQL (boost::str (boost::format ("PRAGMA cache_size=-%d;") % - (theConfig.getSize (siTxnDBCache) * 1024))); - // // Allow peer connections. // @@ -530,7 +517,11 @@ void Application::setup () { try { - mPeerDoor = new PeerDoor (mIOService); + mPeerDoor = PeerDoor::New ( + theConfig.PEER_IP, + theConfig.PEER_PORT, + theConfig.PEER_SSL_CIPHER_LIST, + mIOService); } catch (const std::exception& e) { @@ -548,11 +539,11 @@ void Application::setup () // // Allow RPC connections. // - if (!theConfig.RPC_IP.empty () && theConfig.RPC_PORT) + if (! theConfig.getRpcIP().empty () && theConfig.getRpcPort() != 0) { try { - mRPCDoor = new RPCDoor (mIOService); + mRPCDoor = new RPCDoor (mIOService, m_rpcServerHandler); } catch (const std::exception& e) { @@ -635,7 +626,7 @@ void Application::run () { if (theConfig.NODE_SIZE >= 2) { - boost::thread (boost::bind (runIO, boost::ref (mIOService))).detach (); + boost::thread (BIND_TYPE (runIO, boost::ref (mIOService))).detach (); } if (!theConfig.RUN_STANDALONE) @@ -643,7 +634,7 @@ void Application::run () // VFALCO NOTE This seems unnecessary. If we properly refactor the load // manager then the deadlock detector can just always be "armed" // - theApp->getLoadManager ().activateDeadlockDetector (); + getApp().getLoadManager ().activateDeadlockDetector (); } mIOService.run (); // This blocks @@ -654,7 +645,20 @@ void Application::run () if (mWSPrivateDoor) mWSPrivateDoor->stop (); + // VFALCO TODO Try to not have to do this early, by using observers to + // eliminate LoadManager's dependency inversions. + // + // This deletes the object and therefore, stops the thread. + m_loadManager = nullptr; + + mSweepTimer.cancel(); + WriteLog (lsINFO, Application) << "Done."; + + // VFALCO NOTE This is a sign that something is wrong somewhere, it + // shouldn't be necessary to sleep until some flag is set. + while (mShutdown) + boost::this_thread::sleep (boost::posix_time::milliseconds (100)); } void Application::sweep () @@ -666,7 +670,7 @@ void Application::sweep () if (space.available < (512 * 1024 * 1024)) { WriteLog (lsFATAL, Application) << "Remaining free disk space is less than 512MB"; - theApp->stop (); + getApp().stop (); } // VFALCO NOTE Does the order of calls matter? @@ -674,7 +678,7 @@ void Application::sweep () // have listeners register for "onSweep ()" notification. // mMasterTransaction.sweep (); - mHashedObjectStore.sweep (); + m_nodeStore.sweep (); mLedgerMaster.sweep (); mTempNodeCache.sweep (); mValidations->sweep (); @@ -685,22 +689,7 @@ void Application::sweep () mNetOps.sweepFetchPack (); // VFALCO NOTE does the call to sweep() happen on another thread? mSweepTimer.expires_from_now (boost::posix_time::seconds (theConfig.getSize (siSweepInterval))); - mSweepTimer.async_wait (boost::bind (&Application::sweep, this)); -} - -Application::~Application () -{ - // VFALCO TODO Wrap these in ScopedPointer - delete mTxnDB; - delete mLedgerDB; - delete mWalletDB; - delete mHashNodeDB; - delete mNetNodeDB; - delete mPathFindDB; - delete mHashNodeLDB; - - if (mEphemeralLDB != nullptr) - delete mEphemeralLDB; + mSweepTimer.async_wait (BIND_TYPE (&Application::sweep, this)); } void Application::startNewLedger () @@ -805,40 +794,34 @@ bool serverOkay (std::string& reason) if (!theConfig.ELB_SUPPORT) return true; - if (!theApp) - { - reason = "Server has not started"; - return false; - } - - if (theApp->isShutdown ()) + if (getApp().isShutdown ()) { reason = "Server is shutting down"; return false; } - if (theApp->getOPs ().isNeedNetworkLedger ()) + if (getApp().getOPs ().isNeedNetworkLedger ()) { reason = "Not synchronized with network yet"; return false; } - if (theApp->getOPs ().getOperatingMode () < NetworkOPs::omSYNCING) + if (getApp().getOPs ().getOperatingMode () < NetworkOPs::omSYNCING) { reason = "Not synchronized with network"; return false; } - if (!theApp->getLedgerMaster().isCaughtUp(reason)) + if (!getApp().getLedgerMaster().isCaughtUp(reason)) return false; - if (theApp->getFeeTrack ().isLoaded ()) + if (getApp().getFeeTrack ().isLoadedLocal ()) { reason = "Too much load"; return false; } - if (theApp->getOPs ().isFeatureBlocked ()) + if (getApp().getOPs ().isFeatureBlocked ()) { reason = "Server version too old"; return false; @@ -881,12 +864,12 @@ static bool schemaHas (DatabaseCon* dbc, const std::string& dbName, int line, co static void addTxnSeqField () { - if (schemaHas (theApp->getTxnDB (), "AccountTransactions", 0, "TxnSeq")) + if (schemaHas (getApp().getTxnDB (), "AccountTransactions", 0, "TxnSeq")) return; Log (lsWARNING) << "Transaction sequence field is missing"; - Database* db = theApp->getTxnDB ()->getDB (); + Database* db = getApp().getTxnDB ()->getDB (); std::vector< std::pair > txIDs; txIDs.reserve (300000); @@ -953,44 +936,40 @@ static void addTxnSeqField () db->executeSQL ("END TRANSACTION;"); } -void Application::updateTables (bool ldbImport) +void Application::updateTables () { + if (theConfig.NODE_DB.empty ()) + { + Log (lsFATAL) << "The NODE_DB configuration setting MUST be set"; + StopSustain (); + exit (1); + } + else if (theConfig.NODE_DB == "LevelDB" || theConfig.NODE_DB == "SQLite") + { + Log (lsFATAL) << "The NODE_DB setting has been updated, your value is out of date"; + StopSustain (); + exit (1); + } + // perform any needed table updates - assert (schemaHas (theApp->getTxnDB (), "AccountTransactions", 0, "TransID")); - assert (!schemaHas (theApp->getTxnDB (), "AccountTransactions", 0, "foobar")); + assert (schemaHas (getApp().getTxnDB (), "AccountTransactions", 0, "TransID")); + assert (!schemaHas (getApp().getTxnDB (), "AccountTransactions", 0, "foobar")); addTxnSeqField (); - if (schemaHas (theApp->getTxnDB (), "AccountTransactions", 0, "PRIMARY")) + if (schemaHas (getApp().getTxnDB (), "AccountTransactions", 0, "PRIMARY")) { Log (lsFATAL) << "AccountTransactions database should not have a primary key"; StopSustain (); exit (1); } - if (theApp->getHashedObjectStore ().isLevelDB ()) - { - boost::filesystem::path hashPath = theConfig.DATA_DIR / "hashnode.db"; - - if (boost::filesystem::exists (hashPath)) - { - if (theConfig.LDB_IMPORT) - { - Log (lsWARNING) << "Importing SQLite -> LevelDB"; - theApp->getHashedObjectStore ().import (hashPath.string ()); - Log (lsWARNING) << "Remove or remname the hashnode.db file"; - } - else - { - Log (lsWARNING) << "SQLite hashnode database exists. Please either remove or import"; - Log (lsWARNING) << "To import, start with the '--import' option. Otherwise, remove hashnode.db"; - StopSustain (); - exit (1); - } - } - } + if (!theConfig.DB_IMPORT.empty()) + getApp().getNodeStore().import(theConfig.DB_IMPORT); } -IApplication* IApplication::New () +//------------------------------------------------------------------------------ + +IApplication& getApp () { - return new Application; + return *Application::getInstance (); } diff --git a/src/cpp/ripple/ripple_CanonicalTXSet.h b/src/cpp/ripple/ripple_CanonicalTXSet.h index f10e660aeb..953422d568 100644 --- a/src/cpp/ripple/ripple_CanonicalTXSet.h +++ b/src/cpp/ripple/ripple_CanonicalTXSet.h @@ -15,7 +15,7 @@ */ // VFALCO TODO rename to SortedTxSet -class CanonicalTXSet +class CanonicalTXSet : LeakChecked { public: class Key diff --git a/src/cpp/ripple/ripple_ClusterNodeStatus.h b/src/cpp/ripple/ripple_ClusterNodeStatus.h new file mode 100644 index 0000000000..6520e59c55 --- /dev/null +++ b/src/cpp/ripple/ripple_ClusterNodeStatus.h @@ -0,0 +1,55 @@ + + + +#ifndef RIPPLE_CLUSTERNODESTATUS_H +#define RIPPLE_CLUSTERNODESTATUS_H + +class ClusterNodeStatus +{ +public: + + ClusterNodeStatus() : mLoadFee(0), mReportTime(0) + { ; } + + ClusterNodeStatus(std::string const& name) : mNodeName(name), mLoadFee(0), mReportTime(0) + { ; } + + ClusterNodeStatus(const std::string& name, uint32 fee, uint32 rtime) : + mNodeName(name), + mLoadFee(fee), + mReportTime(rtime) + { ; } + + std::string const& getName() + { + return mNodeName; + } + + uint32 getLoadFee() + { + return mLoadFee; + } + + uint32 getReportTime() + { + return mReportTime; + } + + bool update(ClusterNodeStatus const& status) + { + if (status.mReportTime <= mReportTime) + return false; + mLoadFee = status.mLoadFee; + mReportTime = status.mReportTime; + if (mNodeName.empty() || !status.mNodeName.empty()) + mNodeName = status.mNodeName; + return true; + } + +private: + std::string mNodeName; + uint32 mLoadFee; + uint32 mReportTime; +}; + +#endif diff --git a/src/cpp/ripple/ripple_DBInit.cpp b/src/cpp/ripple/ripple_DBInit.cpp index bd6cbbd79b..8639fe035b 100644 --- a/src/cpp/ripple/ripple_DBInit.cpp +++ b/src/cpp/ripple/ripple_DBInit.cpp @@ -250,6 +250,7 @@ const char* WalletDBInit[] = // Score: // Computed trust score. Higher is better. // Source: + // 'C' = Configuration file // 'V' = Validation file // 'M' = Manually added. // 'I' = Inbound connection. diff --git a/src/cpp/ripple/ripple_DBInit.h b/src/cpp/ripple/ripple_DBInit.h new file mode 100644 index 0000000000..d6111f9612 --- /dev/null +++ b/src/cpp/ripple/ripple_DBInit.h @@ -0,0 +1,30 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_DBINIT_H_INCLUDED +#define RIPPLE_DBINIT_H_INCLUDED + +// VFALCO TODO Tidy these up into a class with functions and return types. +extern const char* RpcDBInit[]; +extern const char* TxnDBInit[]; +extern const char* LedgerDBInit[]; +extern const char* WalletDBInit[]; +extern const char* HashNodeDBInit[]; + +// VFALCO TODO Figure out what these counts are for +extern int RpcDBCount; +extern int TxnDBCount; +extern int LedgerDBCount; +extern int WalletDBCount; +extern int HashNodeDBCount; + +// VFALCO TODO Seems these two aren't used so delete EVERYTHING. +extern const char* NetNodeDBInit[]; +extern const char* PathFindDBInit[]; +extern int NetNodeDBCount; +extern int PathFindDBCount; + +#endif diff --git a/src/cpp/ripple/ripple_Database.cpp b/src/cpp/ripple/ripple_Database.cpp index d52ac30abe..76ca764a04 100644 --- a/src/cpp/ripple/ripple_Database.cpp +++ b/src/cpp/ripple/ripple_Database.cpp @@ -4,11 +4,10 @@ */ //============================================================================== -Database::Database (const char* host, const char* user, const char* pass) : mNumCol (0) +Database::Database (const char* host) + : mNumCol (0) { - mDBPass = pass; mHost = host; - mUser = user; } Database::~Database () diff --git a/src/cpp/ripple/ripple_Database.h b/src/cpp/ripple/ripple_Database.h index 078b5f5068..22a8b8398f 100644 --- a/src/cpp/ripple/ripple_Database.h +++ b/src/cpp/ripple/ripple_Database.h @@ -26,9 +26,7 @@ class JobQueue; class Database { public: - // VFALCO TODO how are user and password even used? - // - Database (const char* host, const char* user, const char* pass); + explicit Database (const char* host); virtual ~Database (); @@ -36,11 +34,6 @@ public: virtual void disconnect () = 0; - std::string& getPass () - { - return (mDBPass); - } - // returns true if the query went ok virtual bool executeSQL (const char* sql, bool fail_okay = false) = 0; @@ -105,9 +98,7 @@ protected: bool getColNumber (const char* colName, int* retIndex); int mNumCol; - std::string mUser; std::string mHost; - std::string mDBPass; std::vector mColNameTable; }; diff --git a/src/cpp/ripple/ripple_DatabaseCon.h b/src/cpp/ripple/ripple_DatabaseCon.h index d9bbdcca25..e1b473a791 100644 --- a/src/cpp/ripple/ripple_DatabaseCon.h +++ b/src/cpp/ripple/ripple_DatabaseCon.h @@ -9,7 +9,7 @@ // VFALCO NOTE This looks like a pointless class. Figure out // what purpose it is really trying to serve and do it better. -class DatabaseCon +class DatabaseCon : LeakChecked { public: DatabaseCon (const std::string& name, const char* initString[], int countInit); diff --git a/src/cpp/ripple/ripple_Features.cpp b/src/cpp/ripple/ripple_Features.cpp index 6807089450..924c7b656e 100644 --- a/src/cpp/ripple/ripple_Features.cpp +++ b/src/cpp/ripple/ripple_Features.cpp @@ -94,8 +94,8 @@ FeatureState* Features::getCreateFeature (uint256 const& featureHash, bool creat query.append (featureHash.GetHex ()); query.append ("';"); - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); - Database* db = theApp->getWalletDB ()->getDB (); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); if (db->executeSQL (query) && db->startIterRows ()) { @@ -327,8 +327,8 @@ void Features::reportValidations (const FeatureSet& set) if (!changedFeatures.empty ()) { - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); - Database* db = theApp->getWalletDB ()->getDB (); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); db->executeSQL ("BEGIN TRANSACTION;"); BOOST_FOREACH (uint256 const & hash, changedFeatures) diff --git a/src/cpp/ripple/ripple_FeeVote.cpp b/src/cpp/ripple/ripple_FeeVote.cpp index 8f6c2ad9dd..cf7a8753df 100644 --- a/src/cpp/ripple/ripple_FeeVote.cpp +++ b/src/cpp/ripple/ripple_FeeVote.cpp @@ -105,7 +105,7 @@ public: VotableInteger incReserveVote (lastClosedLedger->getReserveInc (), mTargetReserveIncrement); // get validations for ledger before flag - ValidationSet set = theApp->getValidations ().getValidations (lastClosedLedger->getParentHash ()); + ValidationSet set = getApp().getValidations ().getValidations (lastClosedLedger->getParentHash ()); BOOST_FOREACH (ValidationSet::value_type const & value, set) { SerializedValidation const& val = *value.second; diff --git a/src/cpp/ripple/ripple_HashRouter.cpp b/src/cpp/ripple/ripple_HashRouter.cpp index 22cbea8d91..c11c824e1e 100644 --- a/src/cpp/ripple/ripple_HashRouter.cpp +++ b/src/cpp/ripple/ripple_HashRouter.cpp @@ -13,6 +13,8 @@ private: class Entry : public CountedObject { public: + static char const* getCountedObjectName () { return "HashRouterEntry"; } + Entry () : mFlags (0) { diff --git a/src/cpp/ripple/ripple_HashedObjectStore.cpp b/src/cpp/ripple/ripple_HashedObjectStore.cpp deleted file mode 100644 index 8c9b9fc45e..0000000000 --- a/src/cpp/ripple/ripple_HashedObjectStore.cpp +++ /dev/null @@ -1,621 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -HashedObjectStore::HashedObjectStore (int cacheSize, int cacheAge) : - mCache ("HashedObjectStore", cacheSize, cacheAge), mNegativeCache ("HashedObjectNegativeCache", 0, 120), - mWriteGeneration (0), mWriteLoad (0), mWritePending (false), mLevelDB (false), mEphemeralDB (false) -{ - mWriteSet.reserve (128); - - if (theConfig.NODE_DB == "leveldb" || theConfig.NODE_DB == "LevelDB") - mLevelDB = true; - else if (theConfig.NODE_DB == "SQLite" || theConfig.NODE_DB == "sqlite") - mLevelDB = false; - else - { - WriteLog (lsFATAL, HashedObject) << "Incorrect database selection"; - assert (false); - } - - if (!theConfig.LDB_EPHEMERAL.empty ()) - mEphemeralDB = true; -} - -void HashedObjectStore::tune (int size, int age) -{ - mCache.setTargetSize (size); - mCache.setTargetAge (age); -} - -void HashedObjectStore::waitWrite () -{ - boost::mutex::scoped_lock sl (mWriteMutex); - int gen = mWriteGeneration; - - while (mWritePending && (mWriteGeneration == gen)) - mWriteCondition.wait (sl); -} - -int HashedObjectStore::getWriteLoad () -{ - boost::mutex::scoped_lock sl (mWriteMutex); - return std::max (mWriteLoad, static_cast (mWriteSet.size ())); -} - -// low-level retrieve -HashedObject::pointer HashedObjectStore::LLRetrieve (uint256 const& hash, leveldb::DB* db) -{ - std::string sData; - - leveldb::Status st = db->Get (leveldb::ReadOptions (), - leveldb::Slice (reinterpret_cast (hash.begin ()), hash.size ()), &sData); - - if (!st.ok ()) - { - assert (st.IsNotFound ()); - return HashedObject::pointer (); - } - - const unsigned char* bufPtr = reinterpret_cast (&sData[0]); - uint32 index = htonl (*reinterpret_cast (bufPtr)); - int htype = bufPtr[8]; - - return boost::make_shared (static_cast (htype), index, - bufPtr + 9, sData.size () - 9, hash); -} - -// low-level write single -void HashedObjectStore::LLWrite (boost::shared_ptr ptr, leveldb::DB* db) -{ - HashedObject& obj = *ptr; - Blob rawData (9 + obj.getData ().size ()); - unsigned char* bufPtr = &rawData.front (); - - *reinterpret_cast (bufPtr + 0) = ntohl (obj.getIndex ()); - *reinterpret_cast (bufPtr + 4) = ntohl (obj.getIndex ()); - * (bufPtr + 8) = static_cast (obj.getType ()); - memcpy (bufPtr + 9, &obj.getData ().front (), obj.getData ().size ()); - - leveldb::Status st = db->Put (leveldb::WriteOptions (), - leveldb::Slice (reinterpret_cast (obj.getHash ().begin ()), obj.getHash ().size ()), - leveldb::Slice (reinterpret_cast (bufPtr), rawData.size ())); - - if (!st.ok ()) - { - WriteLog (lsFATAL, HashedObject) << "Failed to store hash node"; - assert (false); - } -} - -// low-level write set -void HashedObjectStore::LLWrite (const std::vector< boost::shared_ptr >& set, leveldb::DB* db) -{ - leveldb::WriteBatch batch; - - BOOST_FOREACH (const boost::shared_ptr& it, set) - { - const HashedObject& obj = *it; - Blob rawData (9 + obj.getData ().size ()); - unsigned char* bufPtr = &rawData.front (); - - *reinterpret_cast (bufPtr + 0) = ntohl (obj.getIndex ()); - *reinterpret_cast (bufPtr + 4) = ntohl (obj.getIndex ()); - * (bufPtr + 8) = static_cast (obj.getType ()); - memcpy (bufPtr + 9, &obj.getData ().front (), obj.getData ().size ()); - - batch.Put (leveldb::Slice (reinterpret_cast (obj.getHash ().begin ()), obj.getHash ().size ()), - leveldb::Slice (reinterpret_cast (bufPtr), rawData.size ())); - } - - leveldb::Status st = db->Write (leveldb::WriteOptions (), &batch); - - if (!st.ok ()) - { - WriteLog (lsFATAL, HashedObject) << "Failed to store hash node"; - assert (false); - } -} - -bool HashedObjectStore::storeLevelDB (HashedObjectType type, uint32 index, - Blob const& data, uint256 const& hash) -{ - // return: false = already in cache, true = added to cache - if (!theApp->getHashNodeLDB ()) - return true; - - if (mCache.touch (hash)) - return false; - -#ifdef PARANOID - assert (hash == Serializer::getSHA512Half (data)); -#endif - - HashedObject::pointer object = boost::make_shared (type, index, data, hash); - - if (!mCache.canonicalize (hash, object)) - { - boost::mutex::scoped_lock sl (mWriteMutex); - mWriteSet.push_back (object); - - if (!mWritePending) - { - mWritePending = true; - theApp->getJobQueue ().addJob (jtWRITE, "HashedObject::store", - BIND_TYPE (&HashedObjectStore::bulkWriteLevelDB, this, P_1)); - } - } - - mNegativeCache.del (hash); - return true; -} - -void HashedObjectStore::bulkWriteLevelDB (Job&) -{ - assert (mLevelDB); - int setSize = 0; - - while (1) - { - std::vector< boost::shared_ptr > set; - set.reserve (128); - - { - boost::mutex::scoped_lock sl (mWriteMutex); - - mWriteSet.swap (set); - assert (mWriteSet.empty ()); - ++mWriteGeneration; - mWriteCondition.notify_all (); - - if (set.empty ()) - { - mWritePending = false; - mWriteLoad = 0; - return; - } - - mWriteLoad = std::max (setSize, static_cast (mWriteSet.size ())); - setSize = set.size (); - } - - LLWrite (set, theApp->getHashNodeLDB ()); - - if (mEphemeralDB) - LLWrite (set, theApp->getEphemeralLDB ()); - } -} - -HashedObject::pointer HashedObjectStore::retrieveLevelDB (uint256 const& hash) -{ - HashedObject::pointer obj = mCache.fetch (hash); - - if (obj || mNegativeCache.isPresent (hash) || !theApp || !theApp->getHashNodeLDB ()) - return obj; - - if (mEphemeralDB) - { - obj = LLRetrieve (hash, theApp->getEphemeralLDB ()); - - if (obj) - { - mCache.canonicalize (hash, obj); - return obj; - } - } - - { - LoadEvent::autoptr event (theApp->getJobQueue ().getLoadEventAP (jtHO_READ, "HOS::retrieve")); - obj = LLRetrieve (hash, theApp->getHashNodeLDB ()); - - if (!obj) - { - mNegativeCache.add (hash); - return obj; - } - } - - mCache.canonicalize (hash, obj); - - if (mEphemeralDB) - LLWrite (obj, theApp->getEphemeralLDB ()); - - WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " fetch: in db"; - return obj; -} - -bool HashedObjectStore::storeSQLite (HashedObjectType type, uint32 index, - Blob const& data, uint256 const& hash) -{ - // return: false = already in cache, true = added to cache - if (!theApp->getHashNodeDB ()) - { - WriteLog (lsTRACE, HashedObject) << "HOS: no db"; - return true; - } - - if (mCache.touch (hash)) - { - WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " store: incache"; - return false; - } - - assert (hash == Serializer::getSHA512Half (data)); - - HashedObject::pointer object = boost::make_shared (type, index, data, hash); - - if (!mCache.canonicalize (hash, object)) - { - // WriteLog (lsTRACE, HashedObject) << "Queuing write for " << hash; - boost::mutex::scoped_lock sl (mWriteMutex); - mWriteSet.push_back (object); - - if (!mWritePending) - { - mWritePending = true; - theApp->getJobQueue ().addJob (jtWRITE, "HashedObject::store", - BIND_TYPE (&HashedObjectStore::bulkWriteSQLite, this, P_1)); - } - } - - // else - // WriteLog (lsTRACE, HashedObject) << "HOS: already had " << hash; - mNegativeCache.del (hash); - - return true; -} - -void HashedObjectStore::bulkWriteSQLite (Job&) -{ - assert (!mLevelDB); - int setSize = 0; - - while (1) - { - std::vector< boost::shared_ptr > set; - set.reserve (128); - - { - boost::mutex::scoped_lock sl (mWriteMutex); - mWriteSet.swap (set); - assert (mWriteSet.empty ()); - ++mWriteGeneration; - mWriteCondition.notify_all (); - - if (set.empty ()) - { - mWritePending = false; - mWriteLoad = 0; - return; - } - - mWriteLoad = std::max (setSize, static_cast (mWriteSet.size ())); - setSize = set.size (); - } - // WriteLog (lsTRACE, HashedObject) << "HOS: writing " << set.size(); - -#ifndef NO_SQLITE3_PREPARE - - if (mEphemeralDB) - LLWrite (set, theApp->getEphemeralLDB ()); - - { - Database* db = theApp->getHashNodeDB ()->getDB (); - - - // VFALCO TODO Get rid of the last parameter "aux", which is set to !theConfig.RUN_STANDALONE - // - static SqliteStatement pStB (db->getSqliteDB (), "BEGIN TRANSACTION;", !theConfig.RUN_STANDALONE); - static SqliteStatement pStE (db->getSqliteDB (), "END TRANSACTION;", !theConfig.RUN_STANDALONE); - static SqliteStatement pSt (db->getSqliteDB (), - "INSERT OR IGNORE INTO CommittedObjects " - "(Hash,ObjType,LedgerIndex,Object) VALUES (?, ?, ?, ?);", !theConfig.RUN_STANDALONE); - - pStB.step (); - pStB.reset (); - - BOOST_FOREACH (const boost::shared_ptr& it, set) - { - const char* type; - - switch (it->getType ()) - { - case hotLEDGER: - type = "L"; - break; - - case hotTRANSACTION: - type = "T"; - break; - - case hotACCOUNT_NODE: - type = "A"; - break; - - case hotTRANSACTION_NODE: - type = "N"; - break; - - default: - type = "U"; - } - - pSt.bind (1, it->getHash ().GetHex ()); - pSt.bind (2, type); - pSt.bind (3, it->getIndex ()); - pSt.bindStatic (4, it->getData ()); - int ret = pSt.step (); - - if (!pSt.isDone (ret)) - { - WriteLog (lsFATAL, HashedObject) << "Error saving hashed object " << ret; - assert (false); - } - - pSt.reset (); - } - - pStE.step (); - pStE.reset (); - } - -#else - - static boost::format - fAdd ("INSERT OR IGNORE INTO CommittedObjects " - "(Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); - - Database* db = theApp->getHashNodeDB ()->getDB (); - { - ScopedLock sl (theApp->getHashNodeDB ()->getDBLock ()); - - db->executeSQL ("BEGIN TRANSACTION;"); - - BOOST_FOREACH (const boost::shared_ptr& it, set) - { - char type; - - switch (it->getType ()) - { - case hotLEDGER: - type = 'L'; - break; - - case hotTRANSACTION: - type = 'T'; - break; - - case hotACCOUNT_NODE: - type = 'A'; - break; - - case hotTRANSACTION_NODE: - type = 'N'; - break; - - default: - type = 'U'; - } - - db->executeSQL (boost::str (boost::format (fAdd) - % it->getHash ().GetHex () % type % it->getIndex () % sqlEscape (it->getData ()))); - } - - db->executeSQL ("END TRANSACTION;"); - } -#endif - - } -} - -HashedObject::pointer HashedObjectStore::retrieveSQLite (uint256 const& hash) -{ - HashedObject::pointer obj = mCache.fetch (hash); - - if (obj) - return obj; - - if (mNegativeCache.isPresent (hash)) - return obj; - - if (mEphemeralDB) - { - obj = LLRetrieve (hash, theApp->getEphemeralLDB ()); - - if (obj) - { - mCache.canonicalize (hash, obj); - return obj; - } - } - - if (!theApp || !theApp->getHashNodeDB ()) - return obj; - - Blob data; - std::string type; - uint32 index; - -#ifndef NO_SQLITE3_PREPARE - { - ScopedLock sl (theApp->getHashNodeDB ()->getDBLock ()); - static SqliteStatement pSt (theApp->getHashNodeDB ()->getDB ()->getSqliteDB (), - "SELECT ObjType,LedgerIndex,Object FROM CommittedObjects WHERE Hash = ?;"); - LoadEvent::autoptr event (theApp->getJobQueue ().getLoadEventAP (jtDISK, "HOS::retrieve")); - - pSt.bind (1, hash.GetHex ()); - int ret = pSt.step (); - - if (pSt.isDone (ret)) - { - pSt.reset (); - mNegativeCache.add (hash); - WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " fetch: not in db"; - return obj; - } - - type = pSt.peekString (0); - index = pSt.getUInt32 (1); - pSt.getBlob (2).swap (data); - pSt.reset (); - } - -#else - - std::string sql = "SELECT * FROM CommittedObjects WHERE Hash='"; - sql.append (hash.GetHex ()); - sql.append ("';"); - - - { - ScopedLock sl (theApp->getHashNodeDB ()->getDBLock ()); - Database* db = theApp->getHashNodeDB ()->getDB (); - - if (!db->executeSQL (sql) || !db->startIterRows ()) - { - sl.unlock (); - mNegativeCache.add (hash); - return obj; - } - - db->getStr ("ObjType", type); - index = db->getBigInt ("LedgerIndex"); - - int size = db->getBinary ("Object", NULL, 0); - data.resize (size); - db->getBinary ("Object", & (data.front ()), size); - db->endIterRows (); - } -#endif - -#ifdef PARANOID - assert (Serializer::getSHA512Half (data) == hash); -#endif - - HashedObjectType htype = hotUNKNOWN; - - switch (type[0]) - { - case 'L': - htype = hotLEDGER; - break; - - case 'T': - htype = hotTRANSACTION; - break; - - case 'A': - htype = hotACCOUNT_NODE; - break; - - case 'N': - htype = hotTRANSACTION_NODE; - break; - - default: - assert (false); - WriteLog (lsERROR, HashedObject) << "Invalid hashed object"; - mNegativeCache.add (hash); - return obj; - } - - obj = boost::make_shared (htype, index, data, hash); - mCache.canonicalize (hash, obj); - - if (mEphemeralDB) - LLWrite (obj, theApp->getEphemeralLDB ()); - - WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " fetch: in db"; - return obj; -} - -int HashedObjectStore::import (const std::string& file) -{ - WriteLog (lsWARNING, HashedObject) << "Hashed object import from \"" << file << "\"."; - UPTR_T importDB (new SqliteDatabase (file.c_str ())); - importDB->connect (); - - leveldb::DB* db = theApp->getHashNodeLDB (); - leveldb::WriteOptions wo; - - int count = 0; - - SQL_FOREACH (importDB, "SELECT * FROM CommittedObjects;") - { - uint256 hash; - std::string hashStr; - importDB->getStr ("Hash", hashStr); - hash.SetHexExact (hashStr); - - if (hash.isZero ()) - { - WriteLog (lsWARNING, HashedObject) << "zero hash found in import table"; - } - else - { - Blob rawData; - int size = importDB->getBinary ("Object", NULL, 0); - rawData.resize (9 + size); - unsigned char* bufPtr = &rawData.front (); - - importDB->getBinary ("Object", bufPtr + 9, size); - - uint32 index = importDB->getBigInt ("LedgerIndex"); - *reinterpret_cast (bufPtr + 0) = ntohl (index); - *reinterpret_cast (bufPtr + 4) = ntohl (index); - - std::string type; - importDB->getStr ("ObjType", type); - HashedObjectType htype = hotUNKNOWN; - - switch (type[0]) - { - case 'L': - htype = hotLEDGER; - break; - - case 'T': - htype = hotTRANSACTION; - break; - - case 'A': - htype = hotACCOUNT_NODE; - break; - - case 'N': - htype = hotTRANSACTION_NODE; - break; - - default: - assert (false); - WriteLog (lsERROR, HashedObject) << "Invalid hashed object"; - } - - * (bufPtr + 8) = static_cast (htype); - - leveldb::Status st = db->Put (wo, - leveldb::Slice (reinterpret_cast (hash.begin ()), hash.size ()), - leveldb::Slice (reinterpret_cast (bufPtr), rawData.size ())); - - if (!st.ok ()) - { - WriteLog (lsFATAL, HashedObject) << "Failed to store hash node"; - assert (false); - } - - ++count; - } - - if ((count % 10000) == 0) - { - WriteLog (lsINFO, HashedObject) << "Import in progress: " << count; - } - } - - WriteLog (lsWARNING, HashedObject) << "Imported " << count << " nodes"; - return count; -} - -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_HashedObjectStore.h b/src/cpp/ripple/ripple_HashedObjectStore.h deleted file mode 100644 index f5f54fd820..0000000000 --- a/src/cpp/ripple/ripple_HashedObjectStore.h +++ /dev/null @@ -1,88 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_HASHEDOBJECTSTORE_H -#define RIPPLE_HASHEDOBJECTSTORE_H - -/** Persistency layer for hashed objects. -*/ -// VFALCO TODO Move all definitions to the .cpp -class HashedObjectStore -{ -public: - HashedObjectStore (int cacheSize, int cacheAge); - - bool isLevelDB () - { - return mLevelDB; - } - - float getCacheHitRate () - { - return mCache.getHitRate (); - } - - bool store (HashedObjectType type, uint32 index, Blob const& data, - uint256 const& hash) - { - if (mLevelDB) - return storeLevelDB (type, index, data, hash); - - return storeSQLite (type, index, data, hash); - } - - HashedObject::pointer retrieve (uint256 const& hash) - { - if (mLevelDB) - return retrieveLevelDB (hash); - - return retrieveSQLite (hash); - } - - bool storeSQLite (HashedObjectType type, uint32 index, Blob const& data, - uint256 const& hash); - HashedObject::pointer retrieveSQLite (uint256 const& hash); - void bulkWriteSQLite (Job&); - - bool storeLevelDB (HashedObjectType type, uint32 index, Blob const& data, - uint256 const& hash); - HashedObject::pointer retrieveLevelDB (uint256 const& hash); - void bulkWriteLevelDB (Job&); - - - void waitWrite (); - void tune (int size, int age); - void sweep () - { - mCache.sweep (); - mNegativeCache.sweep (); - } - int getWriteLoad (); - - int import (const std::string& fileName); - -private: - static HashedObject::pointer LLRetrieve (uint256 const& hash, leveldb::DB* db); - static void LLWrite (boost::shared_ptr ptr, leveldb::DB* db); - static void LLWrite (const std::vector< boost::shared_ptr >& set, leveldb::DB* db); - -private: - TaggedCache mCache; - KeyCache mNegativeCache; - - boost::mutex mWriteMutex; - boost::condition_variable mWriteCondition; - int mWriteGeneration; - int mWriteLoad; - - std::vector< boost::shared_ptr > mWriteSet; - bool mWritePending; - bool mLevelDB; - bool mEphemeralDB; -}; - -#endif -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_IApplication.h b/src/cpp/ripple/ripple_IApplication.h index 7ed252969e..f7a39e61de 100644 --- a/src/cpp/ripple/ripple_IApplication.h +++ b/src/cpp/ripple/ripple_IApplication.h @@ -14,10 +14,10 @@ class IHashRouter; class ILoadFeeTrack; class IPeers; class IProofOfWorkFactory; -class IUniqueNodeList; +class UniqueNodeList; class IValidations; -class HashedObjectStore; +class NodeStore; class JobQueue; class InboundLedgers; class LedgerMaster; @@ -38,8 +38,6 @@ typedef TaggedCache SLECach class IApplication { public: - static IApplication* New (); - virtual ~IApplication () { } /* VFALCO NOTE @@ -56,7 +54,6 @@ public: virtual boost::recursive_mutex& getMasterLock () = 0; virtual boost::asio::io_service& getIOService () = 0; - virtual boost::asio::io_service& getAuxService () = 0; virtual NodeCache& getTempNodeCache () = 0; virtual SLECache& getSLECache () = 0; @@ -68,10 +65,10 @@ public: virtual ILoadManager& getLoadManager () = 0; virtual IPeers& getPeers () = 0; virtual IProofOfWorkFactory& getProofOfWorkFactory () = 0; - virtual IUniqueNodeList& getUNL () = 0; + virtual UniqueNodeList& getUNL () = 0; virtual IValidations& getValidations () = 0; - virtual HashedObjectStore& getHashedObjectStore () = 0; + virtual NodeStore& getNodeStore () = 0; virtual JobQueue& getJobQueue () = 0; virtual InboundLedgers& getInboundLedgers () = 0; virtual LedgerMaster& getLedgerMaster () = 0; @@ -85,6 +82,12 @@ public: virtual DatabaseCon* getRpcDB () = 0; virtual DatabaseCon* getTxnDB () = 0; virtual DatabaseCon* getLedgerDB () = 0; + + /** Retrieve the "wallet database" + + It looks like this is used to store the unique node list. + */ + // VFALCO TODO Rename, document this virtual DatabaseCon* getWalletDB () = 0; // VFALCO NOTE It looks like this isn't used... //virtual DatabaseCon* getNetNodeDB () = 0; @@ -104,6 +107,6 @@ public: virtual void sweep () = 0; }; -extern IApplication* theApp; +extern IApplication& getApp (); #endif diff --git a/src/cpp/ripple/ripple_ILoadManager.h b/src/cpp/ripple/ripple_ILoadManager.h index 54e24350eb..ee1bbdaa8d 100644 --- a/src/cpp/ripple/ripple_ILoadManager.h +++ b/src/cpp/ripple/ripple_ILoadManager.h @@ -65,7 +65,6 @@ enum LoadType class LoadSource { public: - // VFALCO TODO Why even bother with a warning? Why can't we just drop? // VFALCO TODO Use these dispositions /* enum Disposition @@ -201,6 +200,13 @@ private: This object creates an associated thread to maintain a clock. + When the server is overloaded by a particular peer it issues a warning + first. This allows friendly peers to reduce their consumption of resources, + or disconnect from the server. + + The warning system is used instead of merely dropping, because hostile + peers can just reconnect anyway. + @see LoadSource, LoadType */ class ILoadManager @@ -208,21 +214,26 @@ class ILoadManager public: /** Create a new manager. + The manager thread begins running immediately. + @note The thresholds for warnings and punishments are in the ctor-initializer */ static ILoadManager* New (); + /** Destroy the manager. + + The destructor returns only after the thread has stopped. + */ virtual ~ILoadManager () { } /** Start the associated thread. This is here to prevent the deadlock detector from activating during a lengthy program initialization. - - @note In stand-alone mode, this might not get called. */ // VFALCO TODO Simplify the two stage initialization to one stage (construction). + // NOTE In stand-alone mode the load manager thread isn't started virtual void startThread () = 0; /** Turn on deadlock detection. diff --git a/src/cpp/ripple/ripple_IPeers.h b/src/cpp/ripple/ripple_IPeers.h index ecee1eee63..bfa44ea4bc 100644 --- a/src/cpp/ripple/ripple_IPeers.h +++ b/src/cpp/ripple/ripple_IPeers.h @@ -4,8 +4,8 @@ */ //============================================================================== -#ifndef RIPPLE_IPEERS_H -#define RIPPLE_IPEERS_H +#ifndef RIPPLE_IPEERS_H_INCLUDED +#define RIPPLE_IPEERS_H_INCLUDED /** Manages the set of connected peers. */ @@ -21,6 +21,7 @@ public: // Send message to network. virtual int relayMessage (Peer* fromPeer, const PackedMessage::pointer& msg) = 0; + virtual int relayMessageCluster (Peer* fromPeer, const PackedMessage::pointer& msg) = 0; virtual void relayMessageTo (const std::set& fromPeers, const PackedMessage::pointer& msg) = 0; virtual void relayMessageBut (const std::set& fromPeers, const PackedMessage::pointer& msg) = 0; diff --git a/src/cpp/ripple/ripple_IProofOfWorkFactory.h b/src/cpp/ripple/ripple_IProofOfWorkFactory.h index 58089cdd1d..c9ac0e4bb4 100644 --- a/src/cpp/ripple/ripple_IProofOfWorkFactory.h +++ b/src/cpp/ripple/ripple_IProofOfWorkFactory.h @@ -20,7 +20,7 @@ enum POWResult // VFALCO TODO move this to the class as a static member and rename it bool powResultInfo (POWResult powCode, std::string& strToken, std::string& strHuman); -class IProofOfWorkFactory +class IProofOfWorkFactory : LeakChecked { public: typedef boost::bimap< boost::bimaps::multiset_of, boost::bimaps::unordered_set_of > powMap_t; diff --git a/src/cpp/ripple/ripple_IValidations.h b/src/cpp/ripple/ripple_IValidations.h index fd0951a9c9..38e338776d 100644 --- a/src/cpp/ripple/ripple_IValidations.h +++ b/src/cpp/ripple/ripple_IValidations.h @@ -11,7 +11,7 @@ typedef boost::unordered_map ValidationSet; typedef std::pair currentValidationCount; // nodes validating and highest node ID validating -class IValidations +class IValidations : LeakChecked { public: static IValidations* New (); @@ -27,6 +27,8 @@ public: virtual int getTrustedValidationCount (uint256 const& ledger) = 0; + virtual int getFeeAverage(uint256 const& ledger, uint64 ref, uint64& fee) = 0; + virtual int getNodesAfter (uint256 const& ledger) = 0; virtual int getLoadRatio (bool overLoaded) = 0; diff --git a/src/cpp/ripple/ripple_InboundLedger.cpp b/src/cpp/ripple/ripple_InboundLedger.cpp index 91d942b6e1..5648c6acb8 100644 --- a/src/cpp/ripple/ripple_InboundLedger.cpp +++ b/src/cpp/ripple/ripple_InboundLedger.cpp @@ -11,7 +11,6 @@ SETUP_LOG (InboundLedger) #define LEDGER_ACQUIRE_TIMEOUT 2000 // millisecond for each ledger timeout #define LEDGER_TIMEOUT_COUNT 10 // how many timeouts before we giveup #define LEDGER_TIMEOUT_AGGRESSIVE 6 // how many timeouts before we get aggressive -#define TRUST_NETWORK InboundLedger::InboundLedger (uint256 const& hash, uint32 seq) : PeerSet (hash, LEDGER_ACQUIRE_TIMEOUT) @@ -49,18 +48,18 @@ bool InboundLedger::tryLocal () if (!mHaveBase) { // Nothing we can do without the ledger base - HashedObject::pointer node = theApp->getHashedObjectStore ().retrieve (mHash); + NodeObject::pointer node = getApp().getNodeStore ().retrieve (mHash); if (!node) { Blob data; - if (!theApp->getOPs ().getFetchPack (mHash, data)) + if (!getApp().getOPs ().getFetchPack (mHash, data)) return false; WriteLog (lsTRACE, InboundLedger) << "Ledger base found in fetch pack"; mLedger = boost::make_shared (data, true); - theApp->getHashedObjectStore ().store (hotLEDGER, mLedger->getLedgerSeq (), data, mHash); + getApp().getNodeStore ().store (hotLEDGER, mLedger->getLedgerSeq (), data, mHash); } else { @@ -139,7 +138,7 @@ bool InboundLedger::tryLocal () return mComplete; } -void InboundLedger::onTimer (bool progress) +void InboundLedger::onTimer (bool progress, boost::recursive_mutex::scoped_lock&) { mRecentTXNodes.clear (); mRecentASNodes.clear (); @@ -171,7 +170,7 @@ void InboundLedger::onTimer (bool progress) int pc = getPeerCount (); WriteLog (lsDEBUG, InboundLedger) << "No progress(" << pc << ") for ledger " << pc << mHash; - if (pc == 0) + if (pc < 3) addPeers (); else trigger (Peer::pointer ()); @@ -180,20 +179,24 @@ void InboundLedger::onTimer (bool progress) void InboundLedger::awaitData () { - boost::recursive_mutex::scoped_lock sl (mLock); ++mWaitCount; } void InboundLedger::noAwaitData () -{ - boost::recursive_mutex::scoped_lock sl (mLock); - - if (mWaitCount > 0 ) --mWaitCount; +{ // subtract one if mWaitCount is greater than zero + do + { + int j = mWaitCount.get(); + if (j <= 0) + return; + if (mWaitCount.compareAndSetBool(j - 1, j)) + return; + } while (1); } void InboundLedger::addPeers () { - std::vector peerList = theApp->getPeers ().getPeerVector (); + std::vector peerList = getApp().getPeers ().getPeerVector (); int vSize = peerList.size (); @@ -233,6 +236,9 @@ static void LADispatch ( InboundLedger::pointer la, std::vector< FUNCTION_TYPE > trig) { + Ledger::ref ledger = la->getLedger(); + if (ledger) + getApp().getLedgerMaster().checkAccept (ledger->getHash(), ledger->getLedgerSeq()); for (unsigned int i = 0; i < trig.size (); ++i) trig[i] (la); } @@ -265,14 +271,14 @@ void InboundLedger::done () if (mAccept) mLedger->setAccepted (); - theApp->getLedgerMaster ().storeLedger (mLedger); + getApp().getLedgerMaster ().storeLedger (mLedger); } else - theApp->getInboundLedgers ().logFailure (mHash); + getApp().getInboundLedgers ().logFailure (mHash); - if (!triggers.empty ()) // We hold the PeerSet lock, so must dispatch - theApp->getJobQueue ().addJob (jtLEDGER_DATA, "triggers", - BIND_TYPE (LADispatch, P_1, shared_from_this (), triggers)); + // We hold the PeerSet lock, so must dispatch + getApp().getJobQueue ().addJob (jtLEDGER_DATA, "triggers", + BIND_TYPE (LADispatch, P_1, shared_from_this (), triggers)); } bool InboundLedger::addOnComplete (FUNCTION_TYPE trigger) @@ -297,7 +303,7 @@ void InboundLedger::trigger (Peer::ref peer) return; } - if ((mWaitCount > 0) && peer) + if ((mWaitCount.get() > 0) && peer) { WriteLog (lsTRACE, InboundLedger) << "Skipping peer"; return; @@ -367,7 +373,7 @@ void InboundLedger::trigger (Peer::ref peer) for (boost::unordered_map::iterator it = mPeers.begin (), end = mPeers.end (); it != end; ++it) { - Peer::pointer iPeer = theApp->getPeers ().getPeerById (it->first); + Peer::pointer iPeer = getApp().getPeers ().getPeerById (it->first); if (iPeer) { @@ -539,7 +545,7 @@ void PeerSet::sendRequest (const protocol::TMGetLedger& tmGL) for (boost::unordered_map::iterator it = mPeers.begin (), end = mPeers.end (); it != end; ++it) { - Peer::pointer peer = theApp->getPeers ().getPeerById (it->first); + Peer::pointer peer = getApp().getPeers ().getPeerById (it->first); if (peer) peer->sendPacket (packet, false); @@ -566,7 +572,7 @@ int PeerSet::getPeerCount () const int ret = 0; for (boost::unordered_map::const_iterator it = mPeers.begin (), end = mPeers.end (); it != end; ++it) - if (theApp->getPeers ().hasPeer (it->first)) + if (getApp().getPeers ().hasPeer (it->first)) ++ret; return ret; @@ -666,7 +672,7 @@ bool InboundLedger::takeBase (const std::string& data) // data must not have has Serializer s (data.size () + 4); s.add32 (HashPrefix::ledgerMaster); s.addRaw (data); - theApp->getHashedObjectStore ().store (hotLEDGER, mLedger->getLedgerSeq (), s.peekData (), mHash); + getApp().getNodeStore ().store (hotLEDGER, mLedger->getLedgerSeq (), s.peekData (), mHash); progress (); @@ -838,7 +844,7 @@ std::vector InboundLedger::getNeededHashes () if (!mHaveTransactions) { TransactionStateSF filter (mLedger->getLedgerSeq ()); - std::vector v = mLedger->getNeededAccountStateHashes (4, &filter); + std::vector v = mLedger->getNeededTransactionHashes (4, &filter); BOOST_FOREACH (uint256 const & h, v) { ret.push_back (std::make_pair (protocol::TMGetObjectByHash::otTRANSACTION_NODE, h)); diff --git a/src/cpp/ripple/ripple_InboundLedger.h b/src/cpp/ripple/ripple_InboundLedger.h index 6768508cc8..d994252869 100644 --- a/src/cpp/ripple/ripple_InboundLedger.h +++ b/src/cpp/ripple/ripple_InboundLedger.h @@ -15,6 +15,8 @@ class InboundLedger , public CountedObject { public: + static char const* getCountedObjectName () { return "InboundLedger"; } + typedef boost::shared_ptr pointer; public: @@ -90,7 +92,7 @@ public: private: void done (); - void onTimer (bool progress); + void onTimer (bool progress, boost::recursive_mutex::scoped_lock& peerSetLock); void newPeer (Peer::ref peer) { @@ -100,16 +102,16 @@ private: boost::weak_ptr pmDowncast (); private: - Ledger::pointer mLedger; - bool mHaveBase; - bool mHaveState; - bool mHaveTransactions; - bool mAborted; - bool mSignaled; - bool mAccept; - bool mByHash; - int mWaitCount; - uint32 mSeq; + Ledger::pointer mLedger; + bool mHaveBase; + bool mHaveState; + bool mHaveTransactions; + bool mAborted; + bool mSignaled; + bool mAccept; + bool mByHash; + beast::Atomic mWaitCount; + uint32 mSeq; std::set mRecentTXNodes; std::set mRecentASNodes; diff --git a/src/cpp/ripple/ripple_InboundLedgers.cpp b/src/cpp/ripple/ripple_InboundLedgers.cpp index e78619eb47..b6f091a311 100644 --- a/src/cpp/ripple/ripple_InboundLedgers.cpp +++ b/src/cpp/ripple/ripple_InboundLedgers.cpp @@ -23,12 +23,12 @@ InboundLedger::pointer InboundLedgers::findCreate (uint256 const& hash, uint32 s ptr->addPeers (); ptr->setTimer (); // Cannot call in constructor } - else + else if (ptr->isComplete ()) { Ledger::pointer ledger = ptr->getLedger (); ledger->setClosed (); ledger->setImmutable (); - theApp->getLedgerMaster ().storeLedger (ledger); + getApp().getLedgerMaster ().storeLedger (ledger); WriteLog (lsDEBUG, InboundLedger) << "Acquiring ledger we already have: " << hash; } @@ -75,6 +75,16 @@ bool InboundLedgers::awaitLedgerData (uint256 const& ledgerHash) return true; } +/* +This gets called when + "We got some data from an inbound ledger" + +inboundLedgerTrigger: + "What do we do with this partial data?" + Figures out what to do with the responses to our requests for information. + +*/ +// means "We got some data from an inbound ledger" void InboundLedgers::gotLedgerData (Job&, uint256 hash, boost::shared_ptr packet_ptr, boost::weak_ptr wPeer) { @@ -215,16 +225,21 @@ int InboundLedgers::getFetchCount (int& timeoutCount) { timeoutCount = 0; int ret = 0; + + std::map inboundLedgers; + { - typedef std::pair u256_acq_pair; boost::mutex::scoped_lock sl (mLock); - BOOST_FOREACH (const u256_acq_pair & it, mLedgers) + inboundLedgers = mLedgers; + } + + typedef std::pair u256_acq_pair; + BOOST_FOREACH (const u256_acq_pair & it, inboundLedgers) + { + if (it.second->isActive ()) { - if (it.second->isActive ()) - { - ++ret; - timeoutCount += it.second->getTimeouts (); - } + ++ret; + timeoutCount += it.second->getTimeouts (); } } return ret; diff --git a/src/cpp/ripple/ripple_InboundLedgers.h b/src/cpp/ripple/ripple_InboundLedgers.h index 694fb5db6c..5d034e5010 100644 --- a/src/cpp/ripple/ripple_InboundLedgers.h +++ b/src/cpp/ripple/ripple_InboundLedgers.h @@ -13,7 +13,7 @@ */ // VFALCO TODO Rename to InboundLedgers // VFALCO TODO Create abstract interface -class InboundLedgers +class InboundLedgers : LeakChecked { public: // How long before we try again to acquire the same ledger diff --git a/src/cpp/ripple/ripple_InfoSub.cpp b/src/cpp/ripple/ripple_InfoSub.cpp index 43cc3fa4d9..cc60ea7850 100644 --- a/src/cpp/ripple/ripple_InfoSub.cpp +++ b/src/cpp/ripple/ripple_InfoSub.cpp @@ -27,7 +27,7 @@ InfoSub::InfoSub () InfoSub::~InfoSub () { - NetworkOPs& ops = theApp->getOPs (); + NetworkOPs& ops = getApp().getOPs (); ops.unsubTransactions (mSeq); ops.unsubRTTransactions (mSeq); ops.unsubLedger (mSeq); diff --git a/src/cpp/ripple/ripple_InfoSub.h b/src/cpp/ripple/ripple_InfoSub.h index 3b926870f8..2aa6fa464b 100644 --- a/src/cpp/ripple/ripple_InfoSub.h +++ b/src/cpp/ripple/ripple_InfoSub.h @@ -16,6 +16,8 @@ class InfoSub : public CountedObject { public: + static char const* getCountedObjectName () { return "InfoSub"; } + typedef boost::shared_ptr pointer; // VFALCO TODO Standardize on the names of weak / strong pointer typedefs. diff --git a/src/cpp/ripple/ripple_LedgerConsensus.cpp b/src/cpp/ripple/ripple_LedgerConsensus.cpp index 9accf3174b..704ae9c4f7 100644 --- a/src/cpp/ripple/ripple_LedgerConsensus.cpp +++ b/src/cpp/ripple/ripple_LedgerConsensus.cpp @@ -18,18 +18,18 @@ LedgerConsensus::LedgerConsensus (uint256 const& prevLCLHash, Ledger::ref previo { WriteLog (lsDEBUG, LedgerConsensus) << "Creating consensus object"; WriteLog (lsTRACE, LedgerConsensus) << "LCL:" << previousLedger->getHash () << ", ct=" << closeTime; - mPreviousProposers = theApp->getOPs ().getPreviousProposers (); - mPreviousMSeconds = theApp->getOPs ().getPreviousConvergeTime (); + mPreviousProposers = getApp().getOPs ().getPreviousProposers (); + mPreviousMSeconds = getApp().getOPs ().getPreviousConvergeTime (); assert (mPreviousMSeconds); mCloseResolution = ContinuousLedgerTiming::getNextLedgerTimeResolution ( mPreviousLedger->getCloseResolution (), mPreviousLedger->getCloseAgree (), previousLedger->getLedgerSeq () + 1); - if (mValPublic.isSet () && mValPrivate.isSet () && !theApp->getOPs ().isNeedNetworkLedger ()) + if (mValPublic.isSet () && mValPrivate.isSet () && !getApp().getOPs ().isNeedNetworkLedger ()) { WriteLog (lsINFO, LedgerConsensus) << "Entering consensus process, validating"; mValidating = true; - mProposing = theApp->getOPs ().getOperatingMode () == NetworkOPs::omFULL; + mProposing = getApp().getOPs ().getOperatingMode () == NetworkOPs::omFULL; } else { @@ -41,7 +41,7 @@ LedgerConsensus::LedgerConsensus (uint256 const& prevLCLHash, Ledger::ref previo if (!mHaveCorrectLCL) { - theApp->getOPs ().setProposing (false, false); + getApp().getOPs ().setProposing (false, false); handleLCL (mPrevLedgerHash); if (!mHaveCorrectLCL) @@ -52,16 +52,16 @@ LedgerConsensus::LedgerConsensus (uint256 const& prevLCLHash, Ledger::ref previo } } else - theApp->getOPs ().setProposing (mProposing, mValidating); + getApp().getOPs ().setProposing (mProposing, mValidating); } void LedgerConsensus::checkOurValidation () { // This only covers some cases - Fix for the case where we can't ever acquire the consensus ledger - if (!mHaveCorrectLCL || !mValPublic.isSet () || !mValPrivate.isSet () || theApp->getOPs ().isNeedNetworkLedger ()) + if (!mHaveCorrectLCL || !mValPublic.isSet () || !mValPrivate.isSet () || getApp().getOPs ().isNeedNetworkLedger ()) return; - SerializedValidation::pointer lastVal = theApp->getOPs ().getLastValidation (); + SerializedValidation::pointer lastVal = getApp().getOPs ().getLastValidation (); if (lastVal) { @@ -74,19 +74,20 @@ void LedgerConsensus::checkOurValidation () uint256 signingHash; SerializedValidation::pointer v = boost::make_shared - (mPreviousLedger->getHash (), theApp->getOPs ().getValidationTimeNC (), mValPublic, false); + (mPreviousLedger->getHash (), getApp().getOPs ().getValidationTimeNC (), mValPublic, false); + addLoad(v); v->setTrusted (); v->sign (signingHash, mValPrivate); - theApp->getHashRouter ().addSuppression (signingHash); - theApp->getValidations ().addValidation (v, "localMissing"); + getApp().getHashRouter ().addSuppression (signingHash); + getApp().getValidations ().addValidation (v, "localMissing"); Blob validation = v->getSigned (); protocol::TMValidation val; val.set_validation (&validation[0], validation.size ()); #if 0 - theApp->getPeers ().relayMessage (NULL, + getApp().getPeers ().relayMessage (NULL, boost::make_shared (val, protocol::mtVALIDATION)); #endif - theApp->getOPs ().setLastValidation (v); + getApp().getOPs ().setLastValidation (v); WriteLog (lsWARNING, LedgerConsensus) << "Sending partial validation"; } @@ -102,7 +103,7 @@ void LedgerConsensus::checkLCL () priorLedger = mPreviousLedger->getParentHash (); // don't jump back boost::unordered_map vals = - theApp->getValidations ().getCurrentValidations (favoredLedger, priorLedger); + getApp().getValidations ().getCurrentValidations (favoredLedger, priorLedger); typedef std::map::value_type u256_cvc_pair; BOOST_FOREACH (u256_cvc_pair & it, vals) @@ -153,7 +154,7 @@ void LedgerConsensus::checkLCL () } if (mHaveCorrectLCL) - theApp->getOPs ().consensusViewChange (); + getApp().getOPs ().consensusViewChange (); handleLCL (netLgr); } @@ -190,7 +191,7 @@ void LedgerConsensus::handleLCL (uint256 const& lclHash) return; // we need to switch the ledger we're working from - Ledger::pointer newLCL = theApp->getLedgerMaster ().getLedgerByHash (lclHash); + Ledger::pointer newLCL = getApp().getLedgerMaster ().getLedgerByHash (lclHash); if (newLCL) { @@ -206,9 +207,9 @@ void LedgerConsensus::handleLCL (uint256 const& lclHash) WriteLog (lsWARNING, LedgerConsensus) << "Need consensus ledger " << mPrevLedgerHash; if (mAcquiringLedger) - theApp->getInboundLedgers ().dropLedger (mAcquiringLedger->getHash ()); + getApp().getInboundLedgers ().dropLedger (mAcquiringLedger->getHash ()); - mAcquiringLedger = theApp->getInboundLedgers ().findCreate (mPrevLedgerHash, 0); + mAcquiringLedger = getApp().getInboundLedgers ().findCreate (mPrevLedgerHash, 0); mHaveCorrectLCL = false; return; } @@ -230,8 +231,8 @@ void LedgerConsensus::takeInitialPosition (Ledger& initialLedger) { // previous ledger was flag ledger SHAMap::pointer preSet = initialLedger.peekTransactionMap ()->snapShot (true); - theApp->getFeeVote ().doVoting (mPreviousLedger, preSet); - theApp->getFeatureTable ().doVoting (mPreviousLedger, preSet); + getApp().getFeeVote ().doVoting (mPreviousLedger, preSet); + getApp().getFeatureTable ().doVoting (mPreviousLedger, preSet); initialSet = preSet->snapShot (false); } else @@ -380,7 +381,7 @@ void LedgerConsensus::sendHaveTxSet (uint256 const& hash, bool direct) msg.set_hash (hash.begin (), 256 / 8); msg.set_status (direct ? protocol::tsHAVE : protocol::tsCAN_GET); PackedMessage::pointer packet = boost::make_shared (msg, protocol::mtHAVE_SET); - theApp->getPeers ().relayMessage (NULL, packet); + getApp().getPeers ().relayMessage (NULL, packet); } void LedgerConsensus::adjustCount (SHAMap::ref map, const std::vector& peers) @@ -405,19 +406,19 @@ void LedgerConsensus::statusChange (protocol::NodeEvent event, Ledger& ledger) s.set_newevent (event); s.set_ledgerseq (ledger.getLedgerSeq ()); - s.set_networktime (theApp->getOPs ().getNetworkTimeNC ()); + s.set_networktime (getApp().getOPs ().getNetworkTimeNC ()); uint256 hash = ledger.getParentHash (); s.set_ledgerhashprevious (hash.begin (), hash.size ()); hash = ledger.getHash (); s.set_ledgerhash (hash.begin (), hash.size ()); uint32 uMin, uMax; - theApp->getOPs ().getValidatedRange (uMin, uMax); + getApp().getOPs ().getValidatedRange (uMin, uMax); s.set_firstseq (uMin); s.set_lastseq (uMax); PackedMessage::pointer packet = boost::make_shared (s, protocol::mtSTATUS_CHANGE); - theApp->getPeers ().relayMessage (NULL, packet); + getApp().getPeers ().relayMessage (NULL, packet); WriteLog (lsTRACE, LedgerConsensus) << "send status change to peer"; } @@ -429,9 +430,9 @@ int LedgerConsensus::startup () void LedgerConsensus::statePreClose () { // it is shortly before ledger close time - bool anyTransactions = theApp->getLedgerMaster ().getCurrentLedger ()->peekTransactionMap ()->getHash ().isNonZero (); + bool anyTransactions = getApp().getLedgerMaster ().getCurrentLedger ()->peekTransactionMap ()->getHash ().isNonZero (); int proposersClosed = mPeerPositions.size (); - int proposersValidated = theApp->getValidations ().getTrustedValidationCount (mPrevLedgerHash); + int proposersValidated = getApp().getValidations ().getTrustedValidationCount (mPrevLedgerHash); // This ledger is open. This computes how long since the last ledger closed int sinceClose; @@ -440,7 +441,7 @@ void LedgerConsensus::statePreClose () if (mHaveCorrectLCL && mPreviousLedger->getCloseAgree ()) { // we can use consensus timing - sinceClose = 1000 * (theApp->getOPs ().getCloseTimeNC () - mPreviousLedger->getCloseTimeNC ()); + sinceClose = 1000 * (getApp().getOPs ().getCloseTimeNC () - mPreviousLedger->getCloseTimeNC ()); idleInterval = 2 * mPreviousLedger->getCloseResolution (); if (idleInterval < LEDGER_IDLE_INTERVAL) @@ -448,7 +449,7 @@ void LedgerConsensus::statePreClose () } else { - sinceClose = 1000 * (theApp->getOPs ().getCloseTimeNC () - theApp->getOPs ().getLastCloseTime ()); + sinceClose = 1000 * (getApp().getOPs ().getCloseTimeNC () - getApp().getOPs ().getLastCloseTime ()); idleInterval = LEDGER_IDLE_INTERVAL; } @@ -464,10 +465,10 @@ void LedgerConsensus::closeLedger () checkOurValidation (); mState = lcsESTABLISH; mConsensusStartTime = boost::posix_time::microsec_clock::universal_time (); - mCloseTime = theApp->getOPs ().getCloseTimeNC (); - theApp->getOPs ().setLastCloseTime (mCloseTime); + mCloseTime = getApp().getOPs ().getCloseTimeNC (); + getApp().getOPs ().setLastCloseTime (mCloseTime); statusChange (protocol::neCLOSING_LEDGER, *mPreviousLedger); - takeInitialPosition (*theApp->getLedgerMaster ().closeLedger (true)); + takeInitialPosition (*getApp().getLedgerMaster ().closeLedger (true)); } void LedgerConsensus::stateEstablish () @@ -511,7 +512,7 @@ void LedgerConsensus::timerEntry () if (doShutdown) { WriteLog (lsFATAL, LedgerConsensus) << "Shutdown requested"; - theApp->stop (); + getApp().stop (); } if ((mState != lcsFINISHED) && (mState != lcsACCEPTED)) @@ -714,7 +715,7 @@ bool LedgerConsensus::haveConsensus (bool forReal) } } } - int currentValidations = theApp->getValidations ().getNodesAfter (mPrevLedgerHash); + int currentValidations = getApp().getValidations ().getNodesAfter (mPrevLedgerHash); WriteLog (lsDEBUG, LedgerConsensus) << "Checking for TX consensus: agree=" << agree << ", disagree=" << disagree; @@ -731,7 +732,7 @@ SHAMap::pointer LedgerConsensus::getTransactionTree (uint256 const& hash, bool d if (mState == lcsPRE_CLOSE) { - SHAMap::pointer currentMap = theApp->getLedgerMaster ().getCurrentLedger ()->peekTransactionMap (); + SHAMap::pointer currentMap = getApp().getLedgerMaster ().getCurrentLedger ()->peekTransactionMap (); if (currentMap->getHash () == hash) { @@ -787,7 +788,7 @@ void LedgerConsensus::startAcquiring (TransactionAcquire::pointer acquire) } } - std::vector peerList = theApp->getPeers ().getPeerVector (); + std::vector peerList = getApp().getPeers ().getPeerVector (); BOOST_FOREACH (Peer::ref peer, peerList) { if (peer->hasTxSet (acquire->getHash ())) @@ -812,7 +813,7 @@ void LedgerConsensus::propose () Blob sig = mOurPosition->sign (); prop.set_nodepubkey (&pubKey[0], pubKey.size ()); prop.set_signature (&sig[0], sig.size ()); - theApp->getPeers ().relayMessage (NULL, + getApp().getPeers ().relayMessage (NULL, boost::make_shared (prop, protocol::mtPROPOSE_LEDGER)); } @@ -847,14 +848,14 @@ void LedgerConsensus::addDisputedTransaction (uint256 const& txID, Blob const& t txn->setVote (pit.first, cit->second->hasItem (txID)); } - if (theApp->getHashRouter ().setFlag (txID, SF_RELAYED)) + if (getApp().getHashRouter ().setFlag (txID, SF_RELAYED)) { protocol::TMTransaction msg; msg.set_rawtransaction (& (tx.front ()), tx.size ()); msg.set_status (protocol::tsNEW); - msg.set_receivetimestamp (theApp->getOPs ().getNetworkTimeNC ()); + msg.set_receivetimestamp (getApp().getOPs ().getNetworkTimeNC ()); PackedMessage::pointer packet = boost::make_shared (msg, protocol::mtTRANSACTION); - theApp->getPeers ().relayMessage (NULL, packet); + getApp().getPeers ().relayMessage (NULL, packet); } } @@ -966,21 +967,21 @@ void LedgerConsensus::beginAccept (bool synchronous) return; } - theApp->getOPs ().newLCL (mPeerPositions.size (), mCurrentMSeconds, mNewLedgerHash); + getApp().getOPs ().newLCL (mPeerPositions.size (), mCurrentMSeconds, mNewLedgerHash); if (synchronous) accept (consensusSet, LoadEvent::pointer ()); else { - theApp->getIOService ().post (boost::bind (&LedgerConsensus::accept, shared_from_this (), consensusSet, - theApp->getJobQueue ().getLoadEvent (jtACCEPTLEDGER, "LedgerConsensus::beginAccept"))); + getApp().getIOService ().post (BIND_TYPE (&LedgerConsensus::accept, shared_from_this (), consensusSet, + getApp().getJobQueue ().getLoadEvent (jtACCEPTLEDGER, "LedgerConsensus::beginAccept"))); } } void LedgerConsensus::playbackProposals () { boost::unordered_map < uint160, - std::list > & storedProposals = theApp->getOPs ().peekStoredProposals (); + std::list > & storedProposals = getApp().getOPs ().peekStoredProposals (); for (boost::unordered_map< uint160, std::list >::iterator it = storedProposals.begin (), end = storedProposals.end (); it != end; ++it) @@ -1010,7 +1011,7 @@ void LedgerConsensus::playbackProposals () #if 0 // FIXME: We can't do delayed relay because we don't have the signature std::set peers - if (relay && theApp->getHashRouter ().swapSet (proposal.getSuppress (), set, SF_RELAYED)) + if (relay && getApp().getHashRouter ().swapSet (proposal.getSuppress (), set, SF_RELAYED)) { WriteLog (lsDEBUG, LedgerConsensus) << "Stored proposal delayed relay"; protocol::TMProposeSet set; @@ -1021,7 +1022,7 @@ void LedgerConsensus::playbackProposals () nodepubkey signature PackedMessage::pointer message = boost::make_shared (set, protocol::mtPROPOSE_LEDGER); - theApp->getPeers ().relayMessageBut (peers, message); + getApp().getPeers ().relayMessageBut (peers, message); } #endif @@ -1043,7 +1044,7 @@ int LedgerConsensus::applyTransaction (TransactionEngine& engine, SerializedTran if (retryAssured) parms = static_cast (parms | tapRETRY); - if (theApp->getHashRouter ().setFlag (txn->getTransactionID (), SF_SIGGOOD)) + if (getApp().getHashRouter ().setFlag (txn->getTransactionID (), SF_SIGGOOD)) parms = static_cast (parms | tapNO_CHECK_SIGN); WriteLog (lsDEBUG, LedgerConsensus) << "TXN " << txn->getTransactionID () @@ -1176,12 +1177,12 @@ uint32 LedgerConsensus::roundCloseTime (uint32 closeTime) void LedgerConsensus::accept (SHAMap::ref set, LoadEvent::pointer) { if (set->getHash ().isNonZero ()) // put our set where others can get it later - theApp->getOPs ().takePosition (mPreviousLedger->getLedgerSeq (), set); + getApp().getOPs ().takePosition (mPreviousLedger->getLedgerSeq (), set); - boost::recursive_mutex::scoped_lock masterLock (theApp->getMasterLock ()); + boost::recursive_mutex::scoped_lock masterLock (getApp().getMasterLock ()); assert (set->getHash () == mOurPosition->getCurrentHash ()); - theApp->getOPs ().peekStoredProposals ().clear (); // these are now obsolete + getApp().getOPs ().peekStoredProposals ().clear (); // these are now obsolete uint32 closeTime = roundCloseTime (mOurPosition->getCloseTime ()); bool closeTimeCorrect = true; @@ -1245,24 +1246,25 @@ void LedgerConsensus::accept (SHAMap::ref set, LoadEvent::pointer) { uint256 signingHash; SerializedValidation::pointer v = boost::make_shared - (newLCLHash, theApp->getOPs ().getValidationTimeNC (), mValPublic, mProposing); + (newLCLHash, getApp().getOPs ().getValidationTimeNC (), mValPublic, mProposing); v->setFieldU32 (sfLedgerSequence, newLCL->getLedgerSeq ()); + addLoad(v); if (((newLCL->getLedgerSeq () + 1) % 256) == 0) // next ledger is flag ledger { - theApp->getFeeVote ().doValidation (newLCL, *v); - theApp->getFeatureTable ().doValidation (newLCL, *v); + getApp().getFeeVote ().doValidation (newLCL, *v); + getApp().getFeatureTable ().doValidation (newLCL, *v); } v->sign (signingHash, mValPrivate); v->setTrusted (); - theApp->getHashRouter ().addSuppression (signingHash); // suppress it if we receive it - theApp->getValidations ().addValidation (v, "local"); - theApp->getOPs ().setLastValidation (v); + getApp().getHashRouter ().addSuppression (signingHash); // suppress it if we receive it + getApp().getValidations ().addValidation (v, "local"); + getApp().getOPs ().setLastValidation (v); Blob validation = v->getSigned (); protocol::TMValidation val; val.set_validation (&validation[0], validation.size ()); - int j = theApp->getPeers ().relayMessage (NULL, + int j = getApp().getPeers ().relayMessage (NULL, boost::make_shared (val, protocol::mtVALIDATION)); WriteLog (lsINFO, LedgerConsensus) << "CNF Val " << newLCLHash << " to " << j << " peers"; } @@ -1270,7 +1272,7 @@ void LedgerConsensus::accept (SHAMap::ref set, LoadEvent::pointer) WriteLog (lsINFO, LedgerConsensus) << "CNF newLCL " << newLCLHash; Ledger::pointer newOL = boost::make_shared (true, boost::ref (*newLCL)); - ScopedLock sl ( theApp->getLedgerMaster ().getLock ()); + ScopedLock sl ( getApp().getLedgerMaster ().getLock ()); // Apply disputed transactions that didn't get in TransactionEngine engine (newOL); @@ -1296,9 +1298,9 @@ void LedgerConsensus::accept (SHAMap::ref set, LoadEvent::pointer) } WriteLog (lsDEBUG, LedgerConsensus) << "Applying transactions from current open ledger"; - applyTransactions (theApp->getLedgerMaster ().getCurrentLedger ()->peekTransactionMap (), newOL, newLCL, + applyTransactions (getApp().getLedgerMaster ().getCurrentLedger ()->peekTransactionMap (), newOL, newLCL, failedTransactions, true); - theApp->getLedgerMaster ().pushLedger (newLCL, newOL, !mConsensusFail); + getApp().getLedgerMaster ().pushLedger (newLCL, newOL, !mConsensusFail); mNewLedgerHash = newLCL->getHash (); mState = lcsACCEPTED; sl.unlock (); @@ -1323,14 +1325,24 @@ void LedgerConsensus::accept (SHAMap::ref set, LoadEvent::pointer) closeTotal /= closeCount; int offset = static_cast (closeTotal) - static_cast (mCloseTime); WriteLog (lsINFO, LedgerConsensus) << "Our close offset is estimated at " << offset << " (" << closeCount << ")"; - theApp->getOPs ().closeTimeOffset (offset); + getApp().getOPs ().closeTimeOffset (offset); } } void LedgerConsensus::endConsensus () { - theApp->getOPs ().endConsensus (mHaveCorrectLCL); + getApp().getOPs ().endConsensus (mHaveCorrectLCL); +} + +void LedgerConsensus::addLoad(SerializedValidation::ref val) +{ + uint32 fee = std::max( + getApp().getFeeTrack().getLocalFee(), + getApp().getFeeTrack().getClusterFee()); + uint32 ref = getApp().getFeeTrack().getLoadBase(); + if (fee > ref) + val->setFieldU32(sfLoadFee, fee); } void LedgerConsensus::simulate () diff --git a/src/cpp/ripple/ripple_LedgerConsensus.h b/src/cpp/ripple/ripple_LedgerConsensus.h index 56f1e76c75..b518f62467 100644 --- a/src/cpp/ripple/ripple_LedgerConsensus.h +++ b/src/cpp/ripple/ripple_LedgerConsensus.h @@ -17,6 +17,8 @@ class LedgerConsensus , public CountedObject { public: + static char const* getCountedObjectName () { return "LedgerConsensus"; } + LedgerConsensus (LedgerHash const & prevLCLHash, Ledger::ref previousLedger, uint32 closeTime); int startup (); @@ -106,6 +108,8 @@ private: void beginAccept (bool synchronous); void endConsensus (); + void addLoad (SerializedValidation::ref val); + private: // VFALCO TODO Rename these to look pretty enum LCState diff --git a/src/cpp/ripple/ripple_LedgerEntrySet.cpp b/src/cpp/ripple/ripple_LedgerEntrySet.cpp index 29fb941fa8..ae675476f6 100644 --- a/src/cpp/ripple/ripple_LedgerEntrySet.cpp +++ b/src/cpp/ripple/ripple_LedgerEntrySet.cpp @@ -679,24 +679,9 @@ TER LedgerEntrySet::dirAdd ( } else { - // Have old last point to new node, if it was not root. - if (uNodeDir == 1) - { - // Previous node is root node. - - sleRoot->setFieldU64 (sfIndexNext, uNodeDir); - } - else - { - // Previous node is not root node. - - SLE::pointer slePrevious = entryCache (ltDIR_NODE, Ledger::getDirNodeIndex (uRootIndex, uNodeDir - 1)); - - slePrevious->setFieldU64 (sfIndexNext, uNodeDir); - entryModify (slePrevious); - - sleNode->setFieldU64 (sfIndexPrevious, uNodeDir - 1); - } + // Have old last point to new node + sleNode->setFieldU64(sfIndexNext, uNodeDir); + entryModify(sleNode); // Have root point to new node. sleRoot->setFieldU64 (sfIndexPrevious, uNodeDir); @@ -704,6 +689,8 @@ TER LedgerEntrySet::dirAdd ( // Create the new node. sleNode = entryCreate (ltDIR_NODE, Ledger::getDirNodeIndex (uRootIndex, uNodeDir)); + if (uNodeDir != 1) + sleNode->setFieldU64(sfIndexPrevious, uNodeDir - 1); sleNode->setFieldH256 (sfRootIndex, uRootIndex); fDescriber (sleNode); @@ -893,6 +880,8 @@ TER LedgerEntrySet::dirDelete ( // Fix next to point to its new previous. sleNext->setFieldU64 (sfIndexPrevious, uNodePrevious); entryModify (sleNext); + + entryDelete(sleNode); } // Last node. else if (bKeepRoot || uNodePrevious) diff --git a/src/cpp/ripple/ripple_LedgerEntrySet.h b/src/cpp/ripple/ripple_LedgerEntrySet.h index 1b2cc55a82..236c4af3eb 100644 --- a/src/cpp/ripple/ripple_LedgerEntrySet.h +++ b/src/cpp/ripple/ripple_LedgerEntrySet.h @@ -36,6 +36,8 @@ class LedgerEntrySetEntry : public CountedObject { public: + static char const* getCountedObjectName () { return "LedgerEntrySetEntry"; } + SLE::pointer mEntry; LedgerEntryAction mAction; int mSeq; @@ -60,6 +62,8 @@ class LedgerEntrySet : public CountedObject { public: + static char const* getCountedObjectName () { return "LedgerEntrySet"; } + LedgerEntrySet (Ledger::ref ledger, TransactionEngineParams tep, bool immutable = false) : mLedger (ledger), mParams (tep), mSeq (0), mImmutable (immutable) { @@ -263,17 +267,5 @@ inline LedgerEntrySet::iterator range_end (LedgerEntrySet& x) { return x.end (); } -namespace boost -{ -template<> struct range_mutable_iterator -{ - typedef LedgerEntrySet::iterator type; -}; -template<> struct range_const_iterator -{ - typedef LedgerEntrySet::const_iterator type; -}; -} #endif -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_LedgerHistory.h b/src/cpp/ripple/ripple_LedgerHistory.h index ebde79f4df..a0629a79f4 100644 --- a/src/cpp/ripple/ripple_LedgerHistory.h +++ b/src/cpp/ripple/ripple_LedgerHistory.h @@ -8,7 +8,7 @@ #define RIPPLE_LEDGERHISTORY_H // VFALCO TODO Rename to OldLedgers ? -class LedgerHistory +class LedgerHistory : LeakChecked { public: LedgerHistory (); diff --git a/src/cpp/ripple/ripple_LoadManager.cpp b/src/cpp/ripple/ripple_LoadManager.cpp index 4488e44585..df4ec66980 100644 --- a/src/cpp/ripple/ripple_LoadManager.cpp +++ b/src/cpp/ripple/ripple_LoadManager.cpp @@ -8,7 +8,9 @@ SETUP_LOG (LoadManager) //------------------------------------------------------------------------------ -class LoadManager : public ILoadManager +class LoadManager + : public ILoadManager + , public beast::InterruptibleThread::EntryPoint { private: /* Entry mapping utilization to cost. @@ -58,15 +60,18 @@ private: public: LoadManager () - : mCreditRate (100) + : m_thread ("loadmgr") + , m_logThread ("loadmgr_log") + , mCreditRate (100) , mCreditLimit (500) , mDebitWarn (-500) , mDebitLimit (-1000) - , mShutdown (false) , mArmed (false) , mDeadLock (0) , mCosts (LT_MAX) { + m_logThread.start (); + /** Flags indicating the type of load. Utilization may include any combination of: @@ -104,6 +109,8 @@ public: addCost (Cost (LT_RequestData, -5, flagDisk | flagNet)); addCost (Cost (LT_CheapQuery, -1, flagCpu)); + + UptimeTimer::getInstance ().beginManualUpdates (); } private: @@ -111,25 +118,12 @@ private: { UptimeTimer::getInstance ().endManualUpdates (); - // VFALCO TODO What is the purpose of this loop? Figure out - // a better way to do it. - for (;;) - { - boost::this_thread::sleep (boost::posix_time::milliseconds (100)); - { - boost::mutex::scoped_lock sl (mLock); - - if (!mShutdown) - return; - } - } + m_thread.interrupt (); } void startThread () { - UptimeTimer::getInstance ().beginManualUpdates (); - - boost::thread (boost::bind (&LoadManager::threadEntry, this)).detach (); + m_thread.start (this); } void canonicalize (LoadSource& source, int now) const @@ -318,29 +312,19 @@ private: // VFALCO NOTE Where's the thread object? It's not a data member... // - void threadEntry () + void threadRun () { - setCallingThreadName ("loadmgr"); - // VFALCO TODO replace this with a beast Time object? // // Initialize the clock to the current time. boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time (); - for (;;) + while (! m_thread.interruptionPoint ()) { { // VFALCO NOTE What is this lock protecting? boost::mutex::scoped_lock sl (mLock); - // Check for the shutdown flag. - if (mShutdown) - { - // VFALCO NOTE Why clear the flag now? - mShutdown = false; - return; - } - // VFALCO NOTE I think this is to reduce calls to the operating system // for retrieving the current time. // @@ -364,7 +348,7 @@ private: { // VFALCO TODO Replace this with a dedicated thread with call queue. // - boost::thread (BIND_TYPE (&logDeadlock, timeSpentDeadlocked)).detach (); + m_logThread.call (&logDeadlock, timeSpentDeadlocked); } // If we go over 500 seconds spent deadlocked, it means that the @@ -373,7 +357,6 @@ private: // assert (timeSpentDeadlocked < 500); } - } bool change; @@ -381,20 +364,20 @@ private: // VFALCO TODO Eliminate the dependence on the Application object. // Choices include constructing with the job queue / feetracker. // Another option is using an observer pattern to invert the dependency. - if (theApp->getJobQueue ().isOverloaded ()) + if (getApp().getJobQueue ().isOverloaded ()) { - WriteLog (lsINFO, LoadManager) << theApp->getJobQueue ().getJson (0); - change = theApp->getFeeTrack ().raiseLocalFee (); + WriteLog (lsINFO, LoadManager) << getApp().getJobQueue ().getJson (0); + change = getApp().getFeeTrack ().raiseLocalFee (); } else { - change = theApp->getFeeTrack ().lowerLocalFee (); + change = getApp().getFeeTrack ().lowerLocalFee (); } if (change) { // VFALCO TODO replace this with a Listener / observer and subscribe in NetworkOPs or Application - theApp->getOPs ().reportFeeChange (); + getApp().getOPs ().reportFeeChange (); } t += boost::posix_time::seconds (1); @@ -406,17 +389,21 @@ private: t = boost::posix_time::microsec_clock::universal_time (); } else + { boost::this_thread::sleep (when); + } } } private: + beast::InterruptibleThread m_thread; + beast::ThreadWithCallQueue m_logThread; + int mCreditRate; // credits gained/lost per second int mCreditLimit; // the most credits a source can have int mDebitWarn; // when a source drops below this, we warn int mDebitLimit; // when a source drops below this, we cut it off (should be negative) - bool mShutdown; bool mArmed; int mDeadLock; // Detect server deadlocks diff --git a/src/cpp/ripple/ripple_LocalCredentials.cpp b/src/cpp/ripple/ripple_LocalCredentials.cpp index adb714c9a5..a72c39f9b0 100644 --- a/src/cpp/ripple/ripple_LocalCredentials.cpp +++ b/src/cpp/ripple/ripple_LocalCredentials.cpp @@ -24,17 +24,17 @@ void LocalCredentials::start () } if (!theConfig.QUIET) - std::cerr << "NodeIdentity: " << mNodePublicKey.humanNodePublic () << std::endl; + Log::out() << "NodeIdentity: " << mNodePublicKey.humanNodePublic (); - theApp->getUNL ().start (); + getApp().getUNL ().start (); } // Retrieve network identity. bool LocalCredentials::nodeIdentityLoad () { - Database* db = theApp->getWalletDB ()->getDB (); - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); bool bSuccess = false; if (db->executeSQL ("SELECT * FROM NodeIdentity;") && db->startIterRows ()) @@ -67,7 +67,7 @@ bool LocalCredentials::nodeIdentityLoad () bool LocalCredentials::nodeIdentityCreate () { if (!theConfig.QUIET) - std::cerr << "NodeIdentity: Creating." << std::endl; + Log::out() << "NodeIdentity: Creating."; // // Generate the public and private key @@ -103,9 +103,9 @@ bool LocalCredentials::nodeIdentityCreate () // // Store the node information // - Database* db = theApp->getWalletDB ()->getDB (); + Database* db = getApp().getWalletDB ()->getDB (); - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); db->executeSQL (str (boost::format ("INSERT INTO NodeIdentity (PublicKey,PrivateKey,Dh512,Dh1024) VALUES ('%s','%s',%s,%s);") % naNodePublic.humanNodePublic () % naNodePrivate.humanNodePrivate () @@ -114,16 +114,16 @@ bool LocalCredentials::nodeIdentityCreate () // XXX Check error result. if (!theConfig.QUIET) - std::cerr << "NodeIdentity: Created." << std::endl; + Log::out() << "NodeIdentity: Created."; return true; } bool LocalCredentials::dataDelete (const std::string& strKey) { - Database* db = theApp->getRpcDB ()->getDB (); + Database* db = getApp().getRpcDB ()->getDB (); - ScopedLock sl (theApp->getRpcDB ()->getDBLock ()); + ScopedLock sl (getApp().getRpcDB ()->getDBLock ()); return db->executeSQL (str (boost::format ("DELETE FROM RPCData WHERE Key=%s;") % sqlEscape (strKey))); @@ -131,9 +131,9 @@ bool LocalCredentials::dataDelete (const std::string& strKey) bool LocalCredentials::dataFetch (const std::string& strKey, std::string& strValue) { - Database* db = theApp->getRpcDB ()->getDB (); + Database* db = getApp().getRpcDB ()->getDB (); - ScopedLock sl (theApp->getRpcDB ()->getDBLock ()); + ScopedLock sl (getApp().getRpcDB ()->getDBLock ()); bool bSuccess = false; @@ -153,9 +153,9 @@ bool LocalCredentials::dataFetch (const std::string& strKey, std::string& strVal bool LocalCredentials::dataStore (const std::string& strKey, const std::string& strValue) { - Database* db = theApp->getRpcDB ()->getDB (); + Database* db = getApp().getRpcDB ()->getDB (); - ScopedLock sl (theApp->getRpcDB ()->getDBLock ()); + ScopedLock sl (getApp().getRpcDB ()->getDBLock ()); bool bSuccess = false; diff --git a/src/cpp/ripple/ripple_LocalCredentials.h b/src/cpp/ripple/ripple_LocalCredentials.h index 8d53cb08c0..f1a9c2ce6d 100644 --- a/src/cpp/ripple/ripple_LocalCredentials.h +++ b/src/cpp/ripple/ripple_LocalCredentials.h @@ -9,7 +9,7 @@ /** Holds the cryptographic credentials identifying this instance of the server. */ -class LocalCredentials // derive from Uncopyable +class LocalCredentials : Uncopyable { public: LocalCredentials (); diff --git a/src/cpp/ripple/main.cpp b/src/cpp/ripple/ripple_Main.cpp similarity index 72% rename from src/cpp/ripple/main.cpp rename to src/cpp/ripple/ripple_Main.cpp index bd4c586283..9c4e758546 100644 --- a/src/cpp/ripple/main.cpp +++ b/src/cpp/ripple/ripple_Main.cpp @@ -6,17 +6,21 @@ namespace po = boost::program_options; -// VFALCO TODO make these singletons that initialize statically -extern void TFInit (); -extern void LEFInit (); - -using namespace std; -using namespace boost::unit_test; - void setupServer () { - theApp = IApplication::New (); - theApp->setup (); +#ifdef RLIMIT_NOFILE + struct rlimit rl; + if (getrlimit(RLIMIT_NOFILE, &rl) == 0) + { + if (rl.rlim_cur != rl.rlim_max) + { + rl.rlim_cur = rl.rlim_max; + setrlimit(RLIMIT_NOFILE, &rl); + } + } +#endif + + getApp().setup (); } void startServer () @@ -31,31 +35,33 @@ void startServer () const Json::Value& jvCommand = theConfig.RPC_STARTUP[i]; if (!theConfig.QUIET) - std::cerr << "Startup RPC: " << jvCommand << std::endl; + Log::out() << "Startup RPC: " << jvCommand; - RPCHandler rhHandler (&theApp->getOPs ()); + RPCHandler rhHandler (&getApp().getOPs ()); // VFALCO TODO Clean up this magic number LoadType loadType = LT_RPCReference; Json::Value jvResult = rhHandler.doCommand (jvCommand, RPCHandler::ADMIN, &loadType); if (!theConfig.QUIET) - std::cerr << "Result: " << jvResult << std::endl; + Log::out() << "Result: " << jvResult; } } - theApp->run (); // Blocks till we get a stop RPC. + getApp().run (); // Blocks till we get a stop RPC. } bool init_unit_test () { - theApp = IApplication::New (); + getApp (); return true; } void printHelp (const po::options_description& desc) { + using namespace std; + cerr << SYSTEM_NAME "d [options] " << endl; cerr << desc << endl; @@ -119,12 +125,46 @@ void printHelp (const po::options_description& desc) // cerr << " trust_set [] []" << endl; } -int main (int argc, char* argv[]) +int rippleMain (int argc, char** argv) { + // + // These debug heap calls do nothing in release or non Visual Studio builds. + // + + // Checks the heap at every allocation and deallocation (slow). + // + //Debug::setAlwaysCheckHeap (false); + + // Keeps freed memory blocks and fills them with a guard value. + // + //Debug::setHeapDelayedFree (false); + + // At exit, reports all memory blocks which have not been freed. + // +#if 1 + Debug::setHeapReportLeaks (false); + +#else + // This is some temporary leak checking test code + // + Debug::setHeapReportLeaks (false); + + //malloc (512); // Any leaks before this line in the output are from static initializations. + + ThreadWithCallQueue t ("test"); + GlobalPagedFreeStore::getInstance (); + t.start (); + + return 0; +#endif + + using namespace std; + setCallingThreadName ("main"); - int iResult = 0; + int iResult = 0; po::variables_map vm; // Map of options. + // VFALCO TODO Replace boost program options with something from Beast. // // Set up option parsing. // @@ -133,6 +173,8 @@ int main (int argc, char* argv[]) ("help,h", "Display this message.") ("conf", po::value (), "Specify the configuration file.") ("rpc", "Perform rpc command (default).") + ("rpc_ip", po::value (), "Specify the IP address for RPC command. Format: [':']") + ("rpc_port", po::value (), "Specify the port number for RPC command.") ("standalone,a", "Run with no peers.") ("testnet,t", "Run in test net mode.") ("unittest,u", "Perform unit tests.") @@ -144,20 +186,28 @@ int main (int argc, char* argv[]) ("start", "Start from a fresh Ledger.") ("net", "Get the initial ledger from the network.") ("fg", "Run in the foreground.") - ("import", "Import SQLite node DB into LevelDB.") + ("import", po::value (), "Import old DB into new DB.") ; // Interpret positional arguments as --parameters. po::positional_options_description p; p.add ("parameters", -1); - // - // Prepare to run - // + // These must be added before the Application object is created + NodeStore::addBackendFactory (MemoryBackendFactory::getInstance ()); + NodeStore::addBackendFactory (SqliteBackendFactory::getInstance ()); + NodeStore::addBackendFactory (LevelDBBackendFactory::getInstance ()); + NodeStore::addBackendFactory (NullBackendFactory::getInstance ()); +#if RIPPLE_HYPERLEVELDB_AVAILABLE + NodeStore::addBackendFactory (HyperLevelDBBackendFactory::getInstance ()); +#endif +#if RIPPLE_MDB_AVAILABLE + NodeStore::addBackendFactory (MdbBackendFactory::getInstance ()); +#endif if (! RandomNumbers::getInstance ().initialize ()) { - std::cerr << "Unable to add system entropy" << std::endl; + Log::out() << "Unable to add system entropy"; iResult = 2; } @@ -195,26 +245,31 @@ int main (int argc, char* argv[]) if (HaveSustain () && !iResult && !vm.count ("parameters") && !vm.count ("fg") && !vm.count ("standalone") && !vm.count ("unittest")) { - std::string logMe = DoSustain (); + std::string logMe = DoSustain (theConfig.DEBUG_LOGFILE.string()); if (!logMe.empty ()) Log (lsWARNING) << logMe; } if (vm.count ("quiet")) + { Log::setMinSeverity (lsFATAL, true); + } else if (vm.count ("verbose")) + { Log::setMinSeverity (lsTRACE, true); + } else + { Log::setMinSeverity (lsINFO, true); + } - // VFALCO TODO make these singletons that initialize statically - TFInit (); - LEFInit (); - + // VFALCO TODO make this a singleton that initializes statically + // Or could make it a SharedSingleton + // if (vm.count ("unittest")) { - unit_test_main (init_unit_test, argc, argv); + boost::unit_test::unit_test_main (init_unit_test, argc, argv); return 0; } @@ -235,7 +290,8 @@ int main (int argc, char* argv[]) if (vm.count ("start")) theConfig.START_UP = Config::FRESH; - if (vm.count ("import")) theConfig.LDB_IMPORT = true; + if (vm.count ("import")) + theConfig.DB_IMPORT = vm["import"].as (); if (vm.count ("ledger")) { @@ -254,6 +310,26 @@ int main (int argc, char* argv[]) theConfig.VALIDATION_QUORUM = 2; } + if (iResult == 0) + { + // These overrides must happen after the config file is loaded. + + // Override the RPC destination IP address + // + if (vm.count ("rpc_ip")) + { + theConfig.setRpcIpAndOptionalPort (vm ["rpc_ip"].as ()); + } + + // Override the RPC destination port number + // + if (vm.count ("rpc_port")) + { + // VFALCO TODO This should be a short. + theConfig.setRpcPort (vm ["rpc_port"].as ()); + } + } + if (iResult) { nothing (); diff --git a/src/cpp/ripple/ripple_OrderBook.h b/src/cpp/ripple/ripple_OrderBook.h index 3b2c867695..36f661afa5 100644 --- a/src/cpp/ripple/ripple_OrderBook.h +++ b/src/cpp/ripple/ripple_OrderBook.h @@ -10,7 +10,7 @@ /** Describes a serialized ledger entry for an order book. */ -class OrderBook +class OrderBook : LeakChecked { public: typedef boost::shared_ptr pointer; diff --git a/src/cpp/ripple/ripple_PathRequest.cpp b/src/cpp/ripple/ripple_PathRequest.cpp index cfe7d1f912..7fd2bf2643 100644 --- a/src/cpp/ripple/ripple_PathRequest.cpp +++ b/src/cpp/ripple/ripple_PathRequest.cpp @@ -37,7 +37,7 @@ bool PathRequest::isValid (Ledger::ref lrLedger) if (bValid) { - AccountState::pointer asSrc = theApp->getOPs ().getAccountState (lrLedger, raSrcAccount); + AccountState::pointer asSrc = getApp().getOPs ().getAccountState (lrLedger, raSrcAccount); if (!asSrc) { @@ -47,7 +47,7 @@ bool PathRequest::isValid (Ledger::ref lrLedger) } else { - AccountState::pointer asDst = theApp->getOPs ().getAccountState (lrLedger, raDstAccount); + AccountState::pointer asDst = getApp().getOPs ().getAccountState (lrLedger, raDstAccount); Json::Value jvDestCur; if (!asDst) diff --git a/src/cpp/ripple/ripple_PathRequest.h b/src/cpp/ripple/ripple_PathRequest.h index 9e48fed288..1f988049c7 100644 --- a/src/cpp/ripple/ripple_PathRequest.h +++ b/src/cpp/ripple/ripple_PathRequest.h @@ -10,8 +10,6 @@ // A pathfinding request submitted by a client // The request issuer must maintain a strong pointer -class InfoSub; -class STAmount; class RippleLineCache; // Return values from parseJson <0 = invalid, >0 = valid diff --git a/src/cpp/ripple/ripple_Pathfinder.cpp b/src/cpp/ripple/ripple_Pathfinder.cpp index 9f7b6811e7..d81091a72e 100644 --- a/src/cpp/ripple/ripple_Pathfinder.cpp +++ b/src/cpp/ripple/ripple_Pathfinder.cpp @@ -162,9 +162,9 @@ Pathfinder::Pathfinder (RippleLineCache::ref cache, bValid = true; - theApp->getOrderBookDB ().setup (mLedger); + getApp().getOrderBookDB ().setup (mLedger); - m_loadEvent = theApp->getJobQueue ().getLoadEvent (jtPATH_FIND, "FindPath"); + m_loadEvent = getApp().getJobQueue ().getLoadEvent (jtPATH_FIND, "FindPath"); // Construct the default path for later comparison. @@ -399,7 +399,7 @@ bool Pathfinder::findPaths (const unsigned int iMaxSteps, const unsigned int iMa { // Cursor is for XRP, continue with qualifying books: XRP -> non-XRP std::vector xrpBooks; - theApp->getOrderBookDB ().getBooksByTakerPays (ACCOUNT_XRP, CURRENCY_XRP, xrpBooks); + getApp().getOrderBookDB ().getBooksByTakerPays (ACCOUNT_XRP, CURRENCY_XRP, xrpBooks); BOOST_FOREACH (OrderBook::ref book, xrpBooks) { // New end is an order book with the currency and issuer. @@ -553,7 +553,7 @@ bool Pathfinder::findPaths (const unsigned int iMaxSteps, const unsigned int iMa // XXX Flip argument order to norm. (currency, issuer) std::vector books; - theApp->getOrderBookDB ().getBooksByTakerPays (speEnd.mIssuerID, speEnd.mCurrencyID, books); + getApp().getOrderBookDB ().getBooksByTakerPays (speEnd.mIssuerID, speEnd.mCurrencyID, books); BOOST_FOREACH (OrderBook::ref book, books) { diff --git a/src/cpp/ripple/ripple_Peer.cpp b/src/cpp/ripple/ripple_Peer.cpp index d140a2e776..227ab61324 100644 --- a/src/cpp/ripple/ripple_Peer.cpp +++ b/src/cpp/ripple/ripple_Peer.cpp @@ -4,9 +4,6 @@ */ //============================================================================== -// VFALCO TODO make this an inline function -#define ADDRESS(p) strHex(uint64( ((char*) p) - ((char*) 0))) - SETUP_LOG (Peer) class PeerImp; @@ -24,6 +21,8 @@ class PeerImp : public Peer , public CountedObject { public: + static char const* getCountedObjectName () { return "Peer"; } + PeerImp (boost::asio::io_service & io_service, boost::asio::ssl::context & ctx, uint64 peerId, @@ -31,10 +30,11 @@ public: void handleConnect (const boost::system::error_code & error, boost::asio::ip::tcp::resolver::iterator it); - std::string& getIP () + std::string const& getIP () { return mIpPort.first; } + std::string getDisplayName () { return mCluster ? mNodeName : mIpPort.first; @@ -70,6 +70,10 @@ public: { return mHelloed && !mDetaching; } + bool isInCluster () const + { + return mCluster; + } bool isInbound () const { return mInbound; @@ -114,8 +118,8 @@ private: bool mCluster; // Node in our cluster RippleAddress mNodePublic; // Node public key of peer. std::string mNodeName; - ipPort mIpPort; - ipPort mIpPortConnect; + IPAndPortNumber mIpPort; + IPAndPortNumber mIpPortConnect; uint256 mCookieHash; uint64 mPeerId; bool mPrivate; // Keep peer IP private. @@ -160,6 +164,7 @@ private: void sendHello (); void recvHello (protocol::TMHello & packet); + void recvCluster (protocol::TMCluster & packet); void recvTransaction (protocol::TMTransaction & packet, ScopedLock & MasterLockHolder); void recvValidation (const boost::shared_ptr& packet, ScopedLock & MasterLockHolder); void recvGetValidation (protocol::TMGetValidations & packet); @@ -205,15 +210,15 @@ PeerImp::PeerImp (boost::asio::io_service& io_service, boost::asio::ssl::context mActivityTimer (io_service), mIOStrand (io_service) { - WriteLog (lsDEBUG, Peer) << "CREATING PEER: " << ADDRESS (this); + WriteLog (lsDEBUG, Peer) << "CREATING PEER: " << addressToString (this); } void PeerImp::handleWrite (const boost::system::error_code& error, size_t bytes_transferred) { // Call on IO strand -#ifdef DEBUG +#ifdef BEAST_DEBUG // if (!error) - // std::cerr << "PeerImp::handleWrite bytes: "<< bytes_transferred << std::endl; + // Log::out() << "PeerImp::handleWrite bytes: "<< bytes_transferred; #endif mSendingPacket.reset (); @@ -225,7 +230,7 @@ void PeerImp::handleWrite (const boost::system::error_code& error, size_t bytes_ } else if (error) { - WriteLog (lsINFO, Peer) << "Peer: Write: Error: " << ADDRESS (this) << ": bytes=" << bytes_transferred << ": " << error.category ().name () << ": " << error.message () << ": " << error; + WriteLog (lsINFO, Peer) << "Peer: Write: Error: " << addressToString (this) << ": bytes=" << bytes_transferred << ": " << error.category ().name () << ": " << error.message () << ": " << error; detach ("hw", true); } @@ -247,15 +252,19 @@ void PeerImp::setIpPort (const std::string& strIP, int iPort) mLoad.rename (strIP); WriteLog (lsDEBUG, Peer) << "Peer: Set: " - << ADDRESS (this) << "> " + << addressToString (this) << "> " << (mNodePublic.isValid () ? mNodePublic.humanNodePublic () : "-") << " " << getIP () << " " << getPort (); } void PeerImp::detach (const char* rsn, bool onIOStrand) { + // VFALCO NOTE So essentially, detach() is really two different functions + // depending on the value of onIOStrand. + // TODO Clean this up. + // if (!onIOStrand) { - mIOStrand.post (boost::bind (&Peer::detach, shared_from_this (), rsn, true)); + mIOStrand.post (BIND_TYPE (&Peer::detach, shared_from_this (), rsn, true)); return; } @@ -266,7 +275,7 @@ void PeerImp::detach (const char* rsn, bool onIOStrand) CondLog (mCluster, lsWARNING, Peer) << "Cluster peer detach \"" << mNodeName << "\": " << rsn; /* WriteLog (lsDEBUG, Peer) << "Peer: Detach: " - << ADDRESS(this) << "> " + << addressToString(this) << "> " << rsn << ": " << (mNodePublic.isValid() ? mNodePublic.humanNodePublic() : "-") << " " << getIP() << " " << getPort(); */ @@ -280,7 +289,7 @@ void PeerImp::detach (const char* rsn, bool onIOStrand) if (mNodePublic.isValid ()) { - theApp->getPeers ().peerDisconnected (shared_from_this (), mNodePublic); + getApp().getPeers ().peerDisconnected (shared_from_this (), mNodePublic); mNodePublic.clear (); // Be idempotent. } @@ -289,14 +298,14 @@ void PeerImp::detach (const char* rsn, bool onIOStrand) { // Connection might be part of scanning. Inform connect failed. // Might need to scan. Inform connection closed. - theApp->getPeers ().peerClosed (shared_from_this (), mIpPort.first, mIpPort.second); + getApp().getPeers ().peerClosed (shared_from_this (), mIpPort.first, mIpPort.second); mIpPort.first.clear (); // Be idempotent. } /* WriteLog (lsDEBUG, Peer) << "Peer: Detach: " - << ADDRESS(this) << "< " + << addressToString(this) << "< " << rsn << ": " << (mNodePublic.isValid() ? mNodePublic.humanNodePublic() : "-") << " " << getIP() << " " << getPort(); */ @@ -340,7 +349,7 @@ void PeerImp::handleVerifyTimer (const boost::system::error_code& ecResult) if (ecResult == boost::asio::error::operation_aborted) { // Timer canceled because deadline no longer needed. - // std::cerr << "Deadline cancelled." << std::endl; + // Log::out() << "Deadline cancelled."; nothing (); // Aborter is done. } @@ -370,7 +379,7 @@ void PeerImp::connect (const std::string& strIp, int iPort) boost::asio::ip::tcp::resolver::query query (strIp, boost::lexical_cast (iPortAct), boost::asio::ip::resolver_query_base::numeric_host | boost::asio::ip::resolver_query_base::numeric_service); - boost::asio::ip::tcp::resolver resolver (theApp->getIOService ()); + boost::asio::ip::tcp::resolver resolver (getApp().getIOService ()); boost::system::error_code err; boost::asio::ip::tcp::resolver::iterator itrEndpoint = resolver.resolve (query, err); @@ -399,7 +408,7 @@ void PeerImp::connect (const std::string& strIp, int iPort) if (!err) { - WriteLog (lsINFO, Peer) << "Peer: Connect: Outbound: " << ADDRESS (this) << ": " << mIpPort.first << " " << mIpPort.second; + WriteLog (lsINFO, Peer) << "Peer: Connect: Outbound: " << addressToString (this) << ": " << mIpPort.first << " " << mIpPort.second; boost::asio::async_connect ( getSocket (), @@ -482,7 +491,7 @@ void PeerImp::connected (const boost::system::error_code& error) { // Not redundant ip and port, handshake, and start. - WriteLog (lsINFO, Peer) << "Peer: Inbound: Accepted: " << ADDRESS (this) << ": " << strIp << " " << iPort; + WriteLog (lsINFO, Peer) << "Peer: Inbound: Accepted: " << addressToString (this) << ": " << strIp << " " << iPort; mSocketSsl.set_verify_mode (boost::asio::ssl::verify_none); @@ -492,7 +501,7 @@ void PeerImp::connected (const boost::system::error_code& error) } else if (!mDetaching) { - WriteLog (lsINFO, Peer) << "Peer: Inbound: Error: " << ADDRESS (this) << ": " << strIp << " " << iPort << " : " << error.category ().name () << ": " << error.message () << ": " << error; + WriteLog (lsINFO, Peer) << "Peer: Inbound: Error: " << addressToString (this) << ": " << strIp << " " << iPort << " : " << error.category ().name () << ": " << error.message () << ": " << error; detach ("ctd", false); } @@ -519,7 +528,7 @@ void PeerImp::sendPacket (const PackedMessage::pointer& packet, bool onStrand) { if (!onStrand) { - mIOStrand.post (boost::bind (&Peer::sendPacket, shared_from_this (), packet, true)); + mIOStrand.post (BIND_TYPE (&Peer::sendPacket, shared_from_this (), packet, true)); return; } @@ -621,7 +630,7 @@ void PeerImp::handleReadBody (const boost::system::error_code& error) WriteLog (lsINFO, Peer) << "Peer: Body: Error: " << getIP () << ": " << error.category ().name () << ": " << error.message () << ": " << error; } - boost::recursive_mutex::scoped_lock sl (theApp->getMasterLock ()); + boost::recursive_mutex::scoped_lock sl (getApp().getMasterLock ()); detach ("hrb", true); return; } @@ -634,15 +643,15 @@ void PeerImp::processReadBuffer () { // must not hold peer lock int type = PackedMessage::getType (mReadbuf); -#ifdef DEBUG - // std::cerr << "PRB(" << type << "), len=" << (mReadbuf.size()-PackedMessage::kHeaderBytes) << std::endl; +#ifdef BEAST_DEBUG + // Log::out() << "PRB(" << type << "), len=" << (mReadbuf.size()-PackedMessage::kHeaderBytes); #endif - // std::cerr << "PeerImp::processReadBuffer: " << mIpPort.first << " " << mIpPort.second << std::endl; + // Log::out() << "PeerImp::processReadBuffer: " << mIpPort.first << " " << mIpPort.second; - LoadEvent::autoptr event (theApp->getJobQueue ().getLoadEventAP (jtPEER, "PeerImp::read")); + LoadEvent::autoptr event (getApp().getJobQueue ().getLoadEventAP (jtPEER, "PeerImp::read")); - ScopedLock sl (theApp->getMasterLock ()); + ScopedLock sl (getApp().getMasterLock ()); // If connected and get a mtHELLO or if not connected and get a non-mtHELLO, wrong message was sent. if (mHelloed == (type == protocol::mtHELLO)) @@ -666,6 +675,17 @@ void PeerImp::processReadBuffer () } break; + case protocol::mtCLUSTER: + { + event->reName ("PeerImp::cluster"); + protocol::TMCluster msg; + + if (msg.ParseFromArray (&mReadbuf[PackedMessage::kHeaderBytes], mReadbuf.size () - PackedMessage::kHeaderBytes)) + recvCluster (msg); + else + WriteLog (lsWARNING, Peer) << "parse error: " << type; + } + case protocol::mtERROR_MSG: { event->reName ("PeerImp::errormessage"); @@ -899,11 +919,8 @@ void PeerImp::processReadBuffer () default: event->reName ("PeerImp::unknown"); - if (type != 5) - { // TEMPORARY: Don't warn on cluster message - WriteLog (lsWARNING, Peer) << "Unknown Msg: " << type; - WriteLog (lsWARNING, Peer) << strHex (&mReadbuf[0], mReadbuf.size ()); - } + WriteLog (lsWARNING, Peer) << "Unknown Msg: " << type; + WriteLog (lsWARNING, Peer) << strHex (&mReadbuf[0], mReadbuf.size ()); } } } @@ -917,11 +934,11 @@ void PeerImp::recvHello (protocol::TMHello& packet) mActivityTimer.async_wait (mIOStrand.wrap (boost::bind (&PeerImp::handlePingTimer, boost::static_pointer_cast (shared_from_this ()), boost::asio::placeholders::error))); - uint32 ourTime = theApp->getOPs ().getNetworkTimeNC (); + uint32 ourTime = getApp().getOPs ().getNetworkTimeNC (); uint32 minTime = ourTime - 20; uint32 maxTime = ourTime + 20; -#ifdef DEBUG +#ifdef BEAST_DEBUG if (packet.has_nettime ()) { @@ -974,7 +991,7 @@ void PeerImp::recvHello (protocol::TMHello& packet) (packet.protoversion () >> 16) << "." << (packet.protoversion () & 0xFF); mHello = packet; - if (theApp->getUNL ().nodeInCluster (mNodePublic, mNodeName)) + if (getApp().getUNL ().nodeInCluster (mNodePublic, mNodeName)) { mCluster = true; mLoad.setPrivileged (); @@ -990,10 +1007,10 @@ void PeerImp::recvHello (protocol::TMHello& packet) if (mClientConnect) { // If we connected due to scan, no longer need to scan. - theApp->getPeers ().peerVerified (shared_from_this ()); + getApp().getPeers ().peerVerified (shared_from_this ()); } - if (! theApp->getPeers ().peerConnected (shared_from_this (), mNodePublic, getIP (), getPort ())) + if (! getApp().getPeers ().peerConnected (shared_from_this (), mNodePublic, getIP (), getPort ())) { // Already connected, self, or some other reason. WriteLog (lsINFO, Peer) << "Recv(Hello): Disconnect: Extraneous connection."; @@ -1020,7 +1037,7 @@ void PeerImp::recvHello (protocol::TMHello& packet) // Don't save IP address if the node wants privacy. // Note: We don't go so far as to delete it. If a node which has previously announced itself now wants // privacy, it should at least change its port. - theApp->getPeers ().savePeer (strIP, iPort, IUniqueNodeList::vsInbound); + getApp().getPeers ().savePeer (strIP, iPort, UniqueNodeList::vsInbound); } } @@ -1074,20 +1091,20 @@ static void checkTransaction (Job&, int flags, SerializedTransaction::pointer st if (tx->getStatus () == INVALID) { - theApp->getHashRouter ().setFlag (stx->getTransactionID (), SF_BAD); + getApp().getHashRouter ().setFlag (stx->getTransactionID (), SF_BAD); Peer::applyLoadCharge (peer, LT_InvalidSignature); return; } else - theApp->getHashRouter ().setFlag (stx->getTransactionID (), SF_SIGGOOD); + getApp().getHashRouter ().setFlag (stx->getTransactionID (), SF_SIGGOOD); - theApp->getOPs ().processTransaction (tx, isSetBit (flags, SF_TRUSTED), false); + getApp().getOPs ().processTransaction (tx, isSetBit (flags, SF_TRUSTED), false); #ifndef TRUST_NETWORK } catch (...) { - theApp->getHashRouter ().setFlags (stx->getTransactionID (), SF_BAD); + getApp().getHashRouter ().setFlags (stx->getTransactionID (), SF_BAD); applyLoadCharge (peer, LT_InvalidRequest); } @@ -1110,7 +1127,7 @@ void PeerImp::recvTransaction (protocol::TMTransaction& packet, ScopedLock& Mast int flags; - if (! theApp->getHashRouter ().addSuppressionPeer (txID, mPeerId, flags)) + if (! getApp().getHashRouter ().addSuppressionPeer (txID, mPeerId, flags)) { // we have seen this transaction recently if (isSetBit (flags, SF_BAD)) @@ -1123,7 +1140,7 @@ void PeerImp::recvTransaction (protocol::TMTransaction& packet, ScopedLock& Mast return; } - if (theApp->getMasterTransaction().fetch(txID, true)) + if (getApp().getMasterTransaction().fetch(txID, true)) { WriteLog (lsDEBUG, Peer) << "Peer " << getDisplayName() << " send old TX " << txID; applyLoadCharge (LT_InvalidRequest); @@ -1135,21 +1152,22 @@ void PeerImp::recvTransaction (protocol::TMTransaction& packet, ScopedLock& Mast if (mCluster) flags |= SF_TRUSTED | SF_SIGGOOD; - if (theApp->getJobQueue().getJobCount(jtTRANSACTION) > 100) + if (getApp().getJobQueue().getJobCount(jtTRANSACTION) > 100) WriteLog(lsINFO, Peer) << "Transaction queue is full"; - else if (theApp->getLedgerMaster().getValidatedLedgerAge() > 240) + else if (getApp().getLedgerMaster().getValidatedLedgerAge() > 240) WriteLog(lsINFO, Peer) << "No new transactions until synchronized"; else - theApp->getJobQueue ().addJob (jtTRANSACTION, "recvTransction->checkTransaction", + getApp().getJobQueue ().addJob (jtTRANSACTION, "recvTransction->checkTransaction", BIND_TYPE (&checkTransaction, P_1, flags, stx, boost::weak_ptr (shared_from_this ()))); #ifndef TRUST_NETWORK } catch (...) { -#ifdef DEBUG - std::cerr << "Transaction from peer fails validity tests" << std::endl; +#ifdef BEAST_DEBUG + Log::out() << "Transaction from peer fails validity tests"; Json::StyledStreamWriter w; + // VFALCO NOTE This bypasses the Log bottleneck w.write (std::cerr, tx->getJson (0)); #endif return; @@ -1161,7 +1179,8 @@ void PeerImp::recvTransaction (protocol::TMTransaction& packet, ScopedLock& Mast // Called from our JobQueue static void checkPropose (Job& job, boost::shared_ptr packet, - LedgerProposal::pointer proposal, uint256 consensusLCL, RippleAddress nodePublic, boost::weak_ptr peer) + LedgerProposal::pointer proposal, uint256 consensusLCL, RippleAddress nodePublic, + boost::weak_ptr peer, bool fromCluster) { bool sigGood = false; bool isTrusted = (job.getType () == jtPROPOSAL_t); @@ -1179,7 +1198,7 @@ static void checkPropose (Job& job, boost::shared_ptr pa WriteLog (lsTRACE, Peer) << "proposal with previous ledger"; memcpy (prevLedger.begin (), set.previousledger ().data (), 256 / 8); - if (!proposal->checkSign (set.signature ())) + if (!fromCluster && !proposal->checkSign (set.signature ())) { Peer::pointer p = peer.lock (); WriteLog (lsWARNING, Peer) << "proposal with previous ledger fails signature check: " << @@ -1205,15 +1224,15 @@ static void checkPropose (Job& job, boost::shared_ptr pa } if (isTrusted) - theApp->getOPs ().processTrustedProposal (proposal, packet, nodePublic, prevLedger, sigGood); + getApp().getOPs ().processTrustedProposal (proposal, packet, nodePublic, prevLedger, sigGood); else if (sigGood && (prevLedger == consensusLCL)) { // relay untrusted proposal WriteLog (lsTRACE, Peer) << "relaying untrusted proposal"; std::set peers; - theApp->getHashRouter ().swapSet (proposal->getHashRouter (), peers, SF_RELAYED); + getApp().getHashRouter ().swapSet (proposal->getHashRouter (), peers, SF_RELAYED); PackedMessage::pointer message = boost::make_shared (set, protocol::mtPROPOSE_LEDGER); - theApp->getPeers ().relayMessageBut (peers, message); + getApp().getPeers ().relayMessageBut (peers, message); } else WriteLog (lsDEBUG, Peer) << "Not relaying untrusted proposal"; @@ -1257,7 +1276,7 @@ void PeerImp::recvPropose (const boost::shared_ptr& pack uint256 suppression = s.getSHA512Half (); - if (! theApp->getHashRouter ().addSuppressionPeer (suppression, mPeerId)) + if (! getApp().getHashRouter ().addSuppressionPeer (suppression, mPeerId)) { WriteLog (lsTRACE, Peer) << "Received duplicate proposal from peer " << mPeerId; return; @@ -1271,8 +1290,8 @@ void PeerImp::recvPropose (const boost::shared_ptr& pack return; } - bool isTrusted = theApp->getUNL ().nodeInUNL (signerPublic); - if (!isTrusted && theApp->getFeeTrack ().isLoaded ()) + bool isTrusted = getApp().getUNL ().nodeInUNL (signerPublic); + if (!isTrusted && getApp().getFeeTrack ().isLoadedLocal ()) { WriteLog (lsDEBUG, Peer) << "Dropping untrusted proposal due to load"; return; @@ -1280,14 +1299,14 @@ void PeerImp::recvPropose (const boost::shared_ptr& pack WriteLog (lsTRACE, Peer) << "Received " << (isTrusted ? "trusted" : "UNtrusted") << " proposal from " << mPeerId; - uint256 consensusLCL = theApp->getOPs ().getConsensusLCL (); + uint256 consensusLCL = getApp().getOPs ().getConsensusLCL (); LedgerProposal::pointer proposal = boost::make_shared ( prevLedger.isNonZero () ? prevLedger : consensusLCL, set.proposeseq (), proposeHash, set.closetime (), signerPublic, suppression); - theApp->getJobQueue ().addJob (isTrusted ? jtPROPOSAL_t : jtPROPOSAL_ut, "recvPropose->checkPropose", + getApp().getJobQueue ().addJob (isTrusted ? jtPROPOSAL_t : jtPROPOSAL_ut, "recvPropose->checkPropose", BIND_TYPE (&checkPropose, P_1, packet, proposal, consensusLCL, - mNodePublic, boost::weak_ptr (shared_from_this ()))); + mNodePublic, boost::weak_ptr (shared_from_this ()), mCluster)); } void PeerImp::recvHaveTxSet (protocol::TMHaveTransactionSet& packet) @@ -1301,12 +1320,16 @@ void PeerImp::recvHaveTxSet (protocol::TMHaveTransactionSet& packet) } uint256 hash; + + // VFALCO TODO There should be no use of memcpy() throughout the program. + // TODO Clean up this magic number + // memcpy (hash.begin (), packet.hash ().data (), 32); if (packet.status () == protocol::tsHAVE) addTxSet (hash); - if (!theApp->getOPs ().hasTXSet (shared_from_this (), hash, packet.status ())) + if (!getApp().getOPs ().hasTXSet (shared_from_this (), hash, packet.status ())) applyLoadCharge (LT_UnwantedData); } @@ -1335,11 +1358,11 @@ static void checkValidation (Job&, SerializedValidation::pointer val, uint256 si std::set peers; - if (theApp->getOPs ().recvValidation (val, source) && - theApp->getHashRouter ().swapSet (signingHash, peers, SF_RELAYED)) + if (getApp().getOPs ().recvValidation (val, source) && + getApp().getHashRouter ().swapSet (signingHash, peers, SF_RELAYED)) { PackedMessage::pointer message = boost::make_shared (*packet, protocol::mtVALIDATION); - theApp->getPeers ().relayMessageBut (peers, message); + getApp().getPeers ().relayMessageBut (peers, message); } } @@ -1375,15 +1398,15 @@ void PeerImp::recvValidation (const boost::shared_ptr& p uint256 signingHash = val->getSigningHash (); - if (! theApp->getHashRouter ().addSuppressionPeer (signingHash, mPeerId)) + if (! getApp().getHashRouter ().addSuppressionPeer (signingHash, mPeerId)) { WriteLog (lsTRACE, Peer) << "Validation is duplicate"; return; } - bool isTrusted = theApp->getUNL ().nodeInUNL (val->getSignerPublic ()); - if (isTrusted || !theApp->getFeeTrack ().isLoaded ()) - theApp->getJobQueue ().addJob (isTrusted ? jtVALIDATION_t : jtVALIDATION_ut, "recvValidation->checkValidation", + bool isTrusted = getApp().getUNL ().nodeInUNL (val->getSignerPublic ()); + if (isTrusted || !getApp().getFeeTrack ().isLoadedLocal ()) + getApp().getJobQueue ().addJob (isTrusted ? jtVALIDATION_t : jtVALIDATION_ut, "recvValidation->checkValidation", BIND_TYPE (&checkValidation, P_1, val, signingHash, isTrusted, mCluster, packet, boost::weak_ptr (shared_from_this ()))); else @@ -1400,6 +1423,32 @@ void PeerImp::recvValidation (const boost::shared_ptr& p #endif } +void PeerImp::recvCluster (protocol::TMCluster& packet) +{ + if (!mCluster) + { + applyLoadCharge(LT_UnwantedData); + return; + } + + for (int i = 0; i < packet.clusternodes().size(); ++i) + { + protocol::TMClusterNode const& node = packet.clusternodes(i); + + std::string name; + if (node.has_nodename()) + name = node.nodename(); + ClusterNodeStatus s(name, node.nodeload(), node.reporttime()); + + RippleAddress nodePub; + nodePub.setNodePublic(node.publickey()); + + getApp().getUNL().nodeUpdate(nodePub, s); + } + + getApp().getFeeTrack().setClusterFee(getApp().getUNL().getClusterFee()); +} + void PeerImp::recvGetValidation (protocol::TMGetValidations& packet) { } @@ -1420,7 +1469,7 @@ void PeerImp::recvGetPeers (protocol::TMGetPeers& packet, ScopedLock& MasterLock MasterLockHolder.unlock (); std::vector addrs; - theApp->getPeers ().getTopNAddrs (30, addrs); + getApp().getPeers ().getTopNAddrs (30, addrs); if (!addrs.empty ()) { @@ -1438,7 +1487,7 @@ void PeerImp::recvGetPeers (protocol::TMGetPeers& packet, ScopedLock& MasterLock addr->set_ipv4 (inet_addr (strIP.c_str ())); addr->set_ipv4port (iPort); - //WriteLog (lsINFO, Peer) << "Peer: Teaching: " << ADDRESS(this) << ": " << n << ": " << strIP << " " << iPort; + //WriteLog (lsINFO, Peer) << "Peer: Teaching: " << addressToString(this) << ": " << n << ": " << strIP << " " << iPort; } PackedMessage::pointer message = boost::make_shared (peers, protocol::mtPEERS); @@ -1460,9 +1509,9 @@ void PeerImp::recvPeers (protocol::TMPeers& packet) if (strIP != "0.0.0.0" && strIP != "127.0.0.1") { - //WriteLog (lsINFO, Peer) << "Peer: Learning: " << ADDRESS(this) << ": " << i << ": " << strIP << " " << iPort; + //WriteLog (lsINFO, Peer) << "Peer: Learning: " << addressToString(this) << ": " << i << ": " << strIP << " " << iPort; - theApp->getPeers ().savePeer (strIP, iPort, IUniqueNodeList::vsTold); + getApp().getPeers ().savePeer (strIP, iPort, UniqueNodeList::vsTold); } } } @@ -1501,7 +1550,7 @@ void PeerImp::recvGetObjectByHash (const boost::shared_ptrgetHashedObjectStore ().retrieve (hash); + NodeObject::pointer hObj = getApp().getNodeStore ().retrieve (hash); if (hObj) { @@ -1542,7 +1591,7 @@ void PeerImp::recvGetObjectByHash (const boost::shared_ptrgetOPs ().haveLedger (pLSeq); + pLDo = !getApp().getOPs ().haveLedger (pLSeq); if (!pLDo) { @@ -1561,7 +1610,7 @@ void PeerImp::recvGetObjectByHash (const boost::shared_ptr data = boost::make_shared< Blob > (obj.data ().begin (), obj.data ().end ()); - theApp->getOPs ().addFetchPack (hash, data); + getApp().getOPs ().addFetchPack (hash, data); } } } @@ -1569,7 +1618,7 @@ void PeerImp::recvGetObjectByHash (const boost::shared_ptrgetOPs ().gotFetchPack (progress, pLSeq); + getApp().getOPs ().gotFetchPack (progress, pLSeq); } } @@ -1615,7 +1664,7 @@ void PeerImp::recvProofWork (protocol::TMProofWork& packet) uint256 response; memcpy (response.begin (), packet.response ().data (), 256 / 8); - POWResult r = theApp->getProofOfWorkFactory ().checkProof (packet.token (), response); + POWResult r = getApp().getProofOfWorkFactory ().checkProof (packet.token (), response); if (r == powOK) { @@ -1663,7 +1712,7 @@ void PeerImp::recvProofWork (protocol::TMProofWork& packet) } #if 0 // Until proof of work is completed, don't do it - theApp->getJobQueue ().addJob ( + getApp().getJobQueue ().addJob ( jtPROOFWORK, "recvProof->doProof", BIND_TYPE (&PeerImp::doProofOfWork, P_1, boost::weak_ptr (shared_from_this ()), pow)); @@ -1680,7 +1729,7 @@ void PeerImp::recvStatus (protocol::TMStatusChange& packet) WriteLog (lsTRACE, Peer) << "Received status change from peer " << getIP (); if (!packet.has_networktime ()) - packet.set_networktime (theApp->getOPs ().getNetworkTimeNC ()); + packet.set_networktime (getApp().getOPs ().getNetworkTimeNC ()); if (!mLastStatus.has_newstatus () || packet.has_newstatus ()) mLastStatus = packet; @@ -1756,14 +1805,14 @@ void PeerImp::recvGetLedger (protocol::TMGetLedger& packet, ScopedLock& MasterLo uint256 txHash; memcpy (txHash.begin (), packet.ledgerhash ().data (), 32); - map = theApp->getOPs ().getTXMap (txHash); + map = getApp().getOPs ().getTXMap (txHash); if (!map) { if (packet.has_querytype () && !packet.has_requestcookie ()) { WriteLog (lsDEBUG, Peer) << "Trying to route TX set request"; - std::vector peerList = theApp->getPeers ().getPeerVector (); + std::vector peerList = getApp().getPeers ().getPeerVector (); std::vector usablePeers; BOOST_FOREACH (Peer::ref peer, peerList) { @@ -1814,7 +1863,7 @@ void PeerImp::recvGetLedger (protocol::TMGetLedger& packet, ScopedLock& MasterLo memcpy (ledgerhash.begin (), packet.ledgerhash ().data (), 32); logMe += "LedgerHash:"; logMe += ledgerhash.GetHex (); - ledger = theApp->getLedgerMaster ().getLedgerByHash (ledgerhash); + ledger = getApp().getLedgerMaster ().getLedgerByHash (ledgerhash); CondLog (!ledger, lsTRACE, Peer) << "Don't have ledger " << ledgerhash; @@ -1825,7 +1874,7 @@ void PeerImp::recvGetLedger (protocol::TMGetLedger& packet, ScopedLock& MasterLo if (packet.has_ledgerseq ()) seq = packet.ledgerseq (); - std::vector peerList = theApp->getPeers ().getPeerVector (); + std::vector peerList = getApp().getPeers ().getPeerVector (); std::vector usablePeers; BOOST_FOREACH (Peer::ref peer, peerList) { @@ -1848,17 +1897,17 @@ void PeerImp::recvGetLedger (protocol::TMGetLedger& packet, ScopedLock& MasterLo } else if (packet.has_ledgerseq ()) { - ledger = theApp->getLedgerMaster ().getLedgerBySeq (packet.ledgerseq ()); + ledger = getApp().getLedgerMaster ().getLedgerBySeq (packet.ledgerseq ()); CondLog (!ledger, lsDEBUG, Peer) << "Don't have ledger " << packet.ledgerseq (); } else if (packet.has_ltype () && (packet.ltype () == protocol::ltCURRENT)) - ledger = theApp->getLedgerMaster ().getCurrentLedger (); + ledger = getApp().getLedgerMaster ().getCurrentLedger (); else if (packet.has_ltype () && (packet.ltype () == protocol::ltCLOSED) ) { - ledger = theApp->getLedgerMaster ().getClosedLedger (); + ledger = getApp().getLedgerMaster ().getClosedLedger (); if (ledger && !ledger->isClosed ()) - ledger = theApp->getLedgerMaster ().getLedgerBySeq (ledger->getLedgerSeq () - 1); + ledger = getApp().getLedgerMaster ().getLedgerBySeq (ledger->getLedgerSeq () - 1); } else { @@ -2029,7 +2078,7 @@ void PeerImp::recvLedger (const boost::shared_ptr& packe if (packet.has_requestcookie ()) { - Peer::pointer target = theApp->getPeers ().getPeerById (packet.requestcookie ()); + Peer::pointer target = getApp().getPeers ().getPeerById (packet.requestcookie ()); if (target) { @@ -2077,7 +2126,7 @@ void PeerImp::recvLedger (const boost::shared_ptr& packe nodeData.push_back (Blob (node.nodedata ().begin (), node.nodedata ().end ())); } - SHAMapAddNode san = theApp->getOPs ().gotTXData (shared_from_this (), hash, nodeIDs, nodeData); + SHAMapAddNode san = getApp().getOPs ().gotTXData (shared_from_this (), hash, nodeIDs, nodeData); if (san.isInvalid ()) applyLoadCharge (LT_UnwantedData); @@ -2085,9 +2134,9 @@ void PeerImp::recvLedger (const boost::shared_ptr& packe return; } - if (theApp->getInboundLedgers ().awaitLedgerData (hash)) - theApp->getJobQueue ().addJob (jtLEDGER_DATA, "gotLedgerData", - BIND_TYPE (&InboundLedgers::gotLedgerData, &theApp->getInboundLedgers (), + if (getApp().getInboundLedgers ().awaitLedgerData (hash)) + getApp().getJobQueue ().addLimitJob (jtLEDGER_DATA, "gotLedgerData", 2, + BIND_TYPE (&InboundLedgers::gotLedgerData, &getApp().getInboundLedgers (), P_1, hash, packet_ptr, boost::weak_ptr (shared_from_this ()))); else applyLoadCharge (LT_UnwantedData); @@ -2181,21 +2230,21 @@ void PeerImp::sendHello () getSessionCookie (strCookie); mCookieHash = Serializer::getSHA512Half (strCookie); - theApp->getLocalCredentials ().getNodePrivate ().signNodePrivate (mCookieHash, vchSig); + getApp().getLocalCredentials ().getNodePrivate ().signNodePrivate (mCookieHash, vchSig); protocol::TMHello h; h.set_protoversion (MAKE_VERSION_INT (PROTO_VERSION_MAJOR, PROTO_VERSION_MINOR)); h.set_protoversionmin (MAKE_VERSION_INT (MIN_PROTO_MAJOR, MIN_PROTO_MINOR)); h.set_fullversion (SERVER_VERSION); - h.set_nettime (theApp->getOPs ().getNetworkTimeNC ()); - h.set_nodepublic (theApp->getLocalCredentials ().getNodePublic ().humanNodePublic ()); + h.set_nettime (getApp().getOPs ().getNetworkTimeNC ()); + h.set_nodepublic (getApp().getLocalCredentials ().getNodePublic ().humanNodePublic ()); h.set_nodeproof (&vchSig[0], vchSig.size ()); h.set_ipv4port (theConfig.PEER_PORT); h.set_nodeprivate (theConfig.PEER_PRIVATE); h.set_testnet (theConfig.TESTNET); - Ledger::pointer closedLedger = theApp->getLedgerMaster ().getClosedLedger (); + Ledger::pointer closedLedger = getApp().getLedgerMaster ().getClosedLedger (); if (closedLedger && closedLedger->isClosed ()) { @@ -2223,21 +2272,21 @@ void PeerImp::sendGetPeers () void PeerImp::applyLoadCharge (LoadType loadType) { - // IMPLEMENATION IS INCOMPLETE + // IMPLEMENTATION IS INCOMPLETE - // VFALCO TODO This needs to implemented before open sourcing. + // VFALCO TODO This needs to completed before open sourcing. - if (theApp->getLoadManager ().applyLoadCharge (mLoad, loadType)) + if (getApp().getLoadManager ().applyLoadCharge (mLoad, loadType)) { if (mCluster) { WriteLog (lsWARNING, Peer) << "aLC: " << getDisplayName() << " load from cluster"; } - else if (theApp->getLoadManager ().shouldCutoff(mLoad)) + else if (getApp().getLoadManager ().shouldCutoff(mLoad)) { WriteLog (lsWARNING, Peer) << "aLC: " << getDisplayName() << " should cutoff"; } - else if (theApp->getLoadManager ().shouldWarn (mLoad)) + else if (getApp().getLoadManager ().shouldWarn (mLoad)) { WriteLog (lsWARNING, Peer) << "aLC: " << getDisplayName() << " load warning"; } @@ -2279,7 +2328,8 @@ void PeerImp::doProofOfWork (Job&, boost::weak_ptr peer, ProofOfWork::poi void PeerImp::doFetchPack (const boost::shared_ptr& packet) { - if (theApp->getFeeTrack ().isLoaded ()) + // VFALCO TODO Invert this dependency using an observer and shared state object. + if (getApp().getFeeTrack ().isLoadedLocal ()) { WriteLog (lsINFO, Peer) << "Too busy to make fetch pack"; return; @@ -2295,7 +2345,7 @@ void PeerImp::doFetchPack (const boost::shared_ptr& uint256 hash; memcpy (hash.begin (), packet->ledgerhash ().data (), 32); - Ledger::pointer haveLedger = theApp->getOPs ().getLedgerByHash (hash); + Ledger::pointer haveLedger = getApp().getOPs ().getLedgerByHash (hash); if (!haveLedger) { @@ -2311,7 +2361,7 @@ void PeerImp::doFetchPack (const boost::shared_ptr& return; } - Ledger::pointer wantLedger = theApp->getOPs ().getLedgerByHash (haveLedger->getParentHash ()); + Ledger::pointer wantLedger = getApp().getOPs ().getLedgerByHash (haveLedger->getParentHash ()); if (!wantLedger) { @@ -2320,8 +2370,8 @@ void PeerImp::doFetchPack (const boost::shared_ptr& return; } - theApp->getJobQueue ().addJob (jtPACK, "MakeFetchPack", - BIND_TYPE (&NetworkOPs::makeFetchPack, &theApp->getOPs (), P_1, + getApp().getJobQueue ().addLimitJob (jtPACK, "MakeFetchPack", 1, + BIND_TYPE (&NetworkOPs::makeFetchPack, &getApp().getOPs (), P_1, boost::weak_ptr (shared_from_this ()), packet, wantLedger, haveLedger, UptimeTimer::getInstance ().getElapsedSeconds ())); } @@ -2334,7 +2384,7 @@ Json::Value PeerImp::getJson () { Json::Value ret (Json::objectValue); - //ret["this"] = ADDRESS(this); + //ret["this"] = addressToString(this); ret["public_key"] = mNodePublic.ToString (); ret["ip"] = mIpPortConnect.first; //ret["port"] = mIpPortConnect.second; @@ -2411,12 +2461,13 @@ Peer::pointer Peer::New (boost::asio::io_service& io_service, return Peer::pointer (new PeerImp (io_service, ctx, id, inbound)); } -void Peer::applyLoadCharge (const boost::weak_ptr& wp, LoadType l) +void Peer::applyLoadCharge (boost::weak_ptr & peerToPunish, + LoadType loadThatWasImposed) { - Peer::pointer p = wp.lock (); + Peer::pointer p = peerToPunish.lock (); - if (p) - p->applyLoadCharge (l); + if (p != nullptr) + { + p->applyLoadCharge (loadThatWasImposed); + } } - -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_Peer.h b/src/cpp/ripple/ripple_Peer.h index 4c3dcd3cbd..e24003d6a3 100644 --- a/src/cpp/ripple/ripple_Peer.h +++ b/src/cpp/ripple/ripple_Peer.h @@ -4,25 +4,21 @@ */ //============================================================================== -#ifndef RIPPLE_PEER_H -#define RIPPLE_PEER_H +#ifndef RIPPLE_PEER_H_INCLUDED +#define RIPPLE_PEER_H_INCLUDED // VFALCO TODO Couldn't this be a struct? -typedef std::pair ipPort; +typedef std::pair IPAndPortNumber; -class Peer : public boost::enable_shared_from_this +/** Represents a peer connection in the overlay. +*/ +class Peer + : public boost::enable_shared_from_this + , LeakChecked { public: - typedef boost::shared_ptr pointer; - typedef const boost::shared_ptr& ref; - - static int const psbGotHello = 0; - static int const psbSentHello = 1; - static int const psbInMap = 2; - static int const psbTrusted = 3; - static int const psbNoLedgers = 4; - static int const psbNoTransactions = 5; - static int const psbDownLevel = 6; + typedef boost::shared_ptr pointer; + typedef pointer const& ref; public: static pointer New (boost::asio::io_service& io_service, @@ -34,7 +30,7 @@ public: virtual void handleConnect (const boost::system::error_code& error, boost::asio::ip::tcp::resolver::iterator it) = 0; - virtual std::string& getIP () = 0; + virtual std::string const& getIP () = 0; virtual std::string getDisplayName () = 0; @@ -50,9 +46,6 @@ public: virtual void detach (const char*, bool onIOStrand) = 0; - //virtual bool samePeer (Peer::ref p) = 0; - //virtual bool samePeer (const Peer& p) = 0; - virtual void sendPacket (const PackedMessage::pointer& packet, bool onStrand) = 0; virtual void sendGetPeers () = 0; @@ -60,12 +53,19 @@ public: virtual void applyLoadCharge (LoadType) = 0; // VFALCO NOTE what's with this odd parameter passing? Why the static member? - static void applyLoadCharge (const boost::weak_ptr&, LoadType); + // + /** Adjust this peer's load balance based on the type of load imposed. + + @note Formerly named punishPeer + */ + static void applyLoadCharge (boost::weak_ptr & peerTOCharge, LoadType loadThatWasImposed); virtual Json::Value getJson () = 0; virtual bool isConnected () const = 0; + virtual bool isInCluster () const = 0; + virtual bool isInbound () const = 0; virtual bool isOutbound () const = 0; @@ -88,4 +88,3 @@ public: }; #endif -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_PeerSet.cpp b/src/cpp/ripple/ripple_PeerSet.cpp index b849ed5879..51afcec5bb 100644 --- a/src/cpp/ripple/ripple_PeerSet.cpp +++ b/src/cpp/ripple/ripple_PeerSet.cpp @@ -14,7 +14,7 @@ PeerSet::PeerSet (uint256 const& hash, int interval) , mFailed (false) , mProgress (true) , mAggressive (false) - , mTimer (theApp->getIOService ()) + , mTimer (getApp().getIOService ()) { mLastAction = UptimeTimer::getInstance ().getElapsedSeconds (); assert ((mTimerInterval > 10) && (mTimerInterval < 30000)); @@ -53,12 +53,12 @@ void PeerSet::invokeOnTimer () { ++mTimeouts; WriteLog (lsWARNING, InboundLedger) << "Timeout(" << mTimeouts << ") pc=" << mPeers.size () << " acquiring " << mHash; - onTimer (false); + onTimer (false, sl); } else { mProgress = false; - onTimer (true); + onTimer (true, sl); } if (!isDone ()) @@ -74,7 +74,7 @@ void PeerSet::TimerEntry (boost::weak_ptr wptr, const boost::system::er if (ptr) { - int jc = theApp->getJobQueue ().getJobCountTotal (jtLEDGER_DATA); + int jc = getApp().getJobQueue ().getJobCountTotal (jtLEDGER_DATA); if (jc > 4) { @@ -82,7 +82,8 @@ void PeerSet::TimerEntry (boost::weak_ptr wptr, const boost::system::er ptr->setTimer (); } else - theApp->getJobQueue ().addJob (jtLEDGER_DATA, "timerEntry", BIND_TYPE (&PeerSet::TimerJobEntry, P_1, ptr)); + getApp().getJobQueue ().addLimitJob (jtLEDGER_DATA, "timerEntry", 2, + BIND_TYPE (&PeerSet::TimerJobEntry, P_1, ptr)); } } diff --git a/src/cpp/ripple/ripple_PeerSet.h b/src/cpp/ripple/ripple_PeerSet.h index 7e319610d8..095e0f1ecd 100644 --- a/src/cpp/ripple/ripple_PeerSet.h +++ b/src/cpp/ripple/ripple_PeerSet.h @@ -11,7 +11,7 @@ A peer set is used to acquire a ledger or a transaction set. */ -class PeerSet +class PeerSet : LeakChecked { public: uint256 const& getHash () const @@ -71,7 +71,7 @@ protected: virtual ~PeerSet () { } virtual void newPeer (Peer::ref) = 0; - virtual void onTimer (bool progress) = 0; + virtual void onTimer (bool progress, boost::recursive_mutex::scoped_lock&) = 0; virtual boost::weak_ptr pmDowncast () = 0; void setComplete () diff --git a/src/cpp/ripple/ripple_Peers.cpp b/src/cpp/ripple/ripple_Peers.cpp index b96596ad01..2a6ae72a19 100644 --- a/src/cpp/ripple/ripple_Peers.cpp +++ b/src/cpp/ripple/ripple_Peers.cpp @@ -4,19 +4,18 @@ */ //============================================================================== -// VFALCO TODO make this an inline function -#define ADDRESS_SHARED(p) strHex(uint64( ((char*) (p).get()) - ((char*) 0))) - -// How often to enforce policies. -#define POLICY_INTERVAL_SECONDS 5 - -class Peers; - -SETUP_LOG (Peers) - -class Peers : public IPeers +class Peers + : public IPeers + , LeakChecked { public: + enum + { + /** Frequency of policy enforcement. + */ + policyIntervalSeconds = 5 + }; + explicit Peers (boost::asio::io_service& io_service) : mLastPeer (0) , mPhase (0) @@ -30,6 +29,7 @@ public: // Send message to network. int relayMessage (Peer* fromPeer, const PackedMessage::pointer& msg); + int relayMessageCluster (Peer* fromPeer, const PackedMessage::pointer& msg); void relayMessageTo (const std::set& fromPeers, const PackedMessage::pointer& msg); void relayMessageBut (const std::set& fromPeers, const PackedMessage::pointer& msg); @@ -86,15 +86,15 @@ private: int mPhase; typedef std::pair naPeer; - typedef std::pair pipPeer; - typedef std::map::value_type vtPeer; + typedef std::pair pipPeer; + typedef std::map::value_type vtPeer; // Peers we are connecting with and non-thin peers we are connected to. // Only peers we know the connection ip for are listed. // We know the ip and port for: // - All outbound connections // - Some inbound connections (which we figured out). - boost::unordered_map mIpMap; + boost::unordered_map mIpMap; // Non-thin peers which we are connected to. // Peers we have the public key for. @@ -148,8 +148,8 @@ void Peers::start () bool Peers::getTopNAddrs (int n, std::vector& addrs) { // XXX Filter out other local addresses (like ipv6) - Database* db = theApp->getWalletDB ()->getDB (); - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); SQL_FOREACH (db, str (boost::format ("SELECT IpPort FROM PeerIps LIMIT %d") % n) ) { @@ -167,18 +167,18 @@ bool Peers::savePeer (const std::string& strIp, int iPort, char code) { bool bNew = false; - Database* db = theApp->getWalletDB ()->getDB (); + Database* db = getApp().getWalletDB ()->getDB (); - std::string ipPort = sqlEscape (str (boost::format ("%s %d") % strIp % iPort)); + std::string ipAndPort = sqlEscape (str (boost::format ("%s %d") % strIp % iPort)); - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); - std::string sql = str (boost::format ("SELECT COUNT(*) FROM PeerIps WHERE IpPort=%s;") % ipPort); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); + std::string sql = str (boost::format ("SELECT COUNT(*) FROM PeerIps WHERE IpPort=%s;") % ipAndPort); if (db->executeSQL (sql) && db->startIterRows ()) { if (!db->getInt (0)) { - db->executeSQL (str (boost::format ("INSERT INTO PeerIps (IpPort,Score,Source) values (%s,0,'%c');") % ipPort % code)); + db->executeSQL (str (boost::format ("INSERT INTO PeerIps (IpPort,Score,Source) values (%s,0,'%c');") % ipAndPort % code)); bNew = true; } else @@ -194,7 +194,7 @@ bool Peers::savePeer (const std::string& strIp, int iPort, char code) } else { - std::cerr << "Error saving Peer" << std::endl; + Log::out() << "Error saving Peer"; } if (bNew) @@ -226,7 +226,7 @@ bool Peers::hasPeer (const uint64& id) // <-- true, if a peer is available to connect to bool Peers::peerAvailable (std::string& strIp, int& iPort) { - Database* db = theApp->getWalletDB ()->getDB (); + Database* db = getApp().getWalletDB ()->getDB (); std::vector vstrIpPort; // Convert mIpMap (list of open connections) to a vector of " ". @@ -248,7 +248,7 @@ bool Peers::peerAvailable (std::string& strIp, int& iPort) std::string strIpPort; { - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); if (db->executeSQL (str (boost::format ("SELECT IpPort FROM PeerIps WHERE ScanNext IS NULL AND IpPort NOT IN (%s) LIMIT 1;") % strJoin (vstrIpPort.begin (), vstrIpPort.end (), ","))) @@ -328,8 +328,8 @@ void Peers::policyEnforce () } // Schedule next enforcement. - mPolicyTimer.expires_at (boost::posix_time::second_clock::universal_time () + boost::posix_time::seconds (POLICY_INTERVAL_SECONDS)); - mPolicyTimer.async_wait (boost::bind (&Peers::policyHandler, this, _1)); + mPolicyTimer.expires_at (boost::posix_time::second_clock::universal_time () + boost::posix_time::seconds (policyIntervalSeconds)); + mPolicyTimer.async_wait (BIND_TYPE (&Peers::policyHandler, this, P_1)); } void Peers::policyHandler (const boost::system::error_code& ecResult) @@ -366,6 +366,22 @@ int Peers::relayMessage (Peer* fromPeer, const PackedMessage::pointer& msg) return sentTo; } +int Peers::relayMessageCluster (Peer* fromPeer, const PackedMessage::pointer& msg) +{ + int sentTo = 0; + std::vector peerVector = getPeerVector (); + BOOST_FOREACH (Peer::ref peer, peerVector) + { + if ((!fromPeer || ! (peer.get () == fromPeer)) && peer->isConnected () && peer->isInCluster ()) + { + ++sentTo; + peer->sendPacket (msg, false); + } + } + + return sentTo; +} + void Peers::relayMessageBut (const std::set& fromPeers, const PackedMessage::pointer& msg) { // Relay message to all but the specified peers @@ -397,13 +413,13 @@ void Peers::relayMessageTo (const std::set& fromPeers, const PackedMessa void Peers::connectTo (const std::string& strIp, int iPort) { { - Database* db = theApp->getWalletDB ()->getDB (); - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); db->executeSQL (str (boost::format ("REPLACE INTO PeerIps (IpPort,Score,Source,ScanNext) values (%s,%d,'%c',0);") % sqlEscape (str (boost::format ("%s %d") % strIp % iPort)) - % theApp->getUNL ().iSourceScore (IUniqueNodeList::vsManual) - % char (IUniqueNodeList::vsManual))); + % getApp().getUNL ().iSourceScore (UniqueNodeList::vsManual) + % char (UniqueNodeList::vsManual))); } scanRefresh (); @@ -414,7 +430,7 @@ void Peers::connectTo (const std::string& strIp, int iPort) // <-- true, if already connected. Peer::pointer Peers::peerConnect (const std::string& strIp, int iPort) { - ipPort pipPeer = make_pair (strIp, iPort); + IPAndPortNumber pipPeer = make_pair (strIp, iPort); Peer::pointer ppResult; @@ -423,8 +439,8 @@ Peer::pointer Peers::peerConnect (const std::string& strIp, int iPort) if (mIpMap.find (pipPeer) == mIpMap.end ()) { - ppResult = Peer::New (theApp->getIOService (), - theApp->getPeerDoor ().getSSLContext (), + ppResult = Peer::New (getApp().getIOService (), + getApp().getPeerDoor ().getSSLContext (), ++mLastPeer, false); @@ -499,9 +515,9 @@ bool Peers::peerConnected (Peer::ref peer, const RippleAddress& naPeer, assert (!!peer); - if (naPeer == theApp->getLocalCredentials ().getNodePublic ()) + if (naPeer == getApp().getLocalCredentials ().getNodePublic ()) { - WriteLog (lsINFO, Peers) << "Pool: Connected: self: " << ADDRESS_SHARED (peer) << ": " << naPeer.humanNodePublic () << " " << strIP << " " << iPort; + WriteLog (lsINFO, Peers) << "Pool: Connected: self: " << addressToString (peer.get()) << ": " << naPeer.humanNodePublic () << " " << strIP << " " << iPort; } else { @@ -511,7 +527,7 @@ bool Peers::peerConnected (Peer::ref peer, const RippleAddress& naPeer, if (itCm == mConnectedMap.end ()) { // New connection. - //WriteLog (lsINFO, Peers) << "Pool: Connected: new: " << ADDRESS_SHARED(peer) << ": " << naPeer.humanNodePublic() << " " << strIP << " " << iPort; + //WriteLog (lsINFO, Peers) << "Pool: Connected: new: " << addressToString (peer.get()) << ": " << naPeer.humanNodePublic() << " " << strIP << " " << iPort; mConnectedMap[naPeer] = peer; bNew = true; @@ -527,7 +543,7 @@ bool Peers::peerConnected (Peer::ref peer, const RippleAddress& naPeer, if (itCm->second->getIP ().empty ()) { // Old peer did not know it's IP. - //WriteLog (lsINFO, Peers) << "Pool: Connected: redundant: outbound: " << ADDRESS_SHARED(peer) << " discovered: " << ADDRESS_SHARED(itCm->second) << ": " << strIP << " " << iPort; + //WriteLog (lsINFO, Peers) << "Pool: Connected: redundant: outbound: " << addressToString (peer.get()) << " discovered: " << addressToString(itCm->second) << ": " << strIP << " " << iPort; itCm->second->setIpPort (strIP, iPort); @@ -537,14 +553,14 @@ bool Peers::peerConnected (Peer::ref peer, const RippleAddress& naPeer, else { // Old peer knew its IP. Do nothing. - //WriteLog (lsINFO, Peers) << "Pool: Connected: redundant: outbound: rediscovered: " << ADDRESS_SHARED(peer) << " " << strIP << " " << iPort; + //WriteLog (lsINFO, Peers) << "Pool: Connected: redundant: outbound: rediscovered: " << addressToString (peer.get()) << " " << strIP << " " << iPort; nothing (); } } else { - //WriteLog (lsINFO, Peers) << "Pool: Connected: redundant: inbound: " << ADDRESS_SHARED(peer) << " " << strIP << " " << iPort; + //WriteLog (lsINFO, Peers) << "Pool: Connected: redundant: inbound: " << addressToString (peer.get()) << " " << strIP << " " << iPort; nothing (); } @@ -599,8 +615,8 @@ bool Peers::peerScanSet (const std::string& strIp, int iPort) std::string strIpPort = str (boost::format ("%s %d") % strIp % iPort); bool bScanDirty = false; - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); - Database* db = theApp->getWalletDB ()->getDB (); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); if (db->executeSQL (str (boost::format ("SELECT ScanNext FROM PeerIps WHERE IpPort=%s;") % sqlEscape (strIpPort))) @@ -646,7 +662,7 @@ bool Peers::peerScanSet (const std::string& strIp, int iPort) // --> strIp: not empty void Peers::peerClosed (Peer::ref peer, const std::string& strIp, int iPort) { - ipPort ipPeer = make_pair (strIp, iPort); + IPAndPortNumber ipPeer = make_pair (strIp, iPort); bool bScanRefresh = false; // If the connection was our scan, we are no longer scanning. @@ -662,18 +678,18 @@ void Peers::peerClosed (Peer::ref peer, const std::string& strIp, int iPort) bool bRedundant = true; { boost::recursive_mutex::scoped_lock sl (mPeerLock); - const boost::unordered_map::iterator& itIp = mIpMap.find (ipPeer); + const boost::unordered_map::iterator& itIp = mIpMap.find (ipPeer); if (itIp == mIpMap.end ()) { // Did not find it. Not already connecting or connected. - WriteLog (lsWARNING, Peers) << "Pool: Closed: UNEXPECTED: " << ADDRESS_SHARED (peer) << ": " << strIp << " " << iPort; + WriteLog (lsWARNING, Peers) << "Pool: Closed: UNEXPECTED: " << addressToString (peer.get()) << ": " << strIp << " " << iPort; // XXX Internal error. } else if (mIpMap[ipPeer] == peer) { // We were the identified connection. - //WriteLog (lsINFO, Peers) << "Pool: Closed: identified: " << ADDRESS_SHARED(peer) << ": " << strIp << " " << iPort; + //WriteLog (lsINFO, Peers) << "Pool: Closed: identified: " << addressToString (peer.get()) << ": " << strIp << " " << iPort; // Delete our entry. mIpMap.erase (itIp); @@ -683,7 +699,7 @@ void Peers::peerClosed (Peer::ref peer, const std::string& strIp, int iPort) else { // Found it. But, we were redundant. - //WriteLog (lsINFO, Peers) << "Pool: Closed: redundant: " << ADDRESS_SHARED(peer) << ": " << strIp << " " << iPort; + //WriteLog (lsINFO, Peers) << "Pool: Closed: redundant: " << addressToString (peer.get()) << ": " << strIp << " " << iPort; } } @@ -707,9 +723,9 @@ void Peers::peerVerified (Peer::ref peer) std::string strIpPort = str (boost::format ("%s %d") % strIp % iPort); - //WriteLog (lsINFO, Peers) << str(boost::format("Pool: Scan: connected: %s %s %s (scanned)") % ADDRESS_SHARED(peer) % strIp % iPort); + //WriteLog (lsINFO, Peers) << str(boost::format("Pool: Scan: connected: %s %s %s (scanned)") % addressToString (peer.get()) % strIp % iPort); - if (peer->getNodePublic () == theApp->getLocalCredentials ().getNodePublic ()) + if (peer->getNodePublic () == getApp().getLocalCredentials ().getNodePublic ()) { // Talking to ourself. We will just back off. This lets us maybe advertise our outside address. @@ -718,8 +734,8 @@ void Peers::peerVerified (Peer::ref peer) else { // Talking with a different peer. - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); - Database* db = theApp->getWalletDB ()->getDB (); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); db->executeSQL (boost::str (boost::format ("UPDATE PeerIps SET ScanNext=NULL,ScanInterval=0 WHERE IpPort=%s;") % sqlEscape (strIpPort))); @@ -786,8 +802,8 @@ void Peers::scanRefresh () int iInterval; { - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); - Database* db = theApp->getWalletDB ()->getDB (); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); if (db->executeSQL ("SELECT * FROM PeerIps INDEXED BY PeerScanIndex WHERE ScanNext NOT NULL ORDER BY ScanNext LIMIT 1;") && db->startIterRows ()) @@ -832,8 +848,8 @@ void Peers::scanRefresh () iInterval *= 2; { - ScopedLock sl (theApp->getWalletDB ()->getDBLock ()); - Database* db = theApp->getWalletDB ()->getDB (); + ScopedLock sl (getApp().getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); db->executeSQL (boost::str (boost::format ("UPDATE PeerIps SET ScanNext=%d,ScanInterval=%d WHERE IpPort=%s;") % iToSeconds (tpNext) @@ -856,7 +872,7 @@ void Peers::scanRefresh () // % strIpPort % tpNext % (tpNext-tpNow).total_seconds()); mScanTimer.expires_at (tpNext); - mScanTimer.async_wait (boost::bind (&Peers::scanHandler, this, _1)); + mScanTimer.async_wait (BIND_TYPE (&Peers::scanHandler, this, P_1)); } } } @@ -878,4 +894,5 @@ bool Peers::isMessageKnown (PackedMessage::pointer msg) } #endif -// vim:ts=4 +SETUP_LOG (Peers) + diff --git a/src/cpp/ripple/ripple_ProofOfWork.h b/src/cpp/ripple/ripple_ProofOfWork.h index 6fc891a364..5bff7d4fb0 100644 --- a/src/cpp/ripple/ripple_ProofOfWork.h +++ b/src/cpp/ripple/ripple_ProofOfWork.h @@ -7,7 +7,7 @@ #ifndef RIPPLE_PROOFOFWORK_H #define RIPPLE_PROOFOFWORK_H -class ProofOfWork +class ProofOfWork : LeakChecked { public: static const int sMaxDifficulty; diff --git a/src/cpp/ripple/ripple_ProofOfWorkFactory.cpp b/src/cpp/ripple/ripple_ProofOfWorkFactory.cpp index 5c0a72dfa1..0cd7c776dd 100644 --- a/src/cpp/ripple/ripple_ProofOfWorkFactory.cpp +++ b/src/cpp/ripple/ripple_ProofOfWorkFactory.cpp @@ -4,46 +4,6 @@ */ //============================================================================== -class ProofOfWorkFactory : public IProofOfWorkFactory -{ -public: - ProofOfWorkFactory (); - - ProofOfWork getProof (); - POWResult checkProof (const std::string& token, uint256 const& solution); - uint64 getDifficulty () - { - return ProofOfWork::getDifficulty (mTarget, mIterations); - } - void setDifficulty (int i); - - void loadHigh (); - void loadLow (); - void sweep (void); - - uint256 const& getSecret () const - { - return mSecret; - } - void setSecret (uint256 const& secret) - { - mSecret = secret; - } - - static int getPowEntry (uint256 const& target, int iterations); - -private: - uint256 mSecret; - int mIterations; - uint256 mTarget; - time_t mLastDifficultyChange; - int mValidTime; - int mPowEntry; - - powMap_t mSolvedChallenges; - boost::mutex mLock; -}; - ProofOfWorkFactory::ProofOfWorkFactory () : mValidTime (180) { setDifficulty (1); @@ -271,64 +231,3 @@ IProofOfWorkFactory* IProofOfWorkFactory::New () return new ProofOfWorkFactory; } -BOOST_AUTO_TEST_SUITE (ProofOfWork_suite) - -BOOST_AUTO_TEST_CASE ( ProofOfWork_test ) -{ - ProofOfWorkFactory gen; - ProofOfWork pow = gen.getProof (); - WriteLog (lsINFO, ProofOfWork) << "Estimated difficulty: " << pow.getDifficulty (); - uint256 solution = pow.solve (16777216); - - if (solution.isZero ()) - BOOST_FAIL ("Unable to solve proof of work"); - - if (!pow.checkSolution (solution)) - BOOST_FAIL ("Solution did not check"); - - WriteLog (lsDEBUG, ProofOfWork) << "A bad nonce error is expected"; - POWResult r = gen.checkProof (pow.getToken (), uint256 ()); - - if (r != powBADNONCE) - { - Log (lsFATAL) << "POWResult = " << static_cast (r); - BOOST_FAIL ("Empty solution didn't show bad nonce"); - } - - if (gen.checkProof (pow.getToken (), solution) != powOK) - BOOST_FAIL ("Solution did not check with issuer"); - - WriteLog (lsDEBUG, ProofOfWork) << "A reused nonce error is expected"; - - if (gen.checkProof (pow.getToken (), solution) != powREUSED) - BOOST_FAIL ("Reuse solution not detected"); - -#ifdef SOLVE_POWS - - for (int i = 0; i < 12; ++i) - { - gen.setDifficulty (i); - ProofOfWork pow = gen.getProof (); - WriteLog (lsINFO, ProofOfWork) << "Level: " << i << ", Estimated difficulty: " << pow.getDifficulty (); - uint256 solution = pow.solve (131072); - - if (solution.isZero ()) - WriteLog (lsINFO, ProofOfWork) << "Giving up"; - else - { - WriteLog (lsINFO, ProofOfWork) << "Solution found"; - - if (gen.checkProof (pow.getToken (), solution) != powOK) - { - WriteLog (lsFATAL, ProofOfWork) << "Solution fails"; - } - } - } - -#endif - -} - -BOOST_AUTO_TEST_SUITE_END () - -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_ProofOfWorkFactory.h b/src/cpp/ripple/ripple_ProofOfWorkFactory.h new file mode 100644 index 0000000000..23837718be --- /dev/null +++ b/src/cpp/ripple/ripple_ProofOfWorkFactory.h @@ -0,0 +1,54 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_PROOFOFWORKFACTORY_RIPPLEHEADER +#define RIPPLE_PROOFOFWORKFACTORY_RIPPLEHEADER + +// PRIVATE HEADER + +class ProofOfWorkFactory + : public IProofOfWorkFactory + , LeakChecked +{ +public: + ProofOfWorkFactory (); + + ProofOfWork getProof (); + POWResult checkProof (const std::string& token, uint256 const& solution); + uint64 getDifficulty () + { + return ProofOfWork::getDifficulty (mTarget, mIterations); + } + void setDifficulty (int i); + + void loadHigh (); + void loadLow (); + void sweep (void); + + uint256 const& getSecret () const + { + return mSecret; + } + void setSecret (uint256 const& secret) + { + mSecret = secret; + } + + static int getPowEntry (uint256 const& target, int iterations); + +private: + uint256 mSecret; + int mIterations; + uint256 mTarget; + time_t mLastDifficultyChange; + int mValidTime; + int mPowEntry; + + powMap_t mSolvedChallenges; + boost::mutex mLock; +}; + +#endif diff --git a/src/cpp/ripple/ripple_ProofOfWorkFactoryUnitTests.cpp b/src/cpp/ripple/ripple_ProofOfWorkFactoryUnitTests.cpp new file mode 100644 index 0000000000..e96eccd485 --- /dev/null +++ b/src/cpp/ripple/ripple_ProofOfWorkFactoryUnitTests.cpp @@ -0,0 +1,67 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +BOOST_AUTO_TEST_SUITE (ProofOfWork_suite) + +BOOST_AUTO_TEST_CASE ( ProofOfWork_test ) +{ + using namespace ripple; + + ProofOfWorkFactory gen; + ProofOfWork pow = gen.getProof (); + WriteLog (lsINFO, ProofOfWork) << "Estimated difficulty: " << pow.getDifficulty (); + uint256 solution = pow.solve (16777216); + + if (solution.isZero ()) + BOOST_FAIL ("Unable to solve proof of work"); + + if (!pow.checkSolution (solution)) + BOOST_FAIL ("Solution did not check"); + + WriteLog (lsDEBUG, ProofOfWork) << "A bad nonce error is expected"; + POWResult r = gen.checkProof (pow.getToken (), uint256 ()); + + if (r != powBADNONCE) + { + Log (lsFATAL) << "POWResult = " << static_cast (r); + BOOST_FAIL ("Empty solution didn't show bad nonce"); + } + + if (gen.checkProof (pow.getToken (), solution) != powOK) + BOOST_FAIL ("Solution did not check with issuer"); + + WriteLog (lsDEBUG, ProofOfWork) << "A reused nonce error is expected"; + + if (gen.checkProof (pow.getToken (), solution) != powREUSED) + BOOST_FAIL ("Reuse solution not detected"); + +#ifdef SOLVE_POWS + + for (int i = 0; i < 12; ++i) + { + gen.setDifficulty (i); + ProofOfWork pow = gen.getProof (); + WriteLog (lsINFO, ProofOfWork) << "Level: " << i << ", Estimated difficulty: " << pow.getDifficulty (); + uint256 solution = pow.solve (131072); + + if (solution.isZero ()) + WriteLog (lsINFO, ProofOfWork) << "Giving up"; + else + { + WriteLog (lsINFO, ProofOfWork) << "Solution found"; + + if (gen.checkProof (pow.getToken (), solution) != powOK) + { + WriteLog (lsFATAL, ProofOfWork) << "Solution fails"; + } + } + } + +#endif + +} + +BOOST_AUTO_TEST_SUITE_END () diff --git a/src/cpp/ripple/ripple_SHAMap.cpp b/src/cpp/ripple/ripple_SHAMap.cpp index 16801ce901..d60bc51cd8 100644 --- a/src/cpp/ripple/ripple_SHAMap.cpp +++ b/src/cpp/ripple/ripple_SHAMap.cpp @@ -208,7 +208,7 @@ SHAMapTreeNode::pointer SHAMap::getNode (const SHAMapNode& id, uint256 const& ha if (node) { -#ifdef DEBUG +#ifdef BEAST_DEBUG if (node->getNodeHash () != hash) { @@ -821,17 +821,17 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternalNT (const SHAMapNode& id, uint2 { SHAMapTreeNode::pointer ret; - if (!theApp->running ()) + if (!getApp().running ()) return ret; - HashedObject::pointer obj (theApp->getHashedObjectStore ().retrieve (hash)); + NodeObject::pointer obj (getApp().getNodeStore ().retrieve (hash)); if (!obj) { // WriteLog (lsTRACE, SHAMap) << "fetchNodeExternal: missing " << hash; if (mLedgerSeq != 0) { - theApp->getOPs ().missingNodeInLedger (mLedgerSeq); + getApp().getOPs ().missingNodeInLedger (mLedgerSeq); mLedgerSeq = 0; } @@ -913,7 +913,7 @@ int SHAMap::armDirty () return ++mSeq; } -int SHAMap::flushDirty (DirtyMap& map, int maxNodes, HashedObjectType t, uint32 seq) +int SHAMap::flushDirty (DirtyMap& map, int maxNodes, NodeObjectType t, uint32 seq) { int flushed = 0; Serializer s; @@ -925,7 +925,7 @@ int SHAMap::flushDirty (DirtyMap& map, int maxNodes, HashedObjectType t, uint32 s.erase (); it->second->addRaw (s, snfPREFIX); -#ifdef DEBUG +#ifdef BEAST_DEBUG if (s.getSHA512Half () != it->second->getNodeHash ()) { @@ -937,7 +937,7 @@ int SHAMap::flushDirty (DirtyMap& map, int maxNodes, HashedObjectType t, uint32 #endif - theApp->getHashedObjectStore ().store (t, seq, s.peekData (), it->second->getNodeHash ()); + getApp().getNodeStore ().store (t, seq, s.peekData (), it->second->getNodeHash ()); if (flushed++ >= maxNodes) return flushed; @@ -1046,86 +1046,3 @@ void SHAMap::dump (bool hash) } } - -static Blob IntToVUC (int v) -{ - Blob vuc; - - for (int i = 0; i < 32; ++i) - vuc.push_back (static_cast (v)); - - return vuc; -} - -BOOST_AUTO_TEST_SUITE (SHAMap_suite) - -BOOST_AUTO_TEST_CASE ( SHAMap_test ) -{ - // h3 and h4 differ only in the leaf, same terminal node (level 19) - WriteLog (lsTRACE, SHAMap) << "SHAMap test"; - uint256 h1, h2, h3, h4, h5; - h1.SetHex ("092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); - h2.SetHex ("436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe"); - h3.SetHex ("b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8"); - h4.SetHex ("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8"); - h5.SetHex ("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); - - SHAMap sMap (smtFREE); - SHAMapItem i1 (h1, IntToVUC (1)), i2 (h2, IntToVUC (2)), i3 (h3, IntToVUC (3)), i4 (h4, IntToVUC (4)), i5 (h5, IntToVUC (5)); - - if (!sMap.addItem (i2, true, false)) BOOST_FAIL ("no add"); - - if (!sMap.addItem (i1, true, false)) BOOST_FAIL ("no add"); - - SHAMapItem::pointer i; - - i = sMap.peekFirstItem (); - - if (!i || (*i != i1)) BOOST_FAIL ("bad traverse"); - - i = sMap.peekNextItem (i->getTag ()); - - if (!i || (*i != i2)) BOOST_FAIL ("bad traverse"); - - i = sMap.peekNextItem (i->getTag ()); - - if (i) BOOST_FAIL ("bad traverse"); - - sMap.addItem (i4, true, false); - sMap.delItem (i2.getTag ()); - sMap.addItem (i3, true, false); - - i = sMap.peekFirstItem (); - - if (!i || (*i != i1)) BOOST_FAIL ("bad traverse"); - - i = sMap.peekNextItem (i->getTag ()); - - if (!i || (*i != i3)) BOOST_FAIL ("bad traverse"); - - i = sMap.peekNextItem (i->getTag ()); - - if (!i || (*i != i4)) BOOST_FAIL ("bad traverse"); - - i = sMap.peekNextItem (i->getTag ()); - - if (i) BOOST_FAIL ("bad traverse"); - - WriteLog (lsTRACE, SHAMap) << "SHAMap snap test"; - uint256 mapHash = sMap.getHash (); - SHAMap::pointer map2 = sMap.snapShot (false); - - if (sMap.getHash () != mapHash) BOOST_FAIL ("bad snapshot"); - - if (map2->getHash () != mapHash) BOOST_FAIL ("bad snapshot"); - - if (!sMap.delItem (sMap.peekFirstItem ()->getTag ())) BOOST_FAIL ("bad mod"); - - if (sMap.getHash () == mapHash) BOOST_FAIL ("bad snapshot"); - - if (map2->getHash () != mapHash) BOOST_FAIL ("bad snapshot"); -} - -BOOST_AUTO_TEST_SUITE_END (); - -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_SHAMap.h b/src/cpp/ripple/ripple_SHAMap.h index 0ad68daa7f..d6d196c4be 100644 --- a/src/cpp/ripple/ripple_SHAMap.h +++ b/src/cpp/ripple/ripple_SHAMap.h @@ -20,6 +20,8 @@ class SHAMap : public CountedObject { public: + static char const* getCountedObjectName () { return "SHAMap"; } + typedef boost::shared_ptr pointer; typedef const boost::shared_ptr& ref; @@ -139,7 +141,7 @@ public: bool compare (SHAMap::ref otherMap, Delta & differences, int maxCount); int armDirty (); - static int flushDirty (DirtyMap & dirtyMap, int maxNodes, HashedObjectType t, uint32 seq); + static int flushDirty (DirtyMap & dirtyMap, int maxNodes, NodeObjectType t, uint32 seq); boost::shared_ptr disarmDirty (); void setSeq (uint32 seq) diff --git a/src/cpp/ripple/ripple_SHAMapItem.h b/src/cpp/ripple/ripple_SHAMapItem.h index 3cfdbd5e3e..a7c8e29f70 100644 --- a/src/cpp/ripple/ripple_SHAMapItem.h +++ b/src/cpp/ripple/ripple_SHAMapItem.h @@ -12,6 +12,8 @@ class SHAMapItem : public CountedObject { public: + static char const* getCountedObjectName () { return "SHAMapItem"; } + typedef boost::shared_ptr pointer; typedef const boost::shared_ptr& ref; diff --git a/src/cpp/ripple/ripple_SHAMapNode.cpp b/src/cpp/ripple/ripple_SHAMapNode.cpp index 6b4049f8d5..dcd8c0b3dc 100644 --- a/src/cpp/ripple/ripple_SHAMapNode.cpp +++ b/src/cpp/ripple/ripple_SHAMapNode.cpp @@ -138,8 +138,8 @@ int SHAMapNode::selectBranch (uint256 const& hash) const if ((hash & smMasks[mDepth]) != mNodeID) { - std::cerr << "selectBranch(" << getString () << std::endl; - std::cerr << " " << hash << " off branch" << std::endl; + Log::out() << "selectBranch(" << getString (); + Log::out() << " " << hash << " off branch"; assert (false); return -1; // does not go under this node } diff --git a/src/cpp/ripple/ripple_SHAMapSync.cpp b/src/cpp/ripple/ripple_SHAMapSync.cpp index e80616b8a8..438b83d4b6 100644 --- a/src/cpp/ripple/ripple_SHAMapSync.cpp +++ b/src/cpp/ripple/ripple_SHAMapSync.cpp @@ -227,7 +227,7 @@ SHAMapAddNode SHAMap::addRootNode (Blob const& rootNode, SHANodeFormat format, if (!node) return SHAMapAddNode::invalid (); -#ifdef DEBUG +#ifdef BEAST_DEBUG node->dump (); #endif @@ -568,57 +568,6 @@ void SHAMap::getFetchPack (SHAMap* have, bool includeLeaves, int max, } } -#ifdef DEBUG -#define SMS_DEBUG -#endif - -static SHAMapItem::pointer makeRandomAS () -{ - Serializer s; - - for (int d = 0; d < 3; ++d) s.add32 (rand ()); - - return boost::make_shared (s.getRIPEMD160 ().to256 (), s.peekData ()); -} - -static bool confuseMap (SHAMap& map, int count) -{ - // add a bunch of random states to a map, then remove them - // map should be the same - uint256 beforeHash = map.getHash (); - - std::list items; - - for (int i = 0; i < count; ++i) - { - SHAMapItem::pointer item = makeRandomAS (); - items.push_back (item->getTag ()); - - if (!map.addItem (*item, false, false)) - { - WriteLog (lsFATAL, SHAMap) << "Unable to add item to map"; - return false; - } - } - - for (std::list::iterator it = items.begin (); it != items.end (); ++it) - { - if (!map.delItem (*it)) - { - WriteLog (lsFATAL, SHAMap) << "Unable to remove item from map"; - return false; - } - } - - if (beforeHash != map.getHash ()) - { - WriteLog (lsFATAL, SHAMap) << "Hashes do not match"; - return false; - } - - return true; -} - std::list SHAMap::getTrustedPath (uint256 const& index) { boost::recursive_mutex::scoped_lock sl (mLock); @@ -640,148 +589,3 @@ std::list SHAMap::getTrustedPath (uint256 const& index) return path; } - -BOOST_AUTO_TEST_SUITE ( SHAMapSync ) - -BOOST_AUTO_TEST_CASE ( SHAMapSync_test ) -{ - WriteLog (lsTRACE, SHAMap) << "begin sync test"; - unsigned int seed; - RAND_pseudo_bytes (reinterpret_cast (&seed), sizeof (seed)); - srand (seed); - - WriteLog (lsTRACE, SHAMap) << "Constructing maps"; - SHAMap source (smtFREE), destination (smtFREE); - - // add random data to the source map - WriteLog (lsTRACE, SHAMap) << "Adding random data"; - int items = 10000; - - for (int i = 0; i < items; ++i) - source.addItem (*makeRandomAS (), false, false); - - WriteLog (lsTRACE, SHAMap) << "Adding items, then removing them"; - - if (!confuseMap (source, 500)) BOOST_FAIL ("ConfuseMap"); - - source.setImmutable (); - - WriteLog (lsTRACE, SHAMap) << "SOURCE COMPLETE, SYNCHING"; - - std::vector nodeIDs, gotNodeIDs; - std::list< Blob > gotNodes; - std::vector hashes; - - std::vector::iterator nodeIDIterator; - std::list< Blob >::iterator rawNodeIterator; - - int passes = 0; - int nodes = 0; - - destination.setSynching (); - - if (!source.getNodeFat (SHAMapNode (), nodeIDs, gotNodes, (rand () % 2) == 0, (rand () % 2) == 0)) - { - WriteLog (lsFATAL, SHAMap) << "GetNodeFat(root) fails"; - BOOST_FAIL ("GetNodeFat"); - } - - if (gotNodes.size () < 1) - { - WriteLog (lsFATAL, SHAMap) << "Didn't get root node " << gotNodes.size (); - BOOST_FAIL ("NodeSize"); - } - - if (!destination.addRootNode (*gotNodes.begin (), snfWIRE, NULL)) - { - WriteLog (lsFATAL, SHAMap) << "AddRootNode fails"; - BOOST_FAIL ("AddRootNode"); - } - - nodeIDs.clear (); - gotNodes.clear (); - - WriteLog (lsINFO, SHAMap) << "ROOT COMPLETE, INNER SYNCHING"; -#ifdef SMS_DEBUG - int bytes = 0; -#endif - - do - { - ++passes; - hashes.clear (); - - // get the list of nodes we know we need - destination.getMissingNodes (nodeIDs, hashes, 2048, NULL); - - if (nodeIDs.empty ()) break; - - WriteLog (lsINFO, SHAMap) << nodeIDs.size () << " needed nodes"; - - // get as many nodes as possible based on this information - for (nodeIDIterator = nodeIDs.begin (); nodeIDIterator != nodeIDs.end (); ++nodeIDIterator) - { - if (!source.getNodeFat (*nodeIDIterator, gotNodeIDs, gotNodes, (rand () % 2) == 0, (rand () % 2) == 0)) - { - WriteLog (lsFATAL, SHAMap) << "GetNodeFat fails"; - BOOST_FAIL ("GetNodeFat"); - } - } - - assert (gotNodeIDs.size () == gotNodes.size ()); - nodeIDs.clear (); - hashes.clear (); - - if (gotNodeIDs.empty ()) - { - WriteLog (lsFATAL, SHAMap) << "No nodes gotten"; - BOOST_FAIL ("Got Node ID"); - } - - WriteLog (lsTRACE, SHAMap) << gotNodeIDs.size () << " found nodes"; - - for (nodeIDIterator = gotNodeIDs.begin (), rawNodeIterator = gotNodes.begin (); - nodeIDIterator != gotNodeIDs.end (); ++nodeIDIterator, ++rawNodeIterator) - { - ++nodes; -#ifdef SMS_DEBUG - bytes += rawNodeIterator->size (); -#endif - - if (!destination.addKnownNode (*nodeIDIterator, *rawNodeIterator, NULL)) - { - WriteLog (lsTRACE, SHAMap) << "AddKnownNode fails"; - BOOST_FAIL ("AddKnownNode"); - } - } - - gotNodeIDs.clear (); - gotNodes.clear (); - - - } - while (1); - - destination.clearSynching (); - -#ifdef SMS_DEBUG - WriteLog (lsINFO, SHAMap) << "SYNCHING COMPLETE " << items << " items, " << nodes << " nodes, " << - bytes / 1024 << " KB"; -#endif - - if (!source.deepCompare (destination)) - { - WriteLog (lsFATAL, SHAMap) << "DeepCompare fails"; - BOOST_FAIL ("Deep Compare"); - } - -#ifdef SMS_DEBUG - WriteLog (lsINFO, SHAMap) << "SHAMapSync test passed: " << items << " items, " << - passes << " passes, " << nodes << " nodes"; -#endif - -} - -BOOST_AUTO_TEST_SUITE_END (); - -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_SHAMapSyncFilters.cpp b/src/cpp/ripple/ripple_SHAMapSyncFilters.cpp index f4500f30d2..db451f56c2 100644 --- a/src/cpp/ripple/ripple_SHAMapSyncFilters.cpp +++ b/src/cpp/ripple/ripple_SHAMapSyncFilters.cpp @@ -14,7 +14,7 @@ void ConsensusTransSetSF::gotNode (bool fromFilter, const SHAMapNode& id, uint25 if (fromFilter) return; - theApp->getTempNodeCache ().store (nodeHash, nodeData); + getApp().getTempNodeCache ().store (nodeHash, nodeData); if ((type == SHAMapTreeNode::tnTRANSACTION_NM) && (nodeData.size () > 16)) { @@ -27,8 +27,8 @@ void ConsensusTransSetSF::gotNode (bool fromFilter, const SHAMapNode& id, uint25 SerializerIterator sit (s); SerializedTransaction::pointer stx = boost::make_shared (boost::ref (sit)); assert (stx->getTransactionID () == nodeHash); - theApp->getJobQueue ().addJob (jtTRANSACTION, "TXS->TXN", - BIND_TYPE (&NetworkOPs::submitTransaction, &theApp->getOPs (), P_1, stx, NetworkOPs::stCallback ())); + getApp().getJobQueue ().addJob (jtTRANSACTION, "TXS->TXN", + BIND_TYPE (&NetworkOPs::submitTransaction, &getApp().getOPs (), P_1, stx, NetworkOPs::stCallback ())); } catch (...) { @@ -40,7 +40,7 @@ void ConsensusTransSetSF::gotNode (bool fromFilter, const SHAMapNode& id, uint25 bool ConsensusTransSetSF::haveNode (const SHAMapNode& id, uint256 const& nodeHash, Blob& nodeData) { - if (theApp->getTempNodeCache ().retrieve (nodeHash, nodeData)) + if (getApp().getTempNodeCache ().retrieve (nodeHash, nodeData)) return true; Transaction::pointer txn = Transaction::load (nodeHash); @@ -73,14 +73,14 @@ void AccountStateSF::gotNode (bool fromFilter, Blob const& nodeData, SHAMapTreeNode::TNType) { - theApp->getHashedObjectStore ().store (hotACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash); + getApp().getNodeStore ().store (hotACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash); } bool AccountStateSF::haveNode (SHAMapNode const& id, uint256 const& nodeHash, Blob& nodeData) { - return theApp->getOPs ().getFetchPack (nodeHash, nodeData); + return getApp().getOPs ().getFetchPack (nodeHash, nodeData); } //------------------------------------------------------------------------------ @@ -96,7 +96,7 @@ void TransactionStateSF::gotNode (bool fromFilter, Blob const& nodeData, SHAMapTreeNode::TNType type) { - theApp->getHashedObjectStore ().store ( + getApp().getNodeStore ().store ( (type == SHAMapTreeNode::tnTRANSACTION_NM) ? hotTRANSACTION : hotTRANSACTION_NODE, mLedgerSeq, nodeData, @@ -107,5 +107,5 @@ bool TransactionStateSF::haveNode (SHAMapNode const& id, uint256 const& nodeHash, Blob& nodeData) { - return theApp->getOPs ().getFetchPack (nodeHash, nodeData); + return getApp().getOPs ().getFetchPack (nodeHash, nodeData); } diff --git a/src/cpp/ripple/ripple_SHAMapSyncUnitTests.cpp b/src/cpp/ripple/ripple_SHAMapSyncUnitTests.cpp new file mode 100644 index 0000000000..5c2298f391 --- /dev/null +++ b/src/cpp/ripple/ripple_SHAMapSyncUnitTests.cpp @@ -0,0 +1,206 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifdef BEAST_DEBUG +#define SMS_DEBUG +#endif + +namespace ripple +{ + +static SHAMapItem::pointer makeRandomAS () +{ + Serializer s; + + for (int d = 0; d < 3; ++d) s.add32 (rand ()); + + return boost::make_shared (s.getRIPEMD160 ().to256 (), s.peekData ()); +} + +static bool confuseMap (SHAMap& map, int count) +{ + // add a bunch of random states to a map, then remove them + // map should be the same + uint256 beforeHash = map.getHash (); + + std::list items; + + for (int i = 0; i < count; ++i) + { + SHAMapItem::pointer item = makeRandomAS (); + items.push_back (item->getTag ()); + + if (!map.addItem (*item, false, false)) + { + WriteLog (lsFATAL, SHAMap) << "Unable to add item to map"; + return false; + } + } + + for (std::list::iterator it = items.begin (); it != items.end (); ++it) + { + if (!map.delItem (*it)) + { + WriteLog (lsFATAL, SHAMap) << "Unable to remove item from map"; + return false; + } + } + + if (beforeHash != map.getHash ()) + { + WriteLog (lsFATAL, SHAMap) << "Hashes do not match"; + return false; + } + + return true; +} + +} + +BOOST_AUTO_TEST_SUITE ( SHAMapSync ) + +BOOST_AUTO_TEST_CASE ( SHAMapSync_test ) +{ + using namespace ripple; + + WriteLog (lsTRACE, SHAMap) << "begin sync test"; + unsigned int seed; + RAND_pseudo_bytes (reinterpret_cast (&seed), sizeof (seed)); + srand (seed); + + WriteLog (lsTRACE, SHAMap) << "Constructing maps"; + SHAMap source (smtFREE), destination (smtFREE); + + // add random data to the source map + WriteLog (lsTRACE, SHAMap) << "Adding random data"; + int items = 10000; + + for (int i = 0; i < items; ++i) + source.addItem (*makeRandomAS (), false, false); + + WriteLog (lsTRACE, SHAMap) << "Adding items, then removing them"; + + if (!confuseMap (source, 500)) BOOST_FAIL ("ConfuseMap"); + + source.setImmutable (); + + WriteLog (lsTRACE, SHAMap) << "SOURCE COMPLETE, SYNCHING"; + + std::vector nodeIDs, gotNodeIDs; + std::list< Blob > gotNodes; + std::vector hashes; + + std::vector::iterator nodeIDIterator; + std::list< Blob >::iterator rawNodeIterator; + + int passes = 0; + int nodes = 0; + + destination.setSynching (); + + if (!source.getNodeFat (SHAMapNode (), nodeIDs, gotNodes, (rand () % 2) == 0, (rand () % 2) == 0)) + { + WriteLog (lsFATAL, SHAMap) << "GetNodeFat(root) fails"; + BOOST_FAIL ("GetNodeFat"); + } + + if (gotNodes.size () < 1) + { + WriteLog (lsFATAL, SHAMap) << "Didn't get root node " << gotNodes.size (); + BOOST_FAIL ("NodeSize"); + } + + if (!destination.addRootNode (*gotNodes.begin (), snfWIRE, NULL)) + { + WriteLog (lsFATAL, SHAMap) << "AddRootNode fails"; + BOOST_FAIL ("AddRootNode"); + } + + nodeIDs.clear (); + gotNodes.clear (); + + WriteLog (lsINFO, SHAMap) << "ROOT COMPLETE, INNER SYNCHING"; +#ifdef SMS_DEBUG + int bytes = 0; +#endif + + do + { + ++passes; + hashes.clear (); + + // get the list of nodes we know we need + destination.getMissingNodes (nodeIDs, hashes, 2048, NULL); + + if (nodeIDs.empty ()) break; + + WriteLog (lsINFO, SHAMap) << nodeIDs.size () << " needed nodes"; + + // get as many nodes as possible based on this information + for (nodeIDIterator = nodeIDs.begin (); nodeIDIterator != nodeIDs.end (); ++nodeIDIterator) + { + if (!source.getNodeFat (*nodeIDIterator, gotNodeIDs, gotNodes, (rand () % 2) == 0, (rand () % 2) == 0)) + { + WriteLog (lsFATAL, SHAMap) << "GetNodeFat fails"; + BOOST_FAIL ("GetNodeFat"); + } + } + + assert (gotNodeIDs.size () == gotNodes.size ()); + nodeIDs.clear (); + hashes.clear (); + + if (gotNodeIDs.empty ()) + { + WriteLog (lsFATAL, SHAMap) << "No nodes gotten"; + BOOST_FAIL ("Got Node ID"); + } + + WriteLog (lsTRACE, SHAMap) << gotNodeIDs.size () << " found nodes"; + + for (nodeIDIterator = gotNodeIDs.begin (), rawNodeIterator = gotNodes.begin (); + nodeIDIterator != gotNodeIDs.end (); ++nodeIDIterator, ++rawNodeIterator) + { + ++nodes; +#ifdef SMS_DEBUG + bytes += rawNodeIterator->size (); +#endif + + if (!destination.addKnownNode (*nodeIDIterator, *rawNodeIterator, NULL)) + { + WriteLog (lsTRACE, SHAMap) << "AddKnownNode fails"; + BOOST_FAIL ("AddKnownNode"); + } + } + + gotNodeIDs.clear (); + gotNodes.clear (); + + + } + while (1); + + destination.clearSynching (); + +#ifdef SMS_DEBUG + WriteLog (lsINFO, SHAMap) << "SYNCHING COMPLETE " << items << " items, " << nodes << " nodes, " << + bytes / 1024 << " KB"; +#endif + + if (!source.deepCompare (destination)) + { + WriteLog (lsFATAL, SHAMap) << "DeepCompare fails"; + BOOST_FAIL ("Deep Compare"); + } + +#ifdef SMS_DEBUG + WriteLog (lsINFO, SHAMap) << "SHAMapSync test passed: " << items << " items, " << + passes << " passes, " << nodes << " nodes"; +#endif + +} + +BOOST_AUTO_TEST_SUITE_END (); diff --git a/src/cpp/ripple/ripple_SHAMapTreeNode.cpp b/src/cpp/ripple/ripple_SHAMapTreeNode.cpp index 4f096071b4..d9b4b7eb77 100644 --- a/src/cpp/ripple/ripple_SHAMapTreeNode.cpp +++ b/src/cpp/ripple/ripple_SHAMapTreeNode.cpp @@ -37,9 +37,9 @@ SHAMapTreeNode::SHAMapTreeNode (const SHAMapNode& id, Blob const& rawNode, uint3 if ((type < 0) || (type > 4)) { -#ifdef DEBUG - std::cerr << "Invalid wire format node" << std::endl; - std::cerr << strHex (rawNode) << std::endl; +#ifdef BEAST_DEBUG + Log::out() << "Invalid wire format node"; + Log::out() << strHex (rawNode); assert (false); #endif throw std::runtime_error ("invalid node AW type"); diff --git a/src/cpp/ripple/ripple_SHAMapTreeNode.h b/src/cpp/ripple/ripple_SHAMapTreeNode.h index bc653adeff..bb7933aed5 100644 --- a/src/cpp/ripple/ripple_SHAMapTreeNode.h +++ b/src/cpp/ripple/ripple_SHAMapTreeNode.h @@ -21,6 +21,8 @@ class SHAMapTreeNode , public CountedObject { public: + static char const* getCountedObjectName () { return "SHAMapTreeNode"; } + typedef boost::shared_ptr pointer; typedef const boost::shared_ptr& ref; diff --git a/src/cpp/ripple/ripple_SHAMapUnitTests.cpp b/src/cpp/ripple/ripple_SHAMapUnitTests.cpp new file mode 100644 index 0000000000..24354baaca --- /dev/null +++ b/src/cpp/ripple/ripple_SHAMapUnitTests.cpp @@ -0,0 +1,93 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +// VFALCO TODO Rename this to createFilledVector and pass an unsigned char, tidy up +// +namespace ripple +{ + static Blob IntToVUC (int v) + { + Blob vuc; + + for (int i = 0; i < 32; ++i) + vuc.push_back (static_cast (v)); + + return vuc; + } +} + +BOOST_AUTO_TEST_SUITE (SHAMap_suite) + +BOOST_AUTO_TEST_CASE ( SHAMap_test ) +{ + using namespace ripple; + + // h3 and h4 differ only in the leaf, same terminal node (level 19) + WriteLog (lsTRACE, SHAMap) << "SHAMap test"; + uint256 h1, h2, h3, h4, h5; + h1.SetHex ("092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); + h2.SetHex ("436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe"); + h3.SetHex ("b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8"); + h4.SetHex ("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8"); + h5.SetHex ("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); + + SHAMap sMap (smtFREE); + SHAMapItem i1 (h1, IntToVUC (1)), i2 (h2, IntToVUC (2)), i3 (h3, IntToVUC (3)), i4 (h4, IntToVUC (4)), i5 (h5, IntToVUC (5)); + + if (!sMap.addItem (i2, true, false)) BOOST_FAIL ("no add"); + + if (!sMap.addItem (i1, true, false)) BOOST_FAIL ("no add"); + + SHAMapItem::pointer i; + + i = sMap.peekFirstItem (); + + if (!i || (*i != i1)) BOOST_FAIL ("bad traverse"); + + i = sMap.peekNextItem (i->getTag ()); + + if (!i || (*i != i2)) BOOST_FAIL ("bad traverse"); + + i = sMap.peekNextItem (i->getTag ()); + + if (i) BOOST_FAIL ("bad traverse"); + + sMap.addItem (i4, true, false); + sMap.delItem (i2.getTag ()); + sMap.addItem (i3, true, false); + + i = sMap.peekFirstItem (); + + if (!i || (*i != i1)) BOOST_FAIL ("bad traverse"); + + i = sMap.peekNextItem (i->getTag ()); + + if (!i || (*i != i3)) BOOST_FAIL ("bad traverse"); + + i = sMap.peekNextItem (i->getTag ()); + + if (!i || (*i != i4)) BOOST_FAIL ("bad traverse"); + + i = sMap.peekNextItem (i->getTag ()); + + if (i) BOOST_FAIL ("bad traverse"); + + WriteLog (lsTRACE, SHAMap) << "SHAMap snap test"; + uint256 mapHash = sMap.getHash (); + SHAMap::pointer map2 = sMap.snapShot (false); + + if (sMap.getHash () != mapHash) BOOST_FAIL ("bad snapshot"); + + if (map2->getHash () != mapHash) BOOST_FAIL ("bad snapshot"); + + if (!sMap.delItem (sMap.peekFirstItem ()->getTag ())) BOOST_FAIL ("bad mod"); + + if (sMap.getHash () == mapHash) BOOST_FAIL ("bad snapshot"); + + if (map2->getHash () != mapHash) BOOST_FAIL ("bad snapshot"); +} + +BOOST_AUTO_TEST_SUITE_END (); diff --git a/src/cpp/ripple/ripple_SerializedLedger.cpp b/src/cpp/ripple/ripple_SerializedLedger.cpp index 1a6f3012c6..61b0f7498b 100644 --- a/src/cpp/ripple/ripple_SerializedLedger.cpp +++ b/src/cpp/ripple/ripple_SerializedLedger.cpp @@ -4,24 +4,25 @@ */ //============================================================================== -// For logging -struct SerializedLedgerLog; +struct SerializedLedgerLog; // for Log -SETUP_LOG (SerializedLedgerLog) +SETUP_LOGN (SerializedLedgerLog,"SerializedLedger") SerializedLedgerEntry::SerializedLedgerEntry (SerializerIterator& sit, uint256 const& index) : STObject (sfLedgerEntry), mIndex (index), mMutable (true) { set (sit); uint16 type = getFieldU16 (sfLedgerEntryType); - mFormat = LedgerEntryFormat::getLgrFormat (static_cast (type)); - - if (mFormat == NULL) + + LedgerFormats::Item const* const item = + LedgerFormats::getInstance()->findByType (static_cast (type)); + + if (item == nullptr) throw std::runtime_error ("invalid ledger entry type"); - mType = mFormat->t_type; + mType = item->getType (); - if (!setType (mFormat->elements)) + if (!setType (item->elements)) throw std::runtime_error ("ledger entry not valid for type"); } @@ -32,16 +33,18 @@ SerializedLedgerEntry::SerializedLedgerEntry (const Serializer& s, uint256 const set (sit); uint16 type = getFieldU16 (sfLedgerEntryType); - mFormat = LedgerEntryFormat::getLgrFormat (static_cast (type)); - if (mFormat == NULL) + LedgerFormats::Item const* const item = + LedgerFormats::getInstance()->findByType (static_cast (type)); + + if (item == nullptr) throw std::runtime_error ("invalid ledger entry type"); - mType = mFormat->t_type; + mType = item->getType (); - if (!setType (mFormat->elements)) + if (!setType (item->elements)) { - WriteLog (lsWARNING, SerializedLedgerLog) << "Ledger entry not valid for type " << mFormat->t_name; + WriteLog (lsWARNING, SerializedLedgerLog) << "Ledger entry not valid for type " << mFormat->getName (); WriteLog (lsWARNING, SerializedLedgerLog) << getJson (0); throw std::runtime_error ("ledger entry not valid for type"); } @@ -50,12 +53,19 @@ SerializedLedgerEntry::SerializedLedgerEntry (const Serializer& s, uint256 const SerializedLedgerEntry::SerializedLedgerEntry (LedgerEntryType type, uint256 const& index) : STObject (sfLedgerEntry), mIndex (index), mType (type), mMutable (true) { - mFormat = LedgerEntryFormat::getLgrFormat (type); + LedgerFormats::Item const* const item = + LedgerFormats::getInstance()->findByType (type); - if (mFormat == NULL) throw std::runtime_error ("invalid ledger entry type"); + if (item != nullptr) + { + set (item->elements); - set (mFormat->elements); - setFieldU16 (sfLedgerEntryType, static_cast (mFormat->t_type)); + setFieldU16 (sfLedgerEntryType, static_cast (item->getType ())); + } + else + { + throw std::runtime_error ("invalid ledger entry type"); + } } SerializedLedgerEntry::pointer SerializedLedgerEntry::getMutable () const @@ -70,7 +80,7 @@ std::string SerializedLedgerEntry::getFullText () const std::string ret = "\""; ret += mIndex.GetHex (); ret += "\" = { "; - ret += mFormat->t_name; + ret += mFormat->getName (); ret += ", "; ret += STObject::getFullText (); ret += "}"; diff --git a/src/cpp/ripple/ripple_SerializedLedger.h b/src/cpp/ripple/ripple_SerializedLedger.h index 37984b0ae1..0a7e75d172 100644 --- a/src/cpp/ripple/ripple_SerializedLedger.h +++ b/src/cpp/ripple/ripple_SerializedLedger.h @@ -27,6 +27,8 @@ class SerializedLedgerEntry , public CountedObject { public: + static char const* getCountedObjectName () { return "SerializedLedgerEntry"; } + typedef boost::shared_ptr pointer; typedef const boost::shared_ptr& ref; @@ -70,7 +72,7 @@ public: { return getFieldU16 (sfLedgerEntryType); } - const LedgerEntryFormat* getFormat () + LedgerFormats::Item const* getFormat () { return mFormat; } @@ -96,7 +98,7 @@ private: private: uint256 mIndex; LedgerEntryType mType; - const LedgerEntryFormat* mFormat; + LedgerFormats::Item const* mFormat; bool mMutable; }; diff --git a/src/cpp/ripple/ripple_SerializedTransaction.cpp b/src/cpp/ripple/ripple_SerializedTransaction.cpp index 9d43b7b559..45ba2ffb46 100644 --- a/src/cpp/ripple/ripple_SerializedTransaction.cpp +++ b/src/cpp/ripple/ripple_SerializedTransaction.cpp @@ -6,15 +6,15 @@ SETUP_LOG (SerializedTransaction) -SerializedTransaction::SerializedTransaction (TransactionType type) +SerializedTransaction::SerializedTransaction (TxType type) : STObject (sfTransaction) , mType (type) , mSigGood (false) , mSigBad (false) { - mFormat = TxFormats::getInstance ().findByType (type); + mFormat = TxFormats::getInstance()->findByType (type); - if (mFormat == NULL) + if (mFormat == nullptr) { WriteLog (lsWARNING, SerializedTransaction) << "Transaction type: " << type; throw std::runtime_error ("invalid transaction type"); @@ -29,9 +29,9 @@ SerializedTransaction::SerializedTransaction (STObject const& object) , mSigGood (false) , mSigBad (false) { - mType = static_cast (getFieldU16 (sfTransactionType)); + mType = static_cast (getFieldU16 (sfTransactionType)); - mFormat = TxFormats::getInstance ().findByType (mType); + mFormat = TxFormats::getInstance()->findByType (mType); if (!mFormat) { @@ -57,9 +57,9 @@ SerializedTransaction::SerializedTransaction (SerializerIterator& sit) : STObjec } set (sit); - mType = static_cast (getFieldU16 (sfTransactionType)); + mType = static_cast (getFieldU16 (sfTransactionType)); - mFormat = TxFormats::getInstance ().findByType (mType); + mFormat = TxFormats::getInstance()->findByType (mType); if (!mFormat) { @@ -248,26 +248,6 @@ std::string SerializedTransaction::getMetaSQLValueHeader () return "(TransID, TransType, FromAcct, FromSeq, LedgerSeq, Status, RawTxn, TxnMeta)"; } -std::string SerializedTransaction::getSQLInsertHeader () -{ - return "INSERT INTO Transactions " + getSQLValueHeader () + " VALUES "; -} - -std::string SerializedTransaction::getSQLInsertIgnoreHeader () -{ - return "INSERT OR IGNORE INTO Transactions " + getSQLValueHeader () + " VALUES "; -} - -std::string SerializedTransaction::getSQLInsertReplaceHeader () -{ - return "INSERT OR REPLACE INTO Transactions " + getSQLValueHeader () + " VALUES "; -} - -std::string SerializedTransaction::getMetaSQLInsertHeader () -{ - return "INSERT INTO Transactions " + getMetaSQLValueHeader () + " VALUES "; -} - std::string SerializedTransaction::getMetaSQLInsertReplaceHeader () { return "INSERT OR REPLACE INTO Transactions " + getMetaSQLValueHeader () + " VALUES "; @@ -307,50 +287,3 @@ std::string SerializedTransaction::getMetaSQL (Serializer rawTxn, uint32 inLedge % getTransactionID ().GetHex () % getTransactionType () % getSourceAccount ().humanAccountID () % getSequence () % inLedger % status % rTxn % escapedMetaData); } - - -BOOST_AUTO_TEST_SUITE (SerializedTransactionTS) - -BOOST_AUTO_TEST_CASE ( STrans_test ) -{ - RippleAddress seed; - seed.setSeedRandom (); - RippleAddress generator = RippleAddress::createGeneratorPublic (seed); - RippleAddress publicAcct = RippleAddress::createAccountPublic (generator, 1); - RippleAddress privateAcct = RippleAddress::createAccountPrivate (generator, seed, 1); - - SerializedTransaction j (ttACCOUNT_SET); - j.setSourceAccount (publicAcct); - j.setSigningPubKey (publicAcct); - j.setFieldVL (sfMessageKey, publicAcct.getAccountPublic ()); - j.sign (privateAcct); - - if (!j.checkSign ()) BOOST_FAIL ("Transaction fails signature test"); - - Serializer rawTxn; - j.add (rawTxn); - SerializerIterator sit (rawTxn); - SerializedTransaction copy (sit); - - if (copy != j) - { - Log (lsFATAL) << j.getJson (0); - Log (lsFATAL) << copy.getJson (0); - BOOST_FAIL ("Transaction fails serialize/deserialize test"); - } - - UPTR_T new_obj = STObject::parseJson (j.getJson (0), sfGeneric); - - if (new_obj.get () == NULL) BOOST_FAIL ("Unable to build object from json"); - - if (STObject (j) != *new_obj) - { - Log (lsINFO) << "ORIG: " << j.getJson (0); - Log (lsINFO) << "BUILT " << new_obj->getJson (0); - BOOST_FAIL ("Built a different transaction"); - } -} - -BOOST_AUTO_TEST_SUITE_END (); - -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_SerializedTransaction.h b/src/cpp/ripple/ripple_SerializedTransaction.h index dd17e293f6..31ecc83287 100644 --- a/src/cpp/ripple/ripple_SerializedTransaction.h +++ b/src/cpp/ripple/ripple_SerializedTransaction.h @@ -21,12 +21,14 @@ class SerializedTransaction , public CountedObject { public: + static char const* getCountedObjectName () { return "SerializedTransaction"; } + typedef boost::shared_ptr pointer; typedef const boost::shared_ptr& ref; public: SerializedTransaction (SerializerIterator & sit); - SerializedTransaction (TransactionType type); + SerializedTransaction (TxType type); SerializedTransaction (const STObject & object); // STObject functions @@ -45,7 +47,7 @@ public: } uint256 getSigningHash () const; - TransactionType getTxnType () const + TxType getTxnType () const { return mType; } @@ -112,21 +114,19 @@ public: static std::string getSQLValueHeader (); static std::string getSQLInsertHeader (); static std::string getSQLInsertIgnoreHeader (); - static std::string getSQLInsertReplaceHeader (); std::string getSQL (std::string & sql, uint32 inLedger, char status) const; std::string getSQL (uint32 inLedger, char status) const; std::string getSQL (Serializer rawTxn, uint32 inLedger, char status) const; // SQL Functions with metadata static std::string getMetaSQLValueHeader (); - static std::string getMetaSQLInsertHeader (); static std::string getMetaSQLInsertReplaceHeader (); std::string getMetaSQL (uint32 inLedger, const std::string & escapedMetaData) const; std::string getMetaSQL (Serializer rawTxn, uint32 inLedger, char status, const std::string & escapedMetaData) const; private: - TransactionType mType; - const TxFormat* mFormat; + TxType mType; + TxFormats::Item const* mFormat; SerializedTransaction* duplicate () const { diff --git a/src/cpp/ripple/ripple_SerializedTransactionUnitTests.cpp b/src/cpp/ripple/ripple_SerializedTransactionUnitTests.cpp new file mode 100644 index 0000000000..4beb4c7f12 --- /dev/null +++ b/src/cpp/ripple/ripple_SerializedTransactionUnitTests.cpp @@ -0,0 +1,50 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== +BOOST_AUTO_TEST_SUITE (SerializedTransactionTS) + +BOOST_AUTO_TEST_CASE ( STrans_test ) +{ + using namespace ripple; + + RippleAddress seed; + seed.setSeedRandom (); + RippleAddress generator = RippleAddress::createGeneratorPublic (seed); + RippleAddress publicAcct = RippleAddress::createAccountPublic (generator, 1); + RippleAddress privateAcct = RippleAddress::createAccountPrivate (generator, seed, 1); + + SerializedTransaction j (ttACCOUNT_SET); + j.setSourceAccount (publicAcct); + j.setSigningPubKey (publicAcct); + j.setFieldVL (sfMessageKey, publicAcct.getAccountPublic ()); + j.sign (privateAcct); + + if (!j.checkSign ()) BOOST_FAIL ("Transaction fails signature test"); + + Serializer rawTxn; + j.add (rawTxn); + SerializerIterator sit (rawTxn); + SerializedTransaction copy (sit); + + if (copy != j) + { + Log (lsFATAL) << j.getJson (0); + Log (lsFATAL) << copy.getJson (0); + BOOST_FAIL ("Transaction fails serialize/deserialize test"); + } + + UPTR_T new_obj = STObject::parseJson (j.getJson (0), sfGeneric); + + if (new_obj.get () == NULL) BOOST_FAIL ("Unable to build object from json"); + + if (STObject (j) != *new_obj) + { + Log (lsINFO) << "ORIG: " << j.getJson (0); + Log (lsINFO) << "BUILT " << new_obj->getJson (0); + BOOST_FAIL ("Built a different transaction"); + } +} + +BOOST_AUTO_TEST_SUITE_END (); diff --git a/src/cpp/ripple/ripple_SqliteDatabase.cpp b/src/cpp/ripple/ripple_SqliteDatabase.cpp index 0e29c5a4a2..2ffcb47e97 100644 --- a/src/cpp/ripple/ripple_SqliteDatabase.cpp +++ b/src/cpp/ripple/ripple_SqliteDatabase.cpp @@ -6,8 +6,6 @@ SETUP_LOG (SqliteDatabase) -//using namespace std; - SqliteStatement::SqliteStatement (SqliteDatabase* db, const char* sql, bool aux) { assert (db); @@ -35,7 +33,10 @@ SqliteStatement::~SqliteStatement () sqlite3_finalize (statement); } -SqliteDatabase::SqliteDatabase (const char* host) : Database (host, "", ""), mWalQ (NULL), walRunning (false) +SqliteDatabase::SqliteDatabase (const char* host) + : Database (host) + , mWalQ (NULL) + , walRunning (false) { mConnection = NULL; mAuxConnection = NULL; @@ -102,7 +103,7 @@ bool SqliteDatabase::executeSQL (const char* sql, bool fail_ok) { if (!fail_ok) { -#ifdef DEBUG +#ifdef BEAST_DEBUG WriteLog (lsWARNING, SqliteDatabase) << "Perror:" << mHost << ": " << rc; WriteLog (lsWARNING, SqliteDatabase) << "Statement: " << sql; WriteLog (lsWARNING, SqliteDatabase) << "Error: " << sqlite3_errmsg (mConnection); @@ -136,7 +137,7 @@ bool SqliteDatabase::executeSQL (const char* sql, bool fail_ok) if (!fail_ok) { -#ifdef DEBUG +#ifdef BEAST_DEBUG WriteLog (lsWARNING, SqliteDatabase) << "SQL Serror:" << mHost << ": " << rc; WriteLog (lsWARNING, SqliteDatabase) << "Statement: " << sql; WriteLog (lsWARNING, SqliteDatabase) << "Error: " << sqlite3_errmsg (mConnection); @@ -289,9 +290,9 @@ void SqliteDatabase::doHook (const char* db, int pages) } if (mWalQ) - mWalQ->addJob (jtWAL, std::string ("WAL:") + mHost, boost::bind (&SqliteDatabase::runWal, this)); + mWalQ->addJob (jtWAL, std::string ("WAL:") + mHost, BIND_TYPE (&SqliteDatabase::runWal, this)); else - boost::thread (boost::bind (&SqliteDatabase::runWal, this)).detach (); + boost::thread (BIND_TYPE (&SqliteDatabase::runWal, this)).detach (); } void SqliteDatabase::runWal () diff --git a/src/cpp/ripple/ripple_SqliteDatabase.h b/src/cpp/ripple/ripple_SqliteDatabase.h index 05bf6c1ddb..a1bac93b56 100644 --- a/src/cpp/ripple/ripple_SqliteDatabase.h +++ b/src/cpp/ripple/ripple_SqliteDatabase.h @@ -7,9 +7,6 @@ #ifndef RIPPLE_SQLITEDATABASE_RIPPLEHEADER #define RIPPLE_SQLITEDATABASE_RIPPLEHEADER -struct sqlite3; -struct sqlite3_stmt; - class SqliteDatabase : public Database { public: diff --git a/src/cpp/ripple/ripple_TransactionAcquire.cpp b/src/cpp/ripple/ripple_TransactionAcquire.cpp index 84df233288..7fd5b672b0 100644 --- a/src/cpp/ripple/ripple_TransactionAcquire.cpp +++ b/src/cpp/ripple/ripple_TransactionAcquire.cpp @@ -18,9 +18,9 @@ TransactionAcquire::TransactionAcquire (uint256 const& hash) : PeerSet (hash, TX static void TACompletionHandler (uint256 hash, SHAMap::pointer map) { - boost::recursive_mutex::scoped_lock sl (theApp->getMasterLock ()); - theApp->getOPs ().mapComplete (hash, map); - theApp->getInboundLedgers ().dropLedger (hash); + boost::recursive_mutex::scoped_lock sl (getApp().getMasterLock ()); + getApp().getOPs ().mapComplete (hash, map); + getApp().getInboundLedgers ().dropLedger (hash); } void TransactionAcquire::done () @@ -39,25 +39,29 @@ void TransactionAcquire::done () map = mMap; } - theApp->getIOService ().post (boost::bind (&TACompletionHandler, mHash, map)); + getApp().getIOService ().post (BIND_TYPE (&TACompletionHandler, mHash, map)); } -void TransactionAcquire::onTimer (bool progress) +void TransactionAcquire::onTimer (bool progress, boost::recursive_mutex::scoped_lock& psl) { bool aggressive = false; if (getTimeouts () > 10) { WriteLog (lsWARNING, TransactionAcquire) << "Ten timeouts on TX set " << getHash (); - { - boost::recursive_mutex::scoped_lock sl (theApp->getMasterLock ()); - - if (theApp->getOPs ().stillNeedTXSet (mHash)) + { // FIXME: Acquire the master lock here can deadlock + psl.unlock(); { - WriteLog (lsWARNING, TransactionAcquire) << "Still need it"; - mTimeouts = 0; - aggressive = true; - } + boost::recursive_mutex::scoped_lock sl (getApp().getMasterLock ()); + + if (getApp().getOPs ().stillNeedTXSet (mHash)) + { + WriteLog (lsWARNING, TransactionAcquire) << "Still need it"; + mTimeouts = 0; + aggressive = true; + } + } + psl.lock(); } if (!aggressive) @@ -74,7 +78,7 @@ void TransactionAcquire::onTimer (bool progress) WriteLog (lsWARNING, TransactionAcquire) << "Out of peers for TX set " << getHash (); bool found = false; - std::vector peerList = theApp->getPeers ().getPeerVector (); + std::vector peerList = getApp().getPeers ().getPeerVector (); BOOST_FOREACH (Peer::ref peer, peerList) { if (peer->hasTxSet (getHash ())) diff --git a/src/cpp/ripple/ripple_TransactionAcquire.h b/src/cpp/ripple/ripple_TransactionAcquire.h index 347ae01c83..dcfa7a81b4 100644 --- a/src/cpp/ripple/ripple_TransactionAcquire.h +++ b/src/cpp/ripple/ripple_TransactionAcquire.h @@ -15,6 +15,8 @@ class TransactionAcquire , public CountedObject { public: + static char const* getCountedObjectName () { return "TransactionAcquire"; } + typedef boost::shared_ptr pointer; public: @@ -36,7 +38,7 @@ private: SHAMap::pointer mMap; bool mHaveRoot; - void onTimer (bool progress); + void onTimer (bool progress, boost::recursive_mutex::scoped_lock& peerSetLock); void newPeer (Peer::ref peer) { trigger (peer); diff --git a/src/cpp/ripple/ripple_UniqueNodeList.cpp b/src/cpp/ripple/ripple_UniqueNodeList.cpp index 0d88883374..fb3ff71a95 100644 --- a/src/cpp/ripple/ripple_UniqueNodeList.cpp +++ b/src/cpp/ripple/ripple_UniqueNodeList.cpp @@ -21,134 +21,6 @@ // Don't bother propagating past this number of rounds. #define SCORE_ROUNDS 10 -// VFALCO TODO move all function definitions inlined into the class. -class UniqueNodeList : public IUniqueNodeList -{ -private: - // Misc persistent information - boost::posix_time::ptime mtpScoreUpdated; - boost::posix_time::ptime mtpFetchUpdated; - - boost::recursive_mutex mUNLLock; - // XXX Make this faster, make this the contents vector unsigned char or raw public key. - // XXX Contents needs to based on score. - boost::unordered_set mUNL; - - bool miscLoad (); - bool miscSave (); - - typedef struct - { - std::string strDomain; - RippleAddress naPublicKey; - validatorSource vsSource; - boost::posix_time::ptime tpNext; - boost::posix_time::ptime tpScan; - boost::posix_time::ptime tpFetch; - uint256 iSha256; - std::string strComment; - } seedDomain; - - typedef struct - { - RippleAddress naPublicKey; - validatorSource vsSource; - boost::posix_time::ptime tpNext; - boost::posix_time::ptime tpScan; - boost::posix_time::ptime tpFetch; - uint256 iSha256; - std::string strComment; - } seedNode; - - // Used to distribute scores. - typedef struct - { - int iScore; - int iRoundScore; - int iRoundSeed; - int iSeen; - std::string strValidator; // The public key. - std::vector viReferrals; - } scoreNode; - - std::map sClusterNodes; - - typedef boost::unordered_map strIndex; - typedef std::pair ipPort; - typedef boost::unordered_map, score> epScore; - - void trustedLoad (); - - bool scoreRound (std::vector& vsnNodes); - - bool responseFetch (const std::string& strDomain, const boost::system::error_code& err, int iStatus, const std::string& strSiteFile); - - boost::posix_time::ptime mtpScoreNext; // When to start scoring. - boost::posix_time::ptime mtpScoreStart; // Time currently started scoring. - boost::asio::deadline_timer mdtScoreTimer; // Timer to start scoring. - - void scoreNext (bool bNow); // Update scoring timer. - void scoreCompute (); - void scoreTimerHandler (const boost::system::error_code& err); - - boost::mutex mFetchLock; - int mFetchActive; // Count of active fetches. - - boost::posix_time::ptime mtpFetchNext; // Time of to start next fetch. - boost::asio::deadline_timer mdtFetchTimer; // Timer to start fetching. - - void fetchNext (); - void fetchDirty (); - void fetchFinish (); - void fetchProcess (std::string strDomain); - void fetchTimerHandler (const boost::system::error_code& err); - - void getValidatorsUrl (const RippleAddress& naNodePublic, Section secSite); - void getIpsUrl (const RippleAddress& naNodePublic, Section secSite); - bool responseIps (const std::string& strSite, const RippleAddress& naNodePublic, const boost::system::error_code& err, int iStatus, const std::string& strIpsFile); - bool responseValidators (const std::string& strValidatorsUrl, const RippleAddress& naNodePublic, Section secSite, const std::string& strSite, const boost::system::error_code& err, int iStatus, const std::string& strValidatorsFile); - - void processIps (const std::string& strSite, const RippleAddress& naNodePublic, Section::mapped_type* pmtVecStrIps); - int processValidators (const std::string& strSite, const std::string& strValidatorsSrc, const RippleAddress& naNodePublic, validatorSource vsWhy, Section::mapped_type* pmtVecStrValidators); - - void processFile (const std::string& strDomain, const RippleAddress& naNodePublic, Section secSite); - - bool getSeedDomains (const std::string& strDomain, seedDomain& dstSeedDomain); - void setSeedDomains (const seedDomain& dstSeedDomain, bool bNext); - - bool getSeedNodes (const RippleAddress& naNodePublic, seedNode& dstSeedNode); - void setSeedNodes (const seedNode& snSource, bool bNext); - - bool validatorsResponse (const boost::system::error_code& err, int iStatus, const std::string strResponse); - void nodeProcess (const std::string& strSite, const std::string& strValidators, const std::string& strSource); - -public: - UniqueNodeList (boost::asio::io_service& io_service); - - // Begin processing. - void start (); - - void nodeAddPublic (const RippleAddress& naNodePublic, validatorSource vsWhy, const std::string& strComment); - void nodeAddDomain (std::string strDomain, validatorSource vsWhy, const std::string& strComment = ""); - void nodeRemovePublic (const RippleAddress& naNodePublic); - void nodeRemoveDomain (std::string strDomain); - void nodeReset (); - - void nodeScore (); - - bool nodeInUNL (const RippleAddress& naNodePublic); - bool nodeInCluster (const RippleAddress& naNodePublic); - bool nodeInCluster (const RippleAddress& naNodePublic, std::string& name); - - void nodeBootstrap (); - bool nodeLoad (boost::filesystem::path pConfig); - void nodeNetwork (); - - Json::Value getUnlJson (); - - int iSourceScore (validatorSource vsWhy); -}; - // VFALCO TODO Replace macros with language constructs #define VALIDATORS_FETCH_SECONDS 30 #define VALIDATORS_FILE_BYTES_MAX (50 << 10) @@ -169,185 +41,816 @@ public: SETUP_LOG (UniqueNodeList) -UniqueNodeList::UniqueNodeList (boost::asio::io_service& io_service) : - mdtScoreTimer (io_service), - mFetchActive (0), - mdtFetchTimer (io_service) +// VFALCO TODO move all function definitions inlined into the class. +class UniqueNodeListImp + : public UniqueNodeList + , public DeadlineTimer::Listener { -} - -// This is called when the application is started. -// Get update times and start fetching and scoring as needed. -void UniqueNodeList::start () -{ - miscLoad (); - - WriteLog (lsDEBUG, UniqueNodeList) << "Validator fetch updated: " << mtpFetchUpdated; - WriteLog (lsDEBUG, UniqueNodeList) << "Validator score updated: " << mtpScoreUpdated; - - fetchNext (); // Start fetching. - scoreNext (false); // Start scoring. -} - -// Load information about when we last updated. -bool UniqueNodeList::miscLoad () -{ - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - Database* db = theApp->getWalletDB ()->getDB (); - - if (!db->executeSQL ("SELECT * FROM Misc WHERE Magic=1;")) return false; - - bool bAvail = !!db->startIterRows (); - - mtpFetchUpdated = ptFromSeconds (bAvail ? db->getInt ("FetchUpdated") : -1); - mtpScoreUpdated = ptFromSeconds (bAvail ? db->getInt ("ScoreUpdated") : -1); - - db->endIterRows (); - - trustedLoad (); - - return true; -} - -// Persist update information. -bool UniqueNodeList::miscSave () -{ - Database* db = theApp->getWalletDB ()->getDB (); - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - db->executeSQL (str (boost::format ("REPLACE INTO Misc (Magic,FetchUpdated,ScoreUpdated) VALUES (1,%d,%d);") - % iToSeconds (mtpFetchUpdated) - % iToSeconds (mtpScoreUpdated))); - - return true; -} - -void UniqueNodeList::trustedLoad () -{ - boost::regex rNode ("\\`\\s*(\\S+)[\\s]*(.*)\\'"); - BOOST_FOREACH (const std::string & c, theConfig.CLUSTER_NODES) +private: + // VFALCO TODO Rename these structs? Are they objects with static storage? + // This looks like C and not C++... + // + typedef struct { - boost::smatch match; + std::string strDomain; + RippleAddress naPublicKey; + ValidatorSource vsSource; + boost::posix_time::ptime tpNext; + boost::posix_time::ptime tpScan; + boost::posix_time::ptime tpFetch; + uint256 iSha256; + std::string strComment; + } seedDomain; - if (boost::regex_match (c, match, rNode)) + typedef struct + { + RippleAddress naPublicKey; + ValidatorSource vsSource; + boost::posix_time::ptime tpNext; + boost::posix_time::ptime tpScan; + boost::posix_time::ptime tpFetch; + uint256 iSha256; + std::string strComment; + } seedNode; + + // Used to distribute scores. + typedef struct + { + int iScore; + int iRoundScore; + int iRoundSeed; + int iSeen; + std::string strValidator; // The public key. + std::vector viReferrals; + } scoreNode; + + typedef boost::unordered_map strIndex; + typedef std::pair IPAndPortNumber; + typedef boost::unordered_map, score> epScore; + +public: + UniqueNodeListImp () + : m_scoreTimer (this) + , mFetchActive (0) + , m_fetchTimer (this) + { + } + + //-------------------------------------------------------------------------- + + void onDeadlineTimer (DeadlineTimer& timer) + { + if (timer == m_scoreTimer) { - RippleAddress a = RippleAddress::createNodePublic (match[1]); + mtpScoreNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Timer not set. + mtpScoreStart = boost::posix_time::second_clock::universal_time (); // Scoring. - if (a.isValid ()) - sClusterNodes.insert (std::make_pair (a, match[2])); + WriteLog (lsTRACE, UniqueNodeList) << "Scoring: Start"; + + scoreCompute (); + + WriteLog (lsTRACE, UniqueNodeList) << "Scoring: End"; + + // Save update time. + mtpScoreUpdated = mtpScoreStart; + miscSave (); + + mtpScoreStart = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Not scoring. + + // Score again if needed. + scoreNext (false); + + // Scan may be dirty due to new ips. + getApp().getPeers ().scanRefresh (); } - else - WriteLog (lsWARNING, UniqueNodeList) << "Entry in cluster list invalid: '" << c << "'"; - } - - Database* db = theApp->getWalletDB ()->getDB (); - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - boost::recursive_mutex::scoped_lock slUNL (mUNLLock); - - mUNL.clear (); - - // XXX Needs to limit by quanity and quality. - SQL_FOREACH (db, "SELECT PublicKey FROM TrustedNodes WHERE Score != 0;") - { - mUNL.insert (db->getStrBinary ("PublicKey")); - } -} - -// For a round of scoring we destribute points from a node to nodes it refers to. -// Returns true, iff scores were distributed. -bool UniqueNodeList::scoreRound (std::vector& vsnNodes) -{ - bool bDist = false; - - // For each node, distribute roundSeed to roundScores. - BOOST_FOREACH (scoreNode & sn, vsnNodes) - { - int iEntries = sn.viReferrals.size (); - - if (sn.iRoundSeed && iEntries) + else if (timer == m_fetchTimer) { - score iTotal = (iEntries + 1) * iEntries / 2; - score iBase = sn.iRoundSeed * iEntries / iTotal; + // Time to check for another fetch. + WriteLog (lsTRACE, UniqueNodeList) << "fetchTimerHandler"; + fetchNext (); + } + } - // Distribute the current entires' seed score to validators prioritized by mention order. - for (int i = 0; i != iEntries; i++) + //-------------------------------------------------------------------------- + + // This is called when the application is started. + // Get update times and start fetching and scoring as needed. + void start () + { + miscLoad (); + + WriteLog (lsDEBUG, UniqueNodeList) << "Validator fetch updated: " << mtpFetchUpdated; + WriteLog (lsDEBUG, UniqueNodeList) << "Validator score updated: " << mtpScoreUpdated; + + fetchNext (); // Start fetching. + scoreNext (false); // Start scoring. + } + + //-------------------------------------------------------------------------- + + // Add a trusted node. Called by RPC or other source. + void nodeAddPublic (const RippleAddress& naNodePublic, ValidatorSource vsWhy, const std::string& strComment) + { + seedNode snCurrent; + + bool bFound = getSeedNodes (naNodePublic, snCurrent); + bool bChanged = false; + + if (!bFound) + { + snCurrent.naPublicKey = naNodePublic; + snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); + } + + // Promote source, if needed. + if (!bFound || iSourceScore (vsWhy) >= iSourceScore (snCurrent.vsSource)) + { + snCurrent.vsSource = vsWhy; + snCurrent.strComment = strComment; + bChanged = true; + } + + if (vsManual == vsWhy) + { + // A manual add forces immediate scan. + snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); + bChanged = true; + } + + if (bChanged) + setSeedNodes (snCurrent, true); + } + + //-------------------------------------------------------------------------- + + // Queue a domain for a single attempt fetch a ripple.txt. + // --> strComment: only used on vsManual + // YYY As a lot of these may happen at once, would be nice to wrap multiple calls in a transaction. + void nodeAddDomain (std::string strDomain, ValidatorSource vsWhy, const std::string& strComment) + { + boost::trim (strDomain); + boost::to_lower (strDomain); + + // YYY Would be best to verify strDomain is a valid domain. + // WriteLog (lsTRACE) << str(boost::format("nodeAddDomain: '%s' %c '%s'") + // % strDomain + // % vsWhy + // % strComment); + + seedDomain sdCurrent; + + bool bFound = getSeedDomains (strDomain, sdCurrent); + bool bChanged = false; + + if (!bFound) + { + sdCurrent.strDomain = strDomain; + sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); + } + + // Promote source, if needed. + if (!bFound || iSourceScore (vsWhy) >= iSourceScore (sdCurrent.vsSource)) + { + sdCurrent.vsSource = vsWhy; + sdCurrent.strComment = strComment; + bChanged = true; + } + + if (vsManual == vsWhy) + { + // A manual add forces immediate scan. + sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); + bChanged = true; + } + + if (bChanged) + setSeedDomains (sdCurrent, true); + } + + //-------------------------------------------------------------------------- + + void nodeRemovePublic (const RippleAddress& naNodePublic) + { + { + Database* db = getApp().getWalletDB ()->getDB (); + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + db->executeSQL (str (boost::format ("DELETE FROM SeedNodes WHERE PublicKey=%s") % sqlEscape (naNodePublic.humanNodePublic ()))); + db->executeSQL (str (boost::format ("DELETE FROM TrustedNodes WHERE PublicKey=%s") % sqlEscape (naNodePublic.humanNodePublic ()))); + } + + // YYY Only dirty on successful delete. + fetchDirty (); + + boost::recursive_mutex::scoped_lock sl (mUNLLock); + mUNL.erase (naNodePublic.humanNodePublic ()); + } + + //-------------------------------------------------------------------------- + + void nodeRemoveDomain (std::string strDomain) + { + boost::trim (strDomain); + boost::to_lower (strDomain); + + { + Database* db = getApp().getWalletDB ()->getDB (); + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + db->executeSQL (str (boost::format ("DELETE FROM SeedDomains WHERE Domain=%s") % sqlEscape (strDomain))); + } + + // YYY Only dirty on successful delete. + fetchDirty (); + } + + //-------------------------------------------------------------------------- + + void nodeReset () + { + { + Database* db = getApp().getWalletDB ()->getDB (); + + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + // XXX Check results. + db->executeSQL ("DELETE FROM SeedDomains"); + db->executeSQL ("DELETE FROM SeedNodes"); + } + + fetchDirty (); + } + + //-------------------------------------------------------------------------- + + // For debugging, schedule forced scoring. + void nodeScore () + { + scoreNext (true); + } + + //-------------------------------------------------------------------------- + + bool nodeInUNL (const RippleAddress& naNodePublic) + { + boost::recursive_mutex::scoped_lock sl (mUNLLock); + + return mUNL.end () != mUNL.find (naNodePublic.humanNodePublic ()); + } + + //-------------------------------------------------------------------------- + + bool nodeInCluster (const RippleAddress& naNodePublic) + { + boost::recursive_mutex::scoped_lock sl (mUNLLock); + return m_clusterNodes.end () != m_clusterNodes.find (naNodePublic); + } + + //-------------------------------------------------------------------------- + + bool nodeInCluster (const RippleAddress& naNodePublic, std::string& name) + { + boost::recursive_mutex::scoped_lock sl (mUNLLock); + std::map::iterator it = m_clusterNodes.find (naNodePublic); + + if (it == m_clusterNodes.end ()) + return false; + + name = it->second.getName(); + return true; + } + + //-------------------------------------------------------------------------- + + bool nodeUpdate (const RippleAddress& naNodePublic, ClusterNodeStatus const& cnsStatus) + { + boost::recursive_mutex::scoped_lock sl (mUNLLock); + return m_clusterNodes[naNodePublic].update(cnsStatus); + } + + //-------------------------------------------------------------------------- + + std::map getClusterStatus () + { + std::map ret; + { + boost::recursive_mutex::scoped_lock sl (mUNLLock); + ret = m_clusterNodes; + } + return ret; + } + + //-------------------------------------------------------------------------- + + uint32 getClusterFee () + { + int thresh = getApp().getOPs().getNetworkTimeNC() - 120; + uint32 a = 0, b = 0; + + boost::recursive_mutex::scoped_lock sl (mUNLLock); + { + for (std::map::iterator it = m_clusterNodes.begin(), + end = m_clusterNodes.end(); it != end; ++it) { - score iPoints = iBase * (iEntries - i) / iEntries; + if (it->second.getReportTime() >= thresh) + { + uint32 fee = it->second.getLoadFee(); + if (fee > b) + { + if (fee > a) + { + b = a; + a = fee; + } + else + b = fee; + } + } + } + } - vsnNodes[sn.viReferrals[i]].iRoundScore += iPoints; + return (b == 0) ? a : ((a + b + 1) / 2); + } + + //-------------------------------------------------------------------------- + + void addClusterStatus (Json::Value& obj) + { + boost::recursive_mutex::scoped_lock sl (mUNLLock); + if (m_clusterNodes.size() > 1) // nodes other than us + { + int now = getApp().getOPs().getNetworkTimeNC(); + uint32 ref = getApp().getFeeTrack().getLoadBase(); + Json::Value& nodes = (obj["cluster"] = Json::objectValue); + + for (std::map::iterator it = m_clusterNodes.begin(), + end = m_clusterNodes.end(); it != end; ++it) + { + if (it->first != getApp().getLocalCredentials().getNodePublic()) + { + Json::Value& node = nodes[it->first.humanNodePublic()]; + + if (!it->second.getName().empty()) + node["tag"] = it->second.getName(); + + if ((it->second.getLoadFee() != ref) && (it->second.getLoadFee() != 0)) + node["fee"] = static_cast(it->second.getLoadFee()) / ref; + + if (it->second.getReportTime() != 0) + node["age"] = (it->second.getReportTime() >= now) ? 0 : (now - it->second.getReportTime()); + } } } } - if (ShouldLog (lsTRACE, UniqueNodeList)) + //-------------------------------------------------------------------------- + + void nodeBootstrap () { - WriteLog (lsTRACE, UniqueNodeList) << "midway: "; - BOOST_FOREACH (scoreNode & sn, vsnNodes) + int iDomains = 0; + int iNodes = 0; + Database* db = getApp().getWalletDB ()->getDB (); + { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed - % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedDomains WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) + iDomains = db->getInt ("Count"); + + db->endIterRows (); + + if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedNodes WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) + iNodes = db->getInt ("Count"); + + db->endIterRows (); } - } - // Add roundScore to score. - // Make roundScore new roundSeed. - BOOST_FOREACH (scoreNode & sn, vsnNodes) - { - if (!bDist && sn.iRoundScore) - bDist = true; + bool bLoaded = iDomains || iNodes; - sn.iScore += sn.iRoundScore; - sn.iRoundSeed = sn.iRoundScore; - sn.iRoundScore = 0; - } - - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - WriteLog (lsTRACE, UniqueNodeList) << "finish: "; - BOOST_FOREACH (scoreNode & sn, vsnNodes) + // Always merge in the file specified in the config. + if (!theConfig.VALIDATORS_FILE.empty ()) { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed - % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); + WriteLog (lsINFO, UniqueNodeList) << "Bootstrapping UNL: loading from unl_default."; + + bLoaded = nodeLoad (theConfig.VALIDATORS_FILE); } - } - return bDist; -} - -// From SeedDomains and ValidatorReferrals compute scores and update TrustedNodes. -void UniqueNodeList::scoreCompute () -{ - strIndex umPulicIdx; // Map of public key to index. - strIndex umDomainIdx; // Map of domain to index. - std::vector vsnNodes; // Index to scoring node. - - Database* db = theApp->getWalletDB ()->getDB (); - - // For each entry in SeedDomains with a PublicKey: - // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. - { - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - SQL_FOREACH (db, "SELECT Domain,PublicKey,Source FROM SeedDomains;") + // If never loaded anything try the current directory. + if (!bLoaded && theConfig.VALIDATORS_FILE.empty ()) { - if (db->getNull ("PublicKey")) + WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") + % theConfig.VALIDATORS_BASE); + + bLoaded = nodeLoad (theConfig.VALIDATORS_BASE); + } + + // Always load from rippled.cfg + if (!theConfig.VALIDATORS.empty ()) + { + RippleAddress naInvalid; // Don't want a referrer on added entries. + + WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") + % theConfig.CONFIG_FILE); + + if (processValidators ("local", theConfig.CONFIG_FILE.string (), naInvalid, vsConfig, &theConfig.VALIDATORS)) + bLoaded = true; + } + + if (!bLoaded) + { + WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") + % theConfig.VALIDATORS_SITE); + + nodeNetwork (); + } + + if (!theConfig.IPS.empty ()) + { + std::vector vstrValues; + + vstrValues.reserve (theConfig.IPS.size ()); + + BOOST_FOREACH (const std::string & strPeer, theConfig.IPS) { - nothing (); // We ignore entries we don't have public keys for. + std::string strIP; + int iPort; + + if (parseIpPort (strPeer, strIP, iPort)) + { + vstrValues.push_back (str (boost::format ("(%s,'%c')") + % sqlEscape (str (boost::format ("%s %d") % strIP % iPort)) + % static_cast (vsConfig))); + } + } + + if (!vstrValues.empty ()) + { + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + db->executeSQL (str (boost::format ("REPLACE INTO PeerIps (IpPort,Source) VALUES %s;") + % strJoin (vstrValues.begin (), vstrValues.end (), ","))); + } + + fetchDirty (); + } + } + + //-------------------------------------------------------------------------- + + bool nodeLoad (boost::filesystem::path pConfig) + { + if (pConfig.empty ()) + { + WriteLog (lsINFO, UniqueNodeList) << VALIDATORS_FILE_NAME " path not specified."; + + return false; + } + + if (!boost::filesystem::exists (pConfig)) + { + WriteLog (lsWARNING, UniqueNodeList) << str (boost::format (VALIDATORS_FILE_NAME " not found: %s") % pConfig); + + return false; + } + + if (!boost::filesystem::is_regular_file (pConfig)) + { + WriteLog (lsWARNING, UniqueNodeList) << str (boost::format (VALIDATORS_FILE_NAME " not regular file: %s") % pConfig); + + return false; + } + + std::ifstream ifsDefault (pConfig.native ().c_str (), std::ios::in); + + if (!ifsDefault) + { + WriteLog (lsFATAL, UniqueNodeList) << str (boost::format (VALIDATORS_FILE_NAME " failed to open: %s") % pConfig); + + return false; + } + + std::string strValidators; + + strValidators.assign ((std::istreambuf_iterator (ifsDefault)), + std::istreambuf_iterator ()); + + if (ifsDefault.bad ()) + { + WriteLog (lsFATAL, UniqueNodeList) << str (boost::format ("Failed to read: %s") % pConfig); + + return false; + } + + nodeProcess ("local", strValidators, pConfig.string ()); + + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("Processing: %s") % pConfig); + + return true; + } + + //-------------------------------------------------------------------------- + + void nodeNetwork () + { + if (!theConfig.VALIDATORS_SITE.empty ()) + { + HttpsClient::httpsGet ( + true, + getApp().getIOService (), + theConfig.VALIDATORS_SITE, + 443, + theConfig.VALIDATORS_URI, + VALIDATORS_FILE_BYTES_MAX, + boost::posix_time::seconds (VALIDATORS_FETCH_SECONDS), + BIND_TYPE (&UniqueNodeListImp::validatorsResponse, this, P_1, P_2, P_3)); + } + } + + //-------------------------------------------------------------------------- + + Json::Value getUnlJson () + { + Database* db = getApp().getWalletDB ()->getDB (); + + Json::Value ret (Json::arrayValue); + + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + SQL_FOREACH (db, "SELECT * FROM TrustedNodes;") + { + Json::Value node (Json::objectValue); + + node["publicKey"] = db->getStrBinary ("PublicKey"); + node["comment"] = db->getStrBinary ("Comment"); + + ret.append (node); + } + + return ret; + } + + //-------------------------------------------------------------------------- + + // For each kind of source, have a starting number of points to be distributed. + int iSourceScore (ValidatorSource vsWhy) + { + int iScore = 0; + + switch (vsWhy) + { + case vsConfig: + iScore = 1500; + break; + + case vsInbound: + iScore = 0; + break; + + case vsManual: + iScore = 1500; + break; + + case vsReferral: + iScore = 0; + break; + + case vsTold: + iScore = 0; + break; + + case vsValidator: + iScore = 1000; + break; + + case vsWeb: + iScore = 200; + break; + + default: + throw std::runtime_error ("Internal error: bad ValidatorSource."); + } + + return iScore; + } + + //-------------------------------------------------------------------------- +private: + // Load information about when we last updated. + bool miscLoad () + { + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); + + if (!db->executeSQL ("SELECT * FROM Misc WHERE Magic=1;")) return false; + + bool bAvail = !!db->startIterRows (); + + mtpFetchUpdated = ptFromSeconds (bAvail ? db->getInt ("FetchUpdated") : -1); + mtpScoreUpdated = ptFromSeconds (bAvail ? db->getInt ("ScoreUpdated") : -1); + + db->endIterRows (); + + trustedLoad (); + + return true; + } + + //-------------------------------------------------------------------------- + + // Persist update information. + bool miscSave () + { + Database* db = getApp().getWalletDB ()->getDB (); + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + db->executeSQL (str (boost::format ("REPLACE INTO Misc (Magic,FetchUpdated,ScoreUpdated) VALUES (1,%d,%d);") + % iToSeconds (mtpFetchUpdated) + % iToSeconds (mtpScoreUpdated))); + + return true; + } + + //-------------------------------------------------------------------------- + + void trustedLoad () + { + boost::regex rNode ("\\`\\s*(\\S+)[\\s]*(.*)\\'"); + BOOST_FOREACH (const std::string & c, theConfig.CLUSTER_NODES) + { + boost::smatch match; + + if (boost::regex_match (c, match, rNode)) + { + RippleAddress a = RippleAddress::createNodePublic (match[1]); + + if (a.isValid ()) + m_clusterNodes.insert (std::make_pair (a, ClusterNodeStatus(match[2]))); } else + WriteLog (lsWARNING, UniqueNodeList) << "Entry in cluster list invalid: '" << c << "'"; + } + + Database* db = getApp().getWalletDB ()->getDB (); + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + boost::recursive_mutex::scoped_lock slUNL (mUNLLock); + + mUNL.clear (); + + // XXX Needs to limit by quanity and quality. + SQL_FOREACH (db, "SELECT PublicKey FROM TrustedNodes WHERE Score != 0;") + { + mUNL.insert (db->getStrBinary ("PublicKey")); + } + } + + //-------------------------------------------------------------------------- + + // For a round of scoring we destribute points from a node to nodes it refers to. + // Returns true, iff scores were distributed. + // + bool scoreRound (std::vector& vsnNodes) + { + bool bDist = false; + + // For each node, distribute roundSeed to roundScores. + BOOST_FOREACH (scoreNode & sn, vsnNodes) + { + int iEntries = sn.viReferrals.size (); + + if (sn.iRoundSeed && iEntries) + { + score iTotal = (iEntries + 1) * iEntries / 2; + score iBase = sn.iRoundSeed * iEntries / iTotal; + + // Distribute the current entires' seed score to validators prioritized by mention order. + for (int i = 0; i != iEntries; i++) + { + score iPoints = iBase * (iEntries - i) / iEntries; + + vsnNodes[sn.viReferrals[i]].iRoundScore += iPoints; + } + } + } + + if (ShouldLog (lsTRACE, UniqueNodeList)) + { + WriteLog (lsTRACE, UniqueNodeList) << "midway: "; + BOOST_FOREACH (scoreNode & sn, vsnNodes) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") + % sn.strValidator + % sn.iScore + % sn.iRoundScore + % sn.iRoundSeed + % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); + } + } + + // Add roundScore to score. + // Make roundScore new roundSeed. + BOOST_FOREACH (scoreNode & sn, vsnNodes) + { + if (!bDist && sn.iRoundScore) + bDist = true; + + sn.iScore += sn.iRoundScore; + sn.iRoundSeed = sn.iRoundScore; + sn.iRoundScore = 0; + } + + if (ShouldLog (lsTRACE, UniqueNodeList)) + { + WriteLog (lsTRACE, UniqueNodeList) << "finish: "; + BOOST_FOREACH (scoreNode & sn, vsnNodes) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") + % sn.strValidator + % sn.iScore + % sn.iRoundScore + % sn.iRoundSeed + % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); + } + } + + return bDist; + } + + //-------------------------------------------------------------------------- + + // From SeedDomains and ValidatorReferrals compute scores and update TrustedNodes. + // + // VFALCO TODO Shrink this function, break it up + // + void scoreCompute () + { + strIndex umPulicIdx; // Map of public key to index. + strIndex umDomainIdx; // Map of domain to index. + std::vector vsnNodes; // Index to scoring node. + + Database* db = getApp().getWalletDB ()->getDB (); + + // For each entry in SeedDomains with a PublicKey: + // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. + { + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + SQL_FOREACH (db, "SELECT Domain,PublicKey,Source FROM SeedDomains;") + { + if (db->getNull ("PublicKey")) + { + nothing (); // We ignore entries we don't have public keys for. + } + else + { + std::string strDomain = db->getStrBinary ("Domain"); + std::string strPublicKey = db->getStrBinary ("PublicKey"); + std::string strSource = db->getStrBinary ("Source"); + int iScore = iSourceScore (static_cast (strSource[0])); + strIndex::iterator siOld = umPulicIdx.find (strPublicKey); + + if (siOld == umPulicIdx.end ()) + { + // New node + int iNode = vsnNodes.size (); + + umPulicIdx[strPublicKey] = iNode; + umDomainIdx[strDomain] = iNode; + + scoreNode snCurrent; + + snCurrent.strValidator = strPublicKey; + snCurrent.iScore = iScore; + snCurrent.iRoundSeed = snCurrent.iScore; + snCurrent.iRoundScore = 0; + snCurrent.iSeen = -1; + + vsnNodes.push_back (snCurrent); + } + else + { + scoreNode& snOld = vsnNodes[siOld->second]; + + if (snOld.iScore < iScore) + { + // Update old node + + snOld.iScore = iScore; + snOld.iRoundSeed = snOld.iScore; + } + } + } + } + } + + // For each entry in SeedNodes: + // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. + { + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + SQL_FOREACH (db, "SELECT PublicKey,Source FROM SeedNodes;") { - std::string strDomain = db->getStrBinary ("Domain"); std::string strPublicKey = db->getStrBinary ("PublicKey"); std::string strSource = db->getStrBinary ("Source"); - int iScore = iSourceScore (static_cast (strSource[0])); + int iScore = iSourceScore (static_cast (strSource[0])); strIndex::iterator siOld = umPulicIdx.find (strPublicKey); if (siOld == umPulicIdx.end ()) @@ -356,7 +859,6 @@ void UniqueNodeList::scoreCompute () int iNode = vsnNodes.size (); umPulicIdx[strPublicKey] = iNode; - umDomainIdx[strDomain] = iNode; scoreNode snCurrent; @@ -382,1545 +884,1180 @@ void UniqueNodeList::scoreCompute () } } } - } - // For each entry in SeedNodes: - // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. - { - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - SQL_FOREACH (db, "SELECT PublicKey,Source FROM SeedNodes;") + // For debugging, print out initial scores. + if (ShouldLog (lsTRACE, UniqueNodeList)) { - std::string strPublicKey = db->getStrBinary ("PublicKey"); - std::string strSource = db->getStrBinary ("Source"); - int iScore = iSourceScore (static_cast (strSource[0])); - strIndex::iterator siOld = umPulicIdx.find (strPublicKey); - - if (siOld == umPulicIdx.end ()) + BOOST_FOREACH (scoreNode & sn, vsnNodes) { - // New node - int iNode = vsnNodes.size (); - - umPulicIdx[strPublicKey] = iNode; - - scoreNode snCurrent; - - snCurrent.strValidator = strPublicKey; - snCurrent.iScore = iScore; - snCurrent.iRoundSeed = snCurrent.iScore; - snCurrent.iRoundScore = 0; - snCurrent.iSeen = -1; - - vsnNodes.push_back (snCurrent); - } - else - { - scoreNode& snOld = vsnNodes[siOld->second]; - - if (snOld.iScore < iScore) - { - // Update old node - - snOld.iScore = iScore; - snOld.iRoundSeed = snOld.iScore; - } + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d") + % sn.strValidator + % sn.iScore + % sn.iRoundScore + % sn.iRoundSeed); } } - } - // For debugging, print out initial scores. - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - BOOST_FOREACH (scoreNode & sn, vsnNodes) + // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("vsnNodes.size=%d") % vsnNodes.size()); + + // Step through growing list of nodes adding each validation list. + // - Each validator may have provided referals. Add those referals as validators. + for (int iNode = 0; iNode != vsnNodes.size (); ++iNode) { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed); - } - } + scoreNode& sn = vsnNodes[iNode]; + std::string& strValidator = sn.strValidator; + std::vector& viReferrals = sn.viReferrals; - // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("vsnNodes.size=%d") % vsnNodes.size()); + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); - // Step through growing list of nodes adding each validation list. - // - Each validator may have provided referals. Add those referals as validators. - for (int iNode = 0; iNode != vsnNodes.size (); ++iNode) - { - scoreNode& sn = vsnNodes[iNode]; - std::string& strValidator = sn.strValidator; - std::vector& viReferrals = sn.viReferrals; - - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - SQL_FOREACH (db, boost::str (boost::format ("SELECT Referral FROM ValidatorReferrals WHERE Validator=%s ORDER BY Entry;") - % sqlEscape (strValidator))) - { - std::string strReferral = db->getStrBinary ("Referral"); - int iReferral; - - strIndex::iterator itEntry; - - RippleAddress na; - - if (na.setNodePublic (strReferral)) + SQL_FOREACH (db, boost::str (boost::format ("SELECT Referral FROM ValidatorReferrals WHERE Validator=%s ORDER BY Entry;") + % sqlEscape (strValidator))) { - // Referring a public key. - itEntry = umPulicIdx.find (strReferral); + std::string strReferral = db->getStrBinary ("Referral"); + int iReferral; - if (itEntry == umPulicIdx.end ()) + strIndex::iterator itEntry; + + RippleAddress na; + + if (na.setNodePublic (strReferral)) { - // Not found add public key to list of nodes. - iReferral = vsnNodes.size (); + // Referring a public key. + itEntry = umPulicIdx.find (strReferral); - umPulicIdx[strReferral] = iReferral; + if (itEntry == umPulicIdx.end ()) + { + // Not found add public key to list of nodes. + iReferral = vsnNodes.size (); - scoreNode snCurrent; + umPulicIdx[strReferral] = iReferral; - snCurrent.strValidator = strReferral; - snCurrent.iScore = iSourceScore (vsReferral); - snCurrent.iRoundSeed = snCurrent.iScore; - snCurrent.iRoundScore = 0; - snCurrent.iSeen = -1; + scoreNode snCurrent; + + snCurrent.strValidator = strReferral; + snCurrent.iScore = iSourceScore (vsReferral); + snCurrent.iRoundSeed = snCurrent.iScore; + snCurrent.iRoundScore = 0; + snCurrent.iSeen = -1; + + vsnNodes.push_back (snCurrent); + } + else + { + iReferral = itEntry->second; + } + + // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("%s: Public=%s iReferral=%d") % strValidator % strReferral % iReferral); - vsnNodes.push_back (snCurrent); } else { - iReferral = itEntry->second; + // Referring a domain. + itEntry = umDomainIdx.find (strReferral); + iReferral = itEntry == umDomainIdx.end () + ? -1 // We ignore domains we can't find entires for. + : itEntry->second; + + // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("%s: Domain=%s iReferral=%d") % strValidator % strReferral % iReferral); } - // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("%s: Public=%s iReferral=%d") % strValidator % strReferral % iReferral); - + if (iReferral >= 0 && iNode != iReferral) + viReferrals.push_back (iReferral); } - else + } + + // + // Distribute the points from the seeds. + // + bool bDist = true; + + for (int i = SCORE_ROUNDS; bDist && i--;) + bDist = scoreRound (vsnNodes); + + if (ShouldLog (lsTRACE, UniqueNodeList)) + { + WriteLog (lsTRACE, UniqueNodeList) << "Scored:"; + BOOST_FOREACH (scoreNode & sn, vsnNodes) { - // Referring a domain. - itEntry = umDomainIdx.find (strReferral); - iReferral = itEntry == umDomainIdx.end () - ? -1 // We ignore domains we can't find entires for. - : itEntry->second; + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") + % sn.strValidator + % sn.iScore + % sn.iRoundScore + % sn.iRoundSeed + % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); + } + } - // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("%s: Domain=%s iReferral=%d") % strValidator % strReferral % iReferral); + // Persist validator scores. + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + db->executeSQL ("BEGIN;"); + db->executeSQL ("UPDATE TrustedNodes SET Score = 0 WHERE Score != 0;"); + + if (!vsnNodes.empty ()) + { + // Load existing Seens from DB. + std::vector vstrPublicKeys; + + vstrPublicKeys.resize (vsnNodes.size ()); + + for (int iNode = vsnNodes.size (); iNode--;) + { + vstrPublicKeys[iNode] = sqlEscape (vsnNodes[iNode].strValidator); } - if (iReferral >= 0 && iNode != iReferral) - viReferrals.push_back (iReferral); + SQL_FOREACH (db, str (boost::format ("SELECT PublicKey,Seen FROM TrustedNodes WHERE PublicKey IN (%s);") + % strJoin (vstrPublicKeys.begin (), vstrPublicKeys.end (), ","))) + { + vsnNodes[umPulicIdx[db->getStrBinary ("PublicKey")]].iSeen = db->getNull ("Seen") ? -1 : db->getInt ("Seen"); + } + } + + boost::unordered_set usUNL; + + if (!vsnNodes.empty ()) + { + // Update the score old entries and add new entries as needed. + std::vector vstrValues; + + vstrValues.resize (vsnNodes.size ()); + + for (int iNode = vsnNodes.size (); iNode--;) + { + scoreNode& sn = vsnNodes[iNode]; + std::string strSeen = sn.iSeen >= 0 ? str (boost::format ("%d") % sn.iSeen) : "NULL"; + + vstrValues[iNode] = str (boost::format ("(%s,%s,%s)") + % sqlEscape (sn.strValidator) + % sn.iScore + % strSeen); + + usUNL.insert (sn.strValidator); + } + + db->executeSQL (str (boost::format ("REPLACE INTO TrustedNodes (PublicKey,Score,Seen) VALUES %s;") + % strJoin (vstrValues.begin (), vstrValues.end (), ","))); + } + + { + boost::recursive_mutex::scoped_lock sl (mUNLLock); + + // XXX Should limit to scores above a certain minimum and limit to a certain number. + mUNL.swap (usUNL); + } + + // Score IPs. + db->executeSQL ("UPDATE PeerIps SET Score = 0 WHERE Score != 0;"); + + boost::unordered_map umValidators; + + if (!vsnNodes.empty ()) + { + std::vector vstrPublicKeys; + + // For every IpReferral add a score for the IP and PORT. + SQL_FOREACH (db, "SELECT Validator,COUNT(*) AS Count FROM IpReferrals GROUP BY Validator;") + { + umValidators[db->getStrBinary ("Validator")] = db->getInt ("Count"); + + // WriteLog (lsTRACE, UniqueNodeList) << strValidator << ":" << db->getInt("Count"); + } + } + + // For each validator, get each referral and add its score to ip's score. + // map of pair :: score + epScore umScore; + + typedef boost::unordered_map::value_type vcType; + BOOST_FOREACH (vcType & vc, umValidators) + { + std::string strValidator = vc.first; + + strIndex::iterator itIndex = umPulicIdx.find (strValidator); + + if (itIndex != umPulicIdx.end ()) + { + int iSeed = vsnNodes[itIndex->second].iScore; + int iEntries = vc.second; + score iTotal = (iEntries + 1) * iEntries / 2; + score iBase = iSeed * iEntries / iTotal; + int iEntry = 0; + + SQL_FOREACH (db, str (boost::format ("SELECT IP,Port FROM IpReferrals WHERE Validator=%s ORDER BY Entry;") + % sqlEscape (strValidator))) + { + score iPoints = iBase * (iEntries - iEntry) / iEntries; + int iPort; + + iPort = db->getNull ("Port") ? -1 : db->getInt ("Port"); + + std::pair< std::string, int> ep = std::make_pair (db->getStrBinary ("IP"), iPort); + + epScore::iterator itEp = umScore.find (ep); + + umScore[ep] = itEp == umScore.end () ? iPoints : itEp->second + iPoints; + iEntry++; + } + } + } + + // Apply validator scores to each IP. + if (umScore.size ()) + { + std::vector vstrValues; + + vstrValues.reserve (umScore.size ()); + + typedef boost::unordered_map, score>::value_type ipScoreType; + BOOST_FOREACH (ipScoreType & ipScore, umScore) + { + IPAndPortNumber ipEndpoint = ipScore.first; + std::string strIpPort = str (boost::format ("%s %d") % ipEndpoint.first % ipEndpoint.second); + score iPoints = ipScore.second; + + vstrValues.push_back (str (boost::format ("(%s,%d,'%c')") + % sqlEscape (strIpPort) + % iPoints + % vsValidator)); + } + + // Set scores for each IP. + db->executeSQL (str (boost::format ("REPLACE INTO PeerIps (IpPort,Score,Source) VALUES %s;") + % strJoin (vstrValues.begin (), vstrValues.end (), ","))); + } + + db->executeSQL ("COMMIT;"); + } + + //-------------------------------------------------------------------------- + + // Begin scoring if timer was not cancelled. + void scoreTimerHandler (const boost::system::error_code& err) + { + if (!err) + { + onDeadlineTimer (m_scoreTimer); } } + //-------------------------------------------------------------------------- + + // Start a timer to update scores. + // <-- bNow: true, to force scoring for debugging. + void scoreNext (bool bNow) + { + // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("scoreNext: mtpFetchUpdated=%s mtpScoreStart=%s mtpScoreUpdated=%s mtpScoreNext=%s") % mtpFetchUpdated % mtpScoreStart % mtpScoreUpdated % mtpScoreNext); + bool bCanScore = mtpScoreStart.is_not_a_date_time () // Not scoring. + && !mtpFetchUpdated.is_not_a_date_time (); // Something to score. + + bool bDirty = + (mtpScoreUpdated.is_not_a_date_time () || mtpScoreUpdated <= mtpFetchUpdated) // Not already scored. + && (mtpScoreNext.is_not_a_date_time () // Timer is not fine. + || mtpScoreNext < mtpFetchUpdated + boost::posix_time::seconds (SCORE_DELAY_SECONDS)); + + if (!bCanScore) + { + nothing (); + } + else if (bNow || bDirty) + { + // Need to update or set timer. + double const secondsFromNow = bNow ? 0 : SCORE_DELAY_SECONDS; + mtpScoreNext = boost::posix_time::second_clock::universal_time () // Past now too. + + boost::posix_time::seconds (secondsFromNow); + + // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("scoreNext: @%s") % mtpScoreNext); + m_scoreTimer.setExpiration (secondsFromNow); + } + } + + //-------------------------------------------------------------------------- + + // Given a ripple.txt, process it. // - // Distribute the points from the seeds. + // VFALCO TODO Can't we take a filename or stream instead of a string? // - bool bDist = true; - - for (int i = SCORE_ROUNDS; bDist && i--;) - bDist = scoreRound (vsnNodes); - - if (ShouldLog (lsTRACE, UniqueNodeList)) + bool responseFetch (const std::string& strDomain, const boost::system::error_code& err, int iStatus, const std::string& strSiteFile) { - WriteLog (lsTRACE, UniqueNodeList) << "Scored:"; - BOOST_FOREACH (scoreNode & sn, vsnNodes) + bool bReject = !err && iStatus != 200; + + if (!bReject) { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed - % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); - } - } + Section secSite = ParseSection (strSiteFile, true); + bool bGood = !err; - // Persist validator scores. - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - db->executeSQL ("BEGIN;"); - db->executeSQL ("UPDATE TrustedNodes SET Score = 0 WHERE Score != 0;"); - - if (!vsnNodes.empty ()) - { - // Load existing Seens from DB. - std::vector vstrPublicKeys; - - vstrPublicKeys.resize (vsnNodes.size ()); - - for (int iNode = vsnNodes.size (); iNode--;) - { - vstrPublicKeys[iNode] = sqlEscape (vsnNodes[iNode].strValidator); - } - - SQL_FOREACH (db, str (boost::format ("SELECT PublicKey,Seen FROM TrustedNodes WHERE PublicKey IN (%s);") - % strJoin (vstrPublicKeys.begin (), vstrPublicKeys.end (), ","))) - { - vsnNodes[umPulicIdx[db->getStrBinary ("PublicKey")]].iSeen = db->getNull ("Seen") ? -1 : db->getInt ("Seen"); - } - } - - boost::unordered_set usUNL; - - if (!vsnNodes.empty ()) - { - // Update the score old entries and add new entries as needed. - std::vector vstrValues; - - vstrValues.resize (vsnNodes.size ()); - - for (int iNode = vsnNodes.size (); iNode--;) - { - scoreNode& sn = vsnNodes[iNode]; - std::string strSeen = sn.iSeen >= 0 ? str (boost::format ("%d") % sn.iSeen) : "NULL"; - - vstrValues[iNode] = str (boost::format ("(%s,%s,%s)") - % sqlEscape (sn.strValidator) - % sn.iScore - % strSeen); - - usUNL.insert (sn.strValidator); - } - - db->executeSQL (str (boost::format ("REPLACE INTO TrustedNodes (PublicKey,Score,Seen) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ","))); - } - - { - boost::recursive_mutex::scoped_lock sl (mUNLLock); - - // XXX Should limit to scores above a certain minimum and limit to a certain number. - mUNL.swap (usUNL); - } - - // Score IPs. - db->executeSQL ("UPDATE PeerIps SET Score = 0 WHERE Score != 0;"); - - boost::unordered_map umValidators; - - if (!vsnNodes.empty ()) - { - std::vector vstrPublicKeys; - - // For every IpReferral add a score for the IP and PORT. - SQL_FOREACH (db, "SELECT Validator,COUNT(*) AS Count FROM IpReferrals GROUP BY Validator;") - { - umValidators[db->getStrBinary ("Validator")] = db->getInt ("Count"); - - // WriteLog (lsTRACE, UniqueNodeList) << strValidator << ":" << db->getInt("Count"); - } - } - - // For each validator, get each referral and add its score to ip's score. - // map of pair :: score - epScore umScore; - - typedef boost::unordered_map::value_type vcType; - BOOST_FOREACH (vcType & vc, umValidators) - { - std::string strValidator = vc.first; - - strIndex::iterator itIndex = umPulicIdx.find (strValidator); - - if (itIndex != umPulicIdx.end ()) - { - int iSeed = vsnNodes[itIndex->second].iScore; - int iEntries = vc.second; - score iTotal = (iEntries + 1) * iEntries / 2; - score iBase = iSeed * iEntries / iTotal; - int iEntry = 0; - - SQL_FOREACH (db, str (boost::format ("SELECT IP,Port FROM IpReferrals WHERE Validator=%s ORDER BY Entry;") - % sqlEscape (strValidator))) + if (bGood) { - score iPoints = iBase * (iEntries - iEntry) / iEntries; - int iPort; - - iPort = db->getNull ("Port") ? -1 : db->getInt ("Port"); - - std::pair< std::string, int> ep = std::make_pair (db->getStrBinary ("IP"), iPort); - - epScore::iterator itEp = umScore.find (ep); - - umScore[ep] = itEp == umScore.end () ? iPoints : itEp->second + iPoints; - iEntry++; - } - } - } - - // Apply validator scores to each IP. - if (umScore.size ()) - { - std::vector vstrValues; - - vstrValues.reserve (umScore.size ()); - - typedef boost::unordered_map, score>::value_type ipScoreType; - BOOST_FOREACH (ipScoreType & ipScore, umScore) - { - ipPort ipEndpoint = ipScore.first; - std::string strIpPort = str (boost::format ("%s %d") % ipEndpoint.first % ipEndpoint.second); - score iPoints = ipScore.second; - - vstrValues.push_back (str (boost::format ("(%s,%d,'%c')") - % sqlEscape (strIpPort) - % iPoints - % vsValidator)); - } - - // Set scores for each IP. - db->executeSQL (str (boost::format ("REPLACE INTO PeerIps (IpPort,Score,Source) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ","))); - } - - db->executeSQL ("COMMIT;"); -} - -// Begin scoring if timer was not cancelled. -void UniqueNodeList::scoreTimerHandler (const boost::system::error_code& err) -{ - if (!err) - { - mtpScoreNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Timer not set. - mtpScoreStart = boost::posix_time::second_clock::universal_time (); // Scoring. - - WriteLog (lsTRACE, UniqueNodeList) << "Scoring: Start"; - - scoreCompute (); - - WriteLog (lsTRACE, UniqueNodeList) << "Scoring: End"; - - // Save update time. - mtpScoreUpdated = mtpScoreStart; - miscSave (); - - mtpScoreStart = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Not scoring. - - // Score again if needed. - scoreNext (false); - - // Scan may be dirty due to new ips. - theApp->getPeers ().scanRefresh (); - } -} - -// Start a timer to update scores. -// <-- bNow: true, to force scoring for debugging. -void UniqueNodeList::scoreNext (bool bNow) -{ - // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("scoreNext: mtpFetchUpdated=%s mtpScoreStart=%s mtpScoreUpdated=%s mtpScoreNext=%s") % mtpFetchUpdated % mtpScoreStart % mtpScoreUpdated % mtpScoreNext); - bool bCanScore = mtpScoreStart.is_not_a_date_time () // Not scoring. - && !mtpFetchUpdated.is_not_a_date_time (); // Something to score. - - bool bDirty = - (mtpScoreUpdated.is_not_a_date_time () || mtpScoreUpdated <= mtpFetchUpdated) // Not already scored. - && (mtpScoreNext.is_not_a_date_time () // Timer is not fine. - || mtpScoreNext < mtpFetchUpdated + boost::posix_time::seconds (SCORE_DELAY_SECONDS)); - - if (!bCanScore) - { - nothing (); - } - else if (bNow || bDirty) - { - // Need to update or set timer. - mtpScoreNext = boost::posix_time::second_clock::universal_time () // Past now too. - + boost::posix_time::seconds (bNow ? 0 : SCORE_DELAY_SECONDS); - - // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("scoreNext: @%s") % mtpScoreNext); - mdtScoreTimer.expires_at (mtpScoreNext); - mdtScoreTimer.async_wait (boost::bind (&UniqueNodeList::scoreTimerHandler, this, _1)); - } -} - -// For debugging, schedule forced scoring. -void UniqueNodeList::nodeScore () -{ - scoreNext (true); -} - -void UniqueNodeList::fetchFinish () -{ - { - boost::mutex::scoped_lock sl (mFetchLock); - mFetchActive--; - } - - fetchNext (); -} - -// Called when we need to update scores. -void UniqueNodeList::fetchDirty () -{ - // Note update. - mtpFetchUpdated = boost::posix_time::second_clock::universal_time (); - miscSave (); - - // Update scores. - scoreNext (false); -} - -// Persist the IPs refered to by a Validator. -// --> strSite: source of the IPs (for debugging) -// --> naNodePublic: public key of the validating node. -void UniqueNodeList::processIps (const std::string& strSite, const RippleAddress& naNodePublic, Section::mapped_type* pmtVecStrIps) -{ - Database* db = theApp->getWalletDB ()->getDB (); - - std::string strEscNodePublic = sqlEscape (naNodePublic.humanNodePublic ()); - - WriteLog (lsDEBUG, UniqueNodeList) - << str (boost::format ("Validator: '%s' processing %d ips.") - % strSite % ( pmtVecStrIps ? pmtVecStrIps->size () : 0)); - - // Remove all current Validator's entries in IpReferrals - { - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - db->executeSQL (str (boost::format ("DELETE FROM IpReferrals WHERE Validator=%s;") % strEscNodePublic)); - // XXX Check result. - } - - // Add new referral entries. - if (pmtVecStrIps && !pmtVecStrIps->empty ()) - { - std::vector vstrValues; - - vstrValues.resize (std::min ((int) pmtVecStrIps->size (), REFERRAL_IPS_MAX)); - - int iValues = 0; - BOOST_FOREACH (const std::string & strReferral, *pmtVecStrIps) - { - if (iValues == REFERRAL_VALIDATORS_MAX) - break; - - std::string strIP; - int iPort; - bool bValid = parseIpPort (strReferral, strIP, iPort); - - // XXX Filter out private network ips. - // XXX http://en.wikipedia.org/wiki/Private_network - - if (bValid) - { - vstrValues[iValues] = str (boost::format ("(%s,%d,%s,%d)") - % strEscNodePublic % iValues % sqlEscape (strIP) % iPort); - iValues++; + WriteLog (lsTRACE, UniqueNodeList) << boost::format ("Validator: '%s' received " NODE_FILE_NAME ".") % strDomain; } else { WriteLog (lsTRACE, UniqueNodeList) - << str (boost::format ("Validator: '%s' [" SECTION_IPS "]: rejecting '%s'") - % strSite % strReferral); + << boost::format ("Validator: '%s' unable to retrieve " NODE_FILE_NAME ": %s") + % strDomain + % err.message (); } - } - if (iValues) - { - vstrValues.resize (iValues); + // + // Verify file domain + // + std::string strSite; - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - db->executeSQL (str (boost::format ("INSERT INTO IpReferrals (Validator,Entry,IP,Port) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ","))); - // XXX Check result. - } - } - - fetchDirty (); -} - -// Persist ValidatorReferrals. -// --> strSite: source site for display -// --> strValidatorsSrc: source details for display -// --> naNodePublic: remote source public key - not valid for local -// --> vsWhy: reason for adding validator to SeedDomains or SeedNodes. -int UniqueNodeList::processValidators (const std::string& strSite, const std::string& strValidatorsSrc, const RippleAddress& naNodePublic, validatorSource vsWhy, Section::mapped_type* pmtVecStrValidators) -{ - Database* db = theApp->getWalletDB ()->getDB (); - std::string strNodePublic = naNodePublic.isValid () ? naNodePublic.humanNodePublic () : strValidatorsSrc; - int iValues = 0; - - WriteLog (lsTRACE, UniqueNodeList) - << str (boost::format ("Validator: '%s' : '%s' : processing %d validators.") - % strSite - % strValidatorsSrc - % ( pmtVecStrValidators ? pmtVecStrValidators->size () : 0)); - - // Remove all current Validator's entries in ValidatorReferrals - { - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - db->executeSQL (str (boost::format ("DELETE FROM ValidatorReferrals WHERE Validator='%s';") % strNodePublic)); - // XXX Check result. - } - - // Add new referral entries. - if (pmtVecStrValidators && pmtVecStrValidators->size ()) - { - std::vector vstrValues; - - vstrValues.reserve (std::min ((int) pmtVecStrValidators->size (), REFERRAL_VALIDATORS_MAX)); - - BOOST_FOREACH (const std::string & strReferral, *pmtVecStrValidators) - { - if (iValues == REFERRAL_VALIDATORS_MAX) - break; - - boost::smatch smMatch; - - // domain comment? - // public_key comment? - static boost::regex reReferral ("\\`\\s*(\\S+)(?:\\s+(.+))?\\s*\\'"); - - if (!boost::regex_match (strReferral, smMatch, reReferral)) + if (bGood && !SectionSingleB (secSite, SECTION_DOMAIN, strSite)) { - WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: syntax error: %s: %s") % strSite % strReferral); + bGood = false; + + WriteLog (lsTRACE, UniqueNodeList) + << boost::format ("Validator: '%s' bad " NODE_FILE_NAME " missing single entry for " SECTION_DOMAIN ".") + % strDomain; } - else + + if (bGood && strSite != strDomain) { - std::string strRefered = smMatch[1]; - std::string strComment = smMatch[2]; - RippleAddress naValidator; + bGood = false; - if (naValidator.setSeedGeneric (strRefered)) + WriteLog (lsTRACE, UniqueNodeList) + << boost::format ("Validator: '%s' bad " NODE_FILE_NAME " " SECTION_DOMAIN " does not match: %s") + % strDomain + % strSite; + } + + // + // Process public key + // + std::string strNodePublicKey; + + if (bGood && !SectionSingleB (secSite, SECTION_PUBLIC_KEY, strNodePublicKey)) + { + // Bad [validation_public_key] Section. + bGood = false; + + WriteLog (lsTRACE, UniqueNodeList) + << boost::format ("Validator: '%s' bad " NODE_FILE_NAME " " SECTION_PUBLIC_KEY " does not have single entry.") + % strDomain; + } + + RippleAddress naNodePublic; + + if (bGood && !naNodePublic.setNodePublic (strNodePublicKey)) + { + // Bad public key. + bGood = false; + + WriteLog (lsTRACE, UniqueNodeList) + << boost::format ("Validator: '%s' bad " NODE_FILE_NAME " " SECTION_PUBLIC_KEY " is bad: ") + % strDomain + % strNodePublicKey; + } + + if (bGood) + { + // WriteLog (lsTRACE, UniqueNodeList) << boost::format("naNodePublic: '%s'") % naNodePublic.humanNodePublic(); + + seedDomain sdCurrent; + + bool bFound = getSeedDomains (strDomain, sdCurrent); + + assert (bFound); + + uint256 iSha256 = Serializer::getSHA512Half (strSiteFile); + bool bChangedB = sdCurrent.iSha256 != iSha256; + + sdCurrent.strDomain = strDomain; + // XXX If the node public key is changing, delete old public key information? + // XXX Only if no other refs to keep it arround, other wise we have an attack vector. + sdCurrent.naPublicKey = naNodePublic; + + // WriteLog (lsTRACE, UniqueNodeList) << boost::format("sdCurrent.naPublicKey: '%s'") % sdCurrent.naPublicKey.humanNodePublic(); + + sdCurrent.tpFetch = boost::posix_time::second_clock::universal_time (); + sdCurrent.iSha256 = iSha256; + + setSeedDomains (sdCurrent, true); + + if (bChangedB) { - - WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: domain or public key required: %s %s") % strRefered % strComment); - } - else if (naValidator.setNodePublic (strRefered)) - { - // A public key. - // XXX Schedule for CAS lookup. - nodeAddPublic (naValidator, vsWhy, strComment); - - WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Public: %s %s") % strRefered % strComment); - - if (naNodePublic.isValid ()) - vstrValues.push_back (str (boost::format ("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic ())); - - iValues++; + WriteLog (lsTRACE, UniqueNodeList) << boost::format ("Validator: '%s' processing new " NODE_FILE_NAME ".") % strDomain; + processFile (strDomain, naNodePublic, secSite); } else { - // A domain: need to look it up. - nodeAddDomain (strRefered, vsWhy, strComment); - - WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Domain: %s %s") % strRefered % strComment); - - if (naNodePublic.isValid ()) - vstrValues.push_back (str (boost::format ("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape (strRefered))); - - iValues++; + WriteLog (lsTRACE, UniqueNodeList) << boost::format ("Validator: '%s' no change for " NODE_FILE_NAME ".") % strDomain; + fetchFinish (); } } - } - - if (!vstrValues.empty ()) - { - std::string strSql = str (boost::format ("INSERT INTO ValidatorReferrals (Validator,Entry,Referral) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ",")); - - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - db->executeSQL (strSql); - // XXX Check result. - } - } - - fetchDirty (); - - return iValues; -} - -// Given a Section with IPs, parse and persist it for a validator. -bool UniqueNodeList::responseIps (const std::string& strSite, const RippleAddress& naNodePublic, const boost::system::error_code& err, int iStatus, const std::string& strIpsFile) -{ - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - if (!err) - { - Section secFile = ParseSection (strIpsFile, true); - - processIps (strSite, naNodePublic, SectionEntries (secFile, SECTION_IPS)); - } - - fetchFinish (); - } - - return bReject; -} - -// Process Section [ips_url]. -// If we have a Section with a single entry, fetch the url and process it. -void UniqueNodeList::getIpsUrl (const RippleAddress& naNodePublic, Section secSite) -{ - std::string strIpsUrl; - std::string strScheme; - std::string strDomain; - int iPort; - std::string strPath; - - if (SectionSingleB (secSite, SECTION_IPS_URL, strIpsUrl) - && !strIpsUrl.empty () - && parseUrl (strIpsUrl, strScheme, strDomain, iPort, strPath) - && -1 == iPort - && strScheme == "https") - { - HttpsClient::httpsGet ( - true, - theApp->getIOService (), - strDomain, - 443, - strPath, - NODE_FILE_BYTES_MAX, - boost::posix_time::seconds (NODE_FETCH_SECONDS), - boost::bind (&UniqueNodeList::responseIps, this, strDomain, naNodePublic, _1, _2, _3)); - } - else - { - fetchFinish (); - } -} - -// After fetching a ripple.txt from a web site, given a Section with validators, parse and persist it. -bool UniqueNodeList::responseValidators (const std::string& strValidatorsUrl, const RippleAddress& naNodePublic, Section secSite, const std::string& strSite, const boost::system::error_code& err, int iStatus, const std::string& strValidatorsFile) -{ - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - if (!err) - { - Section secFile = ParseSection (strValidatorsFile, true); - - processValidators (strSite, strValidatorsUrl, naNodePublic, vsValidator, SectionEntries (secFile, SECTION_VALIDATORS)); - } - - getIpsUrl (naNodePublic, secSite); - } - - return bReject; -} - -// Process Section [validators_url]. -void UniqueNodeList::getValidatorsUrl (const RippleAddress& naNodePublic, Section secSite) -{ - std::string strValidatorsUrl; - std::string strScheme; - std::string strDomain; - int iPort; - std::string strPath; - - if (SectionSingleB (secSite, SECTION_VALIDATORS_URL, strValidatorsUrl) - && !strValidatorsUrl.empty () - && parseUrl (strValidatorsUrl, strScheme, strDomain, iPort, strPath) - && -1 == iPort - && strScheme == "https") - { - HttpsClient::httpsGet ( - true, - theApp->getIOService (), - strDomain, - 443, - strPath, - NODE_FILE_BYTES_MAX, - boost::posix_time::seconds (NODE_FETCH_SECONDS), - BIND_TYPE (&UniqueNodeList::responseValidators, this, strValidatorsUrl, naNodePublic, secSite, strDomain, P_1, P_2, P_3)); - } - else - { - getIpsUrl (naNodePublic, secSite); - } -} - -// Process a ripple.txt. -void UniqueNodeList::processFile (const std::string& strDomain, const RippleAddress& naNodePublic, Section secSite) -{ - // - // Process Validators - // - processValidators (strDomain, NODE_FILE_NAME, naNodePublic, vsReferral, SectionEntries (secSite, SECTION_VALIDATORS)); - - // - // Process ips - // - processIps (strDomain, naNodePublic, SectionEntries (secSite, SECTION_IPS)); - - // - // Process currencies - // - Section::mapped_type* pvCurrencies; - - if ((pvCurrencies = SectionEntries (secSite, SECTION_CURRENCIES)) && pvCurrencies->size ()) - { - // XXX Process currencies. - WriteLog (lsWARNING, UniqueNodeList) << "Ignoring currencies: not implemented."; - } - - getValidatorsUrl (naNodePublic, secSite); -} - -// Given a ripple.txt, process it. -bool UniqueNodeList::responseFetch (const std::string& strDomain, const boost::system::error_code& err, int iStatus, const std::string& strSiteFile) -{ - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - Section secSite = ParseSection (strSiteFile, true); - bool bGood = !err; - - if (bGood) - { - WriteLog (lsTRACE, UniqueNodeList) << boost::format ("Validator: '%s' received " NODE_FILE_NAME ".") % strDomain; - } - else - { - WriteLog (lsTRACE, UniqueNodeList) - << boost::format ("Validator: '%s' unable to retrieve " NODE_FILE_NAME ": %s") - % strDomain - % err.message (); - } - - // - // Verify file domain - // - std::string strSite; - - if (bGood && !SectionSingleB (secSite, SECTION_DOMAIN, strSite)) - { - bGood = false; - - WriteLog (lsTRACE, UniqueNodeList) - << boost::format ("Validator: '%s' bad " NODE_FILE_NAME " missing single entry for " SECTION_DOMAIN ".") - % strDomain; - } - - if (bGood && strSite != strDomain) - { - bGood = false; - - WriteLog (lsTRACE, UniqueNodeList) - << boost::format ("Validator: '%s' bad " NODE_FILE_NAME " " SECTION_DOMAIN " does not match: %s") - % strDomain - % strSite; - } - - // - // Process public key - // - std::string strNodePublicKey; - - if (bGood && !SectionSingleB (secSite, SECTION_PUBLIC_KEY, strNodePublicKey)) - { - // Bad [validation_public_key] Section. - bGood = false; - - WriteLog (lsTRACE, UniqueNodeList) - << boost::format ("Validator: '%s' bad " NODE_FILE_NAME " " SECTION_PUBLIC_KEY " does not have single entry.") - % strDomain; - } - - RippleAddress naNodePublic; - - if (bGood && !naNodePublic.setNodePublic (strNodePublicKey)) - { - // Bad public key. - bGood = false; - - WriteLog (lsTRACE, UniqueNodeList) - << boost::format ("Validator: '%s' bad " NODE_FILE_NAME " " SECTION_PUBLIC_KEY " is bad: ") - % strDomain - % strNodePublicKey; - } - - if (bGood) - { - // WriteLog (lsTRACE, UniqueNodeList) << boost::format("naNodePublic: '%s'") % naNodePublic.humanNodePublic(); - - seedDomain sdCurrent; - - bool bFound = getSeedDomains (strDomain, sdCurrent); - - assert (bFound); - - uint256 iSha256 = Serializer::getSHA512Half (strSiteFile); - bool bChangedB = sdCurrent.iSha256 != iSha256; - - sdCurrent.strDomain = strDomain; - // XXX If the node public key is changing, delete old public key information? - // XXX Only if no other refs to keep it arround, other wise we have an attack vector. - sdCurrent.naPublicKey = naNodePublic; - - // WriteLog (lsTRACE, UniqueNodeList) << boost::format("sdCurrent.naPublicKey: '%s'") % sdCurrent.naPublicKey.humanNodePublic(); - - sdCurrent.tpFetch = boost::posix_time::second_clock::universal_time (); - sdCurrent.iSha256 = iSha256; - - setSeedDomains (sdCurrent, true); - - if (bChangedB) - { - WriteLog (lsTRACE, UniqueNodeList) << boost::format ("Validator: '%s' processing new " NODE_FILE_NAME ".") % strDomain; - processFile (strDomain, naNodePublic, secSite); - } else { - WriteLog (lsTRACE, UniqueNodeList) << boost::format ("Validator: '%s' no change for " NODE_FILE_NAME ".") % strDomain; + // Failed: Update + + // XXX If we have public key, perhaps try look up in CAS? fetchFinish (); } } - else - { - // Failed: Update - // XXX If we have public key, perhaps try look up in CAS? - fetchFinish (); - } + return bReject; } - return bReject; -} + //-------------------------------------------------------------------------- -// Get the ripple.txt and process it. -void UniqueNodeList::fetchProcess (std::string strDomain) -{ - WriteLog (lsTRACE, UniqueNodeList) << "Fetching '" NODE_FILE_NAME "' from '" << strDomain << "'."; - - std::deque deqSites; - - // Order searching from most specifically for purpose to generic. - // This order allows the client to take the most burden rather than the servers. - deqSites.push_back (str (boost::format (SYSTEM_NAME ".%s") % strDomain)); - deqSites.push_back (str (boost::format ("www.%s") % strDomain)); - deqSites.push_back (strDomain); - - HttpsClient::httpsGet ( - true, - theApp->getIOService (), - deqSites, - 443, - NODE_FILE_PATH, - NODE_FILE_BYTES_MAX, - boost::posix_time::seconds (NODE_FETCH_SECONDS), - BIND_TYPE (&UniqueNodeList::responseFetch, this, strDomain, P_1, P_2, P_3)); -} - -void UniqueNodeList::fetchTimerHandler (const boost::system::error_code& err) -{ - if (!err) + // Try to process the next fetch of a ripple.txt. + void fetchNext () { - // Time to check for another fetch. - WriteLog (lsTRACE, UniqueNodeList) << "fetchTimerHandler"; - fetchNext (); - } -} + bool bFull; -// Try to process the next fetch of a ripple.txt. -void UniqueNodeList::fetchNext () -{ - bool bFull; - - { - boost::mutex::scoped_lock sl (mFetchLock); - - bFull = mFetchActive == NODE_FETCH_JOBS; - } - - if (!bFull) - { - // Determine next scan. - std::string strDomain; - boost::posix_time::ptime tpNext; - boost::posix_time::ptime tpNow; - - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - Database* db = theApp->getWalletDB ()->getDB (); - - if (db->executeSQL ("SELECT Domain,Next FROM SeedDomains INDEXED BY SeedDomainNext ORDER BY Next LIMIT 1;") - && db->startIterRows ()) - { - int iNext = db->getInt ("Next"); - - tpNext = ptFromSeconds (iNext); - tpNow = boost::posix_time::second_clock::universal_time (); - - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: iNext=%s tpNext=%s tpNow=%s") % iNext % tpNext % tpNow); - strDomain = db->getStrBinary ("Domain"); - - db->endIterRows (); - } - - if (!strDomain.empty ()) { boost::mutex::scoped_lock sl (mFetchLock); bFull = mFetchActive == NODE_FETCH_JOBS; + } - if (!bFull && tpNext <= tpNow) + if (!bFull) + { + // Determine next scan. + std::string strDomain; + boost::posix_time::ptime tpNext; + boost::posix_time::ptime tpNow; + + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + Database* db = getApp().getWalletDB ()->getDB (); + + if (db->executeSQL ("SELECT Domain,Next FROM SeedDomains INDEXED BY SeedDomainNext ORDER BY Next LIMIT 1;") + && db->startIterRows ()) { - mFetchActive++; + int iNext = db->getInt ("Next"); + + tpNext = ptFromSeconds (iNext); + tpNow = boost::posix_time::second_clock::universal_time (); + + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: iNext=%s tpNext=%s tpNow=%s") % iNext % tpNext % tpNow); + strDomain = db->getStrBinary ("Domain"); + + db->endIterRows (); + } + + if (!strDomain.empty ()) + { + boost::mutex::scoped_lock sl (mFetchLock); + + bFull = mFetchActive == NODE_FETCH_JOBS; + + if (!bFull && tpNext <= tpNow) + { + mFetchActive++; + } + } + + if (strDomain.empty () || bFull) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: strDomain=%s bFull=%d") % strDomain % bFull); + + nothing (); + } + else if (tpNext > tpNow) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: set timer : strDomain=%s") % strDomain); + // Fetch needs to happen in the future. Set a timer to wake us. + mtpFetchNext = tpNext; + + double const seconds = (tpNext - tpNow).seconds (); + + m_fetchTimer.setExpiration (seconds); + } + else + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: fetch now: strDomain=%s tpNext=%s tpNow=%s") % strDomain % tpNext % tpNow); + // Fetch needs to happen now. + mtpFetchNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); + + seedDomain sdCurrent; + bool bFound = getSeedDomains (strDomain, sdCurrent); + + assert (bFound); + + // Update time of next fetch and this scan attempt. + sdCurrent.tpScan = tpNow; + + // XXX Use a longer duration if we have lots of validators. + sdCurrent.tpNext = sdCurrent.tpScan + boost::posix_time::hours (7 * 24); + + setSeedDomains (sdCurrent, false); + + WriteLog (lsTRACE, UniqueNodeList) << "Validator: '" << strDomain << "' fetching " NODE_FILE_NAME "."; + + fetchProcess (strDomain); // Go get it. + + fetchNext (); // Look for more. } } - - if (strDomain.empty () || bFull) - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: strDomain=%s bFull=%d") % strDomain % bFull); - - nothing (); - } - else if (tpNext > tpNow) - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: set timer : strDomain=%s") % strDomain); - // Fetch needs to happen in the future. Set a timer to wake us. - mtpFetchNext = tpNext; - - mdtFetchTimer.expires_at (mtpFetchNext); - mdtFetchTimer.async_wait (boost::bind (&UniqueNodeList::fetchTimerHandler, this, _1)); - } - else - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: fetch now: strDomain=%s tpNext=%s tpNow=%s") % strDomain % tpNext % tpNow); - // Fetch needs to happen now. - mtpFetchNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); - - seedDomain sdCurrent; - bool bFound = getSeedDomains (strDomain, sdCurrent); - - assert (bFound); - - // Update time of next fetch and this scan attempt. - sdCurrent.tpScan = tpNow; - - // XXX Use a longer duration if we have lots of validators. - sdCurrent.tpNext = sdCurrent.tpScan + boost::posix_time::hours (7 * 24); - - setSeedDomains (sdCurrent, false); - - WriteLog (lsTRACE, UniqueNodeList) << "Validator: '" << strDomain << "' fetching " NODE_FILE_NAME "."; - - fetchProcess (strDomain); // Go get it. - - fetchNext (); // Look for more. - } - } -} - -// For each kind of source, have a starting number of points to be distributed. -int UniqueNodeList::iSourceScore (validatorSource vsWhy) -{ - int iScore = 0; - - switch (vsWhy) - { - case vsConfig: - iScore = 1500; - break; - - case vsInbound: - iScore = 0; - break; - - case vsManual: - iScore = 1500; - break; - - case vsReferral: - iScore = 0; - break; - - case vsTold: - iScore = 0; - break; - - case vsValidator: - iScore = 1000; - break; - - case vsWeb: - iScore = 200; - break; - - default: - throw std::runtime_error ("Internal error: bad validatorSource."); } - return iScore; -} + //-------------------------------------------------------------------------- -// Retrieve a SeedDomain from DB. -bool UniqueNodeList::getSeedDomains (const std::string& strDomain, seedDomain& dstSeedDomain) -{ - bool bResult; - Database* db = theApp->getWalletDB ()->getDB (); - - std::string strSql = boost::str (boost::format ("SELECT * FROM SeedDomains WHERE Domain=%s;") - % sqlEscape (strDomain)); - - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - bResult = db->executeSQL (strSql) && db->startIterRows (); - - if (bResult) + // Called when we need to update scores. + void fetchDirty () { - std::string strPublicKey; - int iNext; - int iScan; - int iFetch; - std::string strSha256; + // Note update. + mtpFetchUpdated = boost::posix_time::second_clock::universal_time (); + miscSave (); - dstSeedDomain.strDomain = db->getStrBinary ("Domain"); - - if (!db->getNull ("PublicKey") && db->getStr ("PublicKey", strPublicKey)) - { - dstSeedDomain.naPublicKey.setNodePublic (strPublicKey); - } - else - { - dstSeedDomain.naPublicKey.clear (); - } - - std::string strSource = db->getStrBinary ("Source"); - dstSeedDomain.vsSource = static_cast (strSource[0]); - - iNext = db->getInt ("Next"); - dstSeedDomain.tpNext = ptFromSeconds (iNext); - iScan = db->getInt ("Scan"); - dstSeedDomain.tpScan = ptFromSeconds (iScan); - iFetch = db->getInt ("Fetch"); - dstSeedDomain.tpFetch = ptFromSeconds (iFetch); - - if (!db->getNull ("Sha256") && db->getStr ("Sha256", strSha256)) - { - dstSeedDomain.iSha256.SetHex (strSha256); - } - else - { - dstSeedDomain.iSha256.zero (); - } - - dstSeedDomain.strComment = db->getStrBinary ("Comment"); - - db->endIterRows (); + // Update scores. + scoreNext (false); } - return bResult; -} -// Persist a SeedDomain. -void UniqueNodeList::setSeedDomains (const seedDomain& sdSource, bool bNext) -{ - Database* db = theApp->getWalletDB ()->getDB (); + //-------------------------------------------------------------------------- - int iNext = iToSeconds (sdSource.tpNext); - int iScan = iToSeconds (sdSource.tpScan); - int iFetch = iToSeconds (sdSource.tpFetch); - - // WriteLog (lsTRACE) << str(boost::format("setSeedDomains: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); - - std::string strSql = boost::str (boost::format ("REPLACE INTO SeedDomains (Domain,PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES (%s, %s, %s, %d, %d, %d, '%s', %s);") - % sqlEscape (sdSource.strDomain) - % (sdSource.naPublicKey.isValid () ? sqlEscape (sdSource.naPublicKey.humanNodePublic ()) : "NULL") - % sqlEscape (std::string (1, static_cast (sdSource.vsSource))) - % iNext - % iScan - % iFetch - % sdSource.iSha256.GetHex () - % sqlEscape (sdSource.strComment) - ); - - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - if (!db->executeSQL (strSql)) + void fetchFinish () { - // XXX Check result. - WriteLog (lsWARNING, UniqueNodeList) << "setSeedDomains: failed."; - } - - if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > sdSource.tpNext)) - { - // Schedule earlier wake up. - fetchNext (); - } -} - -// Queue a domain for a single attempt fetch a ripple.txt. -// --> strComment: only used on vsManual -// YYY As a lot of these may happen at once, would be nice to wrap multiple calls in a transaction. -void UniqueNodeList::nodeAddDomain (std::string strDomain, validatorSource vsWhy, const std::string& strComment) -{ - boost::trim (strDomain); - boost::to_lower (strDomain); - - // YYY Would be best to verify strDomain is a valid domain. - // WriteLog (lsTRACE) << str(boost::format("nodeAddDomain: '%s' %c '%s'") - // % strDomain - // % vsWhy - // % strComment); - - seedDomain sdCurrent; - - bool bFound = getSeedDomains (strDomain, sdCurrent); - bool bChanged = false; - - if (!bFound) - { - sdCurrent.strDomain = strDomain; - sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - } - - // Promote source, if needed. - if (!bFound || iSourceScore (vsWhy) >= iSourceScore (sdCurrent.vsSource)) - { - sdCurrent.vsSource = vsWhy; - sdCurrent.strComment = strComment; - bChanged = true; - } - - if (vsManual == vsWhy) - { - // A manual add forces immediate scan. - sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - bChanged = true; - } - - if (bChanged) - setSeedDomains (sdCurrent, true); -} - -// Retrieve a SeedNode from DB. -bool UniqueNodeList::getSeedNodes (const RippleAddress& naNodePublic, seedNode& dstSeedNode) -{ - bool bResult; - Database* db = theApp->getWalletDB ()->getDB (); - - std::string strSql = str (boost::format ("SELECT * FROM SeedNodes WHERE PublicKey='%s';") - % naNodePublic.humanNodePublic ()); - - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - bResult = db->executeSQL (strSql) && db->startIterRows (); - - if (bResult) - { - std::string strPublicKey; - std::string strSource; - int iNext; - int iScan; - int iFetch; - std::string strSha256; - - if (!db->getNull ("PublicKey") && db->getStr ("PublicKey", strPublicKey)) { - dstSeedNode.naPublicKey.setNodePublic (strPublicKey); - } - else - { - dstSeedNode.naPublicKey.clear (); + boost::mutex::scoped_lock sl (mFetchLock); + mFetchActive--; } - strSource = db->getStrBinary ("Source"); - dstSeedNode.vsSource = static_cast (strSource[0]); - - iNext = db->getInt ("Next"); - dstSeedNode.tpNext = ptFromSeconds (iNext); - iScan = db->getInt ("Scan"); - dstSeedNode.tpScan = ptFromSeconds (iScan); - iFetch = db->getInt ("Fetch"); - dstSeedNode.tpFetch = ptFromSeconds (iFetch); - - if (!db->getNull ("Sha256") && db->getStr ("Sha256", strSha256)) - { - dstSeedNode.iSha256.SetHex (strSha256); - } - else - { - dstSeedNode.iSha256.zero (); - } - - dstSeedNode.strComment = db->getStrBinary ("Comment"); - - db->endIterRows (); - } - - return bResult; -} - -// Persist a SeedNode. -// <-- bNext: true, to do fetching if needed. -void UniqueNodeList::setSeedNodes (const seedNode& snSource, bool bNext) -{ - Database* db = theApp->getWalletDB ()->getDB (); - - int iNext = iToSeconds (snSource.tpNext); - int iScan = iToSeconds (snSource.tpScan); - int iFetch = iToSeconds (snSource.tpFetch); - - // WriteLog (lsTRACE) << str(boost::format("setSeedNodes: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); - - assert (snSource.naPublicKey.isValid ()); - - std::string strSql = str (boost::format ("REPLACE INTO SeedNodes (PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES ('%s', '%c', %d, %d, %d, '%s', %s);") - % snSource.naPublicKey.humanNodePublic () - % static_cast (snSource.vsSource) - % iNext - % iScan - % iFetch - % snSource.iSha256.GetHex () - % sqlEscape (snSource.strComment) - ); - - { - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - if (!db->executeSQL (strSql)) - { - // XXX Check result. - WriteLog (lsTRACE, UniqueNodeList) << "setSeedNodes: failed."; - } - } - -#if 0 - - // YYY When we have a cas schedule lookups similar to this. - if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > snSource.tpNext)) - { - // Schedule earlier wake up. fetchNext (); } -#else - fetchDirty (); -#endif -} + //-------------------------------------------------------------------------- -// Add a trusted node. Called by RPC or other source. -void UniqueNodeList::nodeAddPublic (const RippleAddress& naNodePublic, validatorSource vsWhy, const std::string& strComment) -{ - seedNode snCurrent; - - bool bFound = getSeedNodes (naNodePublic, snCurrent); - bool bChanged = false; - - if (!bFound) + // Get the ripple.txt and process it. + void fetchProcess (std::string strDomain) { - snCurrent.naPublicKey = naNodePublic; - snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - } + WriteLog (lsTRACE, UniqueNodeList) << "Fetching '" NODE_FILE_NAME "' from '" << strDomain << "'."; - // Promote source, if needed. - if (!bFound || iSourceScore (vsWhy) >= iSourceScore (snCurrent.vsSource)) - { - snCurrent.vsSource = vsWhy; - snCurrent.strComment = strComment; - bChanged = true; - } + std::deque deqSites; - if (vsManual == vsWhy) - { - // A manual add forces immediate scan. - snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - bChanged = true; - } + // Order searching from most specifically for purpose to generic. + // This order allows the client to take the most burden rather than the servers. + deqSites.push_back (str (boost::format (SYSTEM_NAME ".%s") % strDomain)); + deqSites.push_back (str (boost::format ("www.%s") % strDomain)); + deqSites.push_back (strDomain); - if (bChanged) - setSeedNodes (snCurrent, true); -} - -void UniqueNodeList::nodeRemovePublic (const RippleAddress& naNodePublic) -{ - { - Database* db = theApp->getWalletDB ()->getDB (); - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - db->executeSQL (str (boost::format ("DELETE FROM SeedNodes WHERE PublicKey=%s") % sqlEscape (naNodePublic.humanNodePublic ()))); - db->executeSQL (str (boost::format ("DELETE FROM TrustedNodes WHERE PublicKey=%s") % sqlEscape (naNodePublic.humanNodePublic ()))); - } - - // YYY Only dirty on successful delete. - fetchDirty (); - - boost::recursive_mutex::scoped_lock sl (mUNLLock); - mUNL.erase (naNodePublic.humanNodePublic ()); -} - -void UniqueNodeList::nodeRemoveDomain (std::string strDomain) -{ - boost::trim (strDomain); - boost::to_lower (strDomain); - - { - Database* db = theApp->getWalletDB ()->getDB (); - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - db->executeSQL (str (boost::format ("DELETE FROM SeedDomains WHERE Domain=%s") % sqlEscape (strDomain))); - } - - // YYY Only dirty on successful delete. - fetchDirty (); -} - -void UniqueNodeList::nodeReset () -{ - { - Database* db = theApp->getWalletDB ()->getDB (); - - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - // XXX Check results. - db->executeSQL ("DELETE FROM SeedDomains"); - db->executeSQL ("DELETE FROM SeedNodes"); - } - - fetchDirty (); -} - -Json::Value UniqueNodeList::getUnlJson () -{ - Database* db = theApp->getWalletDB ()->getDB (); - - Json::Value ret (Json::arrayValue); - - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - SQL_FOREACH (db, "SELECT * FROM TrustedNodes;") - { - Json::Value node (Json::objectValue); - - node["publicKey"] = db->getStrBinary ("PublicKey"); - node["comment"] = db->getStrBinary ("Comment"); - - ret.append (node); - } - - return ret; -} - -bool UniqueNodeList::nodeLoad (boost::filesystem::path pConfig) -{ - if (pConfig.empty ()) - { - WriteLog (lsINFO, UniqueNodeList) << VALIDATORS_FILE_NAME " path not specified."; - - return false; - } - - if (!boost::filesystem::exists (pConfig)) - { - WriteLog (lsWARNING, UniqueNodeList) << str (boost::format (VALIDATORS_FILE_NAME " not found: %s") % pConfig); - - return false; - } - - if (!boost::filesystem::is_regular_file (pConfig)) - { - WriteLog (lsWARNING, UniqueNodeList) << str (boost::format (VALIDATORS_FILE_NAME " not regular file: %s") % pConfig); - - return false; - } - - std::ifstream ifsDefault (pConfig.native ().c_str (), std::ios::in); - - if (!ifsDefault) - { - WriteLog (lsFATAL, UniqueNodeList) << str (boost::format (VALIDATORS_FILE_NAME " failed to open: %s") % pConfig); - - return false; - } - - std::string strValidators; - - strValidators.assign ((std::istreambuf_iterator (ifsDefault)), - std::istreambuf_iterator ()); - - if (ifsDefault.bad ()) - { - WriteLog (lsFATAL, UniqueNodeList) << str (boost::format ("Failed to read: %s") % pConfig); - - return false; - } - - nodeProcess ("local", strValidators, pConfig.string ()); - - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("Processing: %s") % pConfig); - - return true; -} - -bool UniqueNodeList::validatorsResponse (const boost::system::error_code& err, int iStatus, std::string strResponse) -{ - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - WriteLog (lsTRACE, UniqueNodeList) << "Fetch '" VALIDATORS_FILE_NAME "' complete."; - - if (!err) - { - nodeProcess ("network", strResponse, theConfig.VALIDATORS_SITE); - } - else - { - WriteLog (lsWARNING, UniqueNodeList) << "Error: " << err.message (); - } - } - - return bReject; -} - -void UniqueNodeList::nodeNetwork () -{ - if (!theConfig.VALIDATORS_SITE.empty ()) - { HttpsClient::httpsGet ( true, - theApp->getIOService (), - theConfig.VALIDATORS_SITE, + getApp().getIOService (), + deqSites, 443, - theConfig.VALIDATORS_URI, - VALIDATORS_FILE_BYTES_MAX, - boost::posix_time::seconds (VALIDATORS_FETCH_SECONDS), - BIND_TYPE (&UniqueNodeList::validatorsResponse, this, P_1, P_2, P_3)); - } -} - -void UniqueNodeList::nodeBootstrap () -{ - int iDomains = 0; - int iNodes = 0; - Database* db = theApp->getWalletDB ()->getDB (); - - { - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); - - if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedDomains WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) - iDomains = db->getInt ("Count"); - - db->endIterRows (); - - if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedNodes WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) - iNodes = db->getInt ("Count"); - - db->endIterRows (); + NODE_FILE_PATH, + NODE_FILE_BYTES_MAX, + boost::posix_time::seconds (NODE_FETCH_SECONDS), + BIND_TYPE (&UniqueNodeListImp::responseFetch, this, strDomain, P_1, P_2, P_3)); } - bool bLoaded = iDomains || iNodes; + //-------------------------------------------------------------------------- - // Always merge in the file specified in the config. - if (!theConfig.VALIDATORS_FILE.empty ()) + void fetchTimerHandler (const boost::system::error_code& err) { - WriteLog (lsINFO, UniqueNodeList) << "Bootstrapping UNL: loading from unl_default."; - - bLoaded = nodeLoad (theConfig.VALIDATORS_FILE); - } - - // If never loaded anything try the current directory. - if (!bLoaded && theConfig.VALIDATORS_FILE.empty ()) - { - WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") - % theConfig.VALIDATORS_BASE); - - bLoaded = nodeLoad (theConfig.VALIDATORS_BASE); - } - - // Always load from rippled.cfg - if (!theConfig.VALIDATORS.empty ()) - { - RippleAddress naInvalid; // Don't want a referrer on added entries. - - WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") - % theConfig.CONFIG_FILE); - - if (processValidators ("local", theConfig.CONFIG_FILE.string (), naInvalid, vsConfig, &theConfig.VALIDATORS)) - bLoaded = true; - } - - if (!bLoaded) - { - WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") - % theConfig.VALIDATORS_SITE); - - nodeNetwork (); - } - - if (!theConfig.IPS.empty ()) - { - std::vector vstrValues; - - vstrValues.reserve (theConfig.IPS.size ()); - - BOOST_FOREACH (const std::string & strPeer, theConfig.IPS) + if (!err) { - std::string strIP; - int iPort; + onDeadlineTimer (m_fetchTimer); + } + } - if (parseIpPort (strPeer, strIP, iPort)) + + //-------------------------------------------------------------------------- + + // Process Section [validators_url]. + void getValidatorsUrl (const RippleAddress& naNodePublic, Section secSite) + { + std::string strValidatorsUrl; + std::string strScheme; + std::string strDomain; + int iPort; + std::string strPath; + + if (SectionSingleB (secSite, SECTION_VALIDATORS_URL, strValidatorsUrl) + && !strValidatorsUrl.empty () + && parseUrl (strValidatorsUrl, strScheme, strDomain, iPort, strPath) + && -1 == iPort + && strScheme == "https") + { + HttpsClient::httpsGet ( + true, + getApp().getIOService (), + strDomain, + 443, + strPath, + NODE_FILE_BYTES_MAX, + boost::posix_time::seconds (NODE_FETCH_SECONDS), + BIND_TYPE (&UniqueNodeListImp::responseValidators, this, strValidatorsUrl, naNodePublic, secSite, strDomain, P_1, P_2, P_3)); + } + else + { + getIpsUrl (naNodePublic, secSite); + } + } + + //-------------------------------------------------------------------------- + + // Process Section [ips_url]. + // If we have a Section with a single entry, fetch the url and process it. + void getIpsUrl (const RippleAddress& naNodePublic, Section secSite) + { + std::string strIpsUrl; + std::string strScheme; + std::string strDomain; + int iPort; + std::string strPath; + + if (SectionSingleB (secSite, SECTION_IPS_URL, strIpsUrl) + && !strIpsUrl.empty () + && parseUrl (strIpsUrl, strScheme, strDomain, iPort, strPath) + && -1 == iPort + && strScheme == "https") + { + HttpsClient::httpsGet ( + true, + getApp().getIOService (), + strDomain, + 443, + strPath, + NODE_FILE_BYTES_MAX, + boost::posix_time::seconds (NODE_FETCH_SECONDS), + BIND_TYPE (&UniqueNodeListImp::responseIps, this, strDomain, naNodePublic, P_1, P_2, P_3)); + } + else + { + fetchFinish (); + } + } + + + //-------------------------------------------------------------------------- + + // Given a Section with IPs, parse and persist it for a validator. + bool responseIps (const std::string& strSite, const RippleAddress& naNodePublic, const boost::system::error_code& err, int iStatus, const std::string& strIpsFile) + { + bool bReject = !err && iStatus != 200; + + if (!bReject) + { + if (!err) { - vstrValues.push_back (str (boost::format ("(%s,'%c')") - % sqlEscape (str (boost::format ("%s %d") % strIP % iPort)) - % static_cast (vsConfig))); + Section secFile = ParseSection (strIpsFile, true); + + processIps (strSite, naNodePublic, SectionEntries (secFile, SECTION_IPS)); } + + fetchFinish (); } - if (!vstrValues.empty ()) - { - boost::recursive_mutex::scoped_lock sl (theApp->getWalletDB ()->getDBLock ()); + return bReject; + } - db->executeSQL (str (boost::format ("REPLACE INTO PeerIps (IpPort,Source) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ","))); + // After fetching a ripple.txt from a web site, given a Section with validators, parse and persist it. + bool responseValidators (const std::string& strValidatorsUrl, const RippleAddress& naNodePublic, Section secSite, const std::string& strSite, const boost::system::error_code& err, int iStatus, const std::string& strValidatorsFile) + { + bool bReject = !err && iStatus != 200; + + if (!bReject) + { + if (!err) + { + Section secFile = ParseSection (strValidatorsFile, true); + + processValidators (strSite, strValidatorsUrl, naNodePublic, vsValidator, SectionEntries (secFile, SECTION_VALIDATORS)); + } + + getIpsUrl (naNodePublic, secSite); + } + + return bReject; + } + + + //-------------------------------------------------------------------------- + + // Persist the IPs refered to by a Validator. + // --> strSite: source of the IPs (for debugging) + // --> naNodePublic: public key of the validating node. + void processIps (const std::string& strSite, const RippleAddress& naNodePublic, Section::mapped_type* pmtVecStrIps) + { + Database* db = getApp().getWalletDB ()->getDB (); + + std::string strEscNodePublic = sqlEscape (naNodePublic.humanNodePublic ()); + + WriteLog (lsDEBUG, UniqueNodeList) + << str (boost::format ("Validator: '%s' processing %d ips.") + % strSite % ( pmtVecStrIps ? pmtVecStrIps->size () : 0)); + + // Remove all current Validator's entries in IpReferrals + { + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + db->executeSQL (str (boost::format ("DELETE FROM IpReferrals WHERE Validator=%s;") % strEscNodePublic)); + // XXX Check result. + } + + // Add new referral entries. + if (pmtVecStrIps && !pmtVecStrIps->empty ()) + { + std::vector vstrValues; + + vstrValues.resize (std::min ((int) pmtVecStrIps->size (), REFERRAL_IPS_MAX)); + + int iValues = 0; + BOOST_FOREACH (const std::string & strReferral, *pmtVecStrIps) + { + if (iValues == REFERRAL_VALIDATORS_MAX) + break; + + std::string strIP; + int iPort; + bool bValid = parseIpPort (strReferral, strIP, iPort); + + // XXX Filter out private network ips. + // XXX http://en.wikipedia.org/wiki/Private_network + + if (bValid) + { + vstrValues[iValues] = str (boost::format ("(%s,%d,%s,%d)") + % strEscNodePublic % iValues % sqlEscape (strIP) % iPort); + iValues++; + } + else + { + WriteLog (lsTRACE, UniqueNodeList) + << str (boost::format ("Validator: '%s' [" SECTION_IPS "]: rejecting '%s'") + % strSite % strReferral); + } + } + + if (iValues) + { + vstrValues.resize (iValues); + + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + db->executeSQL (str (boost::format ("INSERT INTO IpReferrals (Validator,Entry,IP,Port) VALUES %s;") + % strJoin (vstrValues.begin (), vstrValues.end (), ","))); + // XXX Check result. + } } fetchDirty (); } -} -// Process a validators.txt. -// --> strSite: source of validators -// --> strValidators: contents of a validators.txt -void UniqueNodeList::nodeProcess (const std::string& strSite, const std::string& strValidators, const std::string& strSource) -{ - Section secValidators = ParseSection (strValidators, true); + //-------------------------------------------------------------------------- - Section::mapped_type* pmtEntries = SectionEntries (secValidators, SECTION_VALIDATORS); - - if (pmtEntries) + // Persist ValidatorReferrals. + // --> strSite: source site for display + // --> strValidatorsSrc: source details for display + // --> naNodePublic: remote source public key - not valid for local + // --> vsWhy: reason for adding validator to SeedDomains or SeedNodes. + int processValidators (const std::string& strSite, const std::string& strValidatorsSrc, const RippleAddress& naNodePublic, ValidatorSource vsWhy, Section::mapped_type* pmtVecStrValidators) { - RippleAddress naInvalid; // Don't want a referrer on added entries. + Database* db = getApp().getWalletDB ()->getDB (); + std::string strNodePublic = naNodePublic.isValid () ? naNodePublic.humanNodePublic () : strValidatorsSrc; + int iValues = 0; - // YYY Unspecified might be bootstrap or rpc command - processValidators (strSite, strSource, naInvalid, vsValidator, pmtEntries); + WriteLog (lsTRACE, UniqueNodeList) + << str (boost::format ("Validator: '%s' : '%s' : processing %d validators.") + % strSite + % strValidatorsSrc + % ( pmtVecStrValidators ? pmtVecStrValidators->size () : 0)); + + // Remove all current Validator's entries in ValidatorReferrals + { + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + db->executeSQL (str (boost::format ("DELETE FROM ValidatorReferrals WHERE Validator='%s';") % strNodePublic)); + // XXX Check result. + } + + // Add new referral entries. + if (pmtVecStrValidators && pmtVecStrValidators->size ()) + { + std::vector vstrValues; + + vstrValues.reserve (std::min ((int) pmtVecStrValidators->size (), REFERRAL_VALIDATORS_MAX)); + + BOOST_FOREACH (const std::string & strReferral, *pmtVecStrValidators) + { + if (iValues == REFERRAL_VALIDATORS_MAX) + break; + + boost::smatch smMatch; + + // domain comment? + // public_key comment? + static boost::regex reReferral ("\\`\\s*(\\S+)(?:\\s+(.+))?\\s*\\'"); + + if (!boost::regex_match (strReferral, smMatch, reReferral)) + { + WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: syntax error: %s: %s") % strSite % strReferral); + } + else + { + std::string strRefered = smMatch[1]; + std::string strComment = smMatch[2]; + RippleAddress naValidator; + + if (naValidator.setSeedGeneric (strRefered)) + { + + WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: domain or public key required: %s %s") % strRefered % strComment); + } + else if (naValidator.setNodePublic (strRefered)) + { + // A public key. + // XXX Schedule for CAS lookup. + nodeAddPublic (naValidator, vsWhy, strComment); + + WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Public: %s %s") % strRefered % strComment); + + if (naNodePublic.isValid ()) + vstrValues.push_back (str (boost::format ("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic ())); + + iValues++; + } + else + { + // A domain: need to look it up. + nodeAddDomain (strRefered, vsWhy, strComment); + + WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Domain: %s %s") % strRefered % strComment); + + if (naNodePublic.isValid ()) + vstrValues.push_back (str (boost::format ("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape (strRefered))); + + iValues++; + } + } + } + + if (!vstrValues.empty ()) + { + std::string strSql = str (boost::format ("INSERT INTO ValidatorReferrals (Validator,Entry,Referral) VALUES %s;") + % strJoin (vstrValues.begin (), vstrValues.end (), ",")); + + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + db->executeSQL (strSql); + // XXX Check result. + } + } + + fetchDirty (); + + return iValues; } - else + + //-------------------------------------------------------------------------- + + // Process a ripple.txt. + void processFile (const std::string& strDomain, const RippleAddress& naNodePublic, Section secSite) { - WriteLog (lsWARNING, UniqueNodeList) << boost::str (boost::format ("'%s' missing [" SECTION_VALIDATORS "].") - % theConfig.VALIDATORS_BASE); + // + // Process Validators + // + processValidators (strDomain, NODE_FILE_NAME, naNodePublic, vsReferral, SectionEntries (secSite, SECTION_VALIDATORS)); + + // + // Process ips + // + processIps (strDomain, naNodePublic, SectionEntries (secSite, SECTION_IPS)); + + // + // Process currencies + // + Section::mapped_type* pvCurrencies; + + if ((pvCurrencies = SectionEntries (secSite, SECTION_CURRENCIES)) && pvCurrencies->size ()) + { + // XXX Process currencies. + WriteLog (lsWARNING, UniqueNodeList) << "Ignoring currencies: not implemented."; + } + + getValidatorsUrl (naNodePublic, secSite); } -} -bool UniqueNodeList::nodeInUNL (const RippleAddress& naNodePublic) + //-------------------------------------------------------------------------- + + // Retrieve a SeedDomain from DB. + bool getSeedDomains (const std::string& strDomain, seedDomain& dstSeedDomain) + { + bool bResult; + Database* db = getApp().getWalletDB ()->getDB (); + + std::string strSql = boost::str (boost::format ("SELECT * FROM SeedDomains WHERE Domain=%s;") + % sqlEscape (strDomain)); + + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + bResult = db->executeSQL (strSql) && db->startIterRows (); + + if (bResult) + { + std::string strPublicKey; + int iNext; + int iScan; + int iFetch; + std::string strSha256; + + dstSeedDomain.strDomain = db->getStrBinary ("Domain"); + + if (!db->getNull ("PublicKey") && db->getStr ("PublicKey", strPublicKey)) + { + dstSeedDomain.naPublicKey.setNodePublic (strPublicKey); + } + else + { + dstSeedDomain.naPublicKey.clear (); + } + + std::string strSource = db->getStrBinary ("Source"); + dstSeedDomain.vsSource = static_cast (strSource[0]); + + iNext = db->getInt ("Next"); + dstSeedDomain.tpNext = ptFromSeconds (iNext); + iScan = db->getInt ("Scan"); + dstSeedDomain.tpScan = ptFromSeconds (iScan); + iFetch = db->getInt ("Fetch"); + dstSeedDomain.tpFetch = ptFromSeconds (iFetch); + + if (!db->getNull ("Sha256") && db->getStr ("Sha256", strSha256)) + { + dstSeedDomain.iSha256.SetHex (strSha256); + } + else + { + dstSeedDomain.iSha256.zero (); + } + + dstSeedDomain.strComment = db->getStrBinary ("Comment"); + + db->endIterRows (); + } + + return bResult; + } + + //-------------------------------------------------------------------------- + + // Persist a SeedDomain. + void setSeedDomains (const seedDomain& sdSource, bool bNext) + { + Database* db = getApp().getWalletDB ()->getDB (); + + int iNext = iToSeconds (sdSource.tpNext); + int iScan = iToSeconds (sdSource.tpScan); + int iFetch = iToSeconds (sdSource.tpFetch); + + // WriteLog (lsTRACE) << str(boost::format("setSeedDomains: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); + + std::string strSql = boost::str (boost::format ("REPLACE INTO SeedDomains (Domain,PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES (%s, %s, %s, %d, %d, %d, '%s', %s);") + % sqlEscape (sdSource.strDomain) + % (sdSource.naPublicKey.isValid () ? sqlEscape (sdSource.naPublicKey.humanNodePublic ()) : "NULL") + % sqlEscape (std::string (1, static_cast (sdSource.vsSource))) + % iNext + % iScan + % iFetch + % sdSource.iSha256.GetHex () + % sqlEscape (sdSource.strComment) + ); + + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + if (!db->executeSQL (strSql)) + { + // XXX Check result. + WriteLog (lsWARNING, UniqueNodeList) << "setSeedDomains: failed."; + } + + if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > sdSource.tpNext)) + { + // Schedule earlier wake up. + fetchNext (); + } + } + + + //-------------------------------------------------------------------------- + + // Retrieve a SeedNode from DB. + bool getSeedNodes (const RippleAddress& naNodePublic, seedNode& dstSeedNode) + { + bool bResult; + Database* db = getApp().getWalletDB ()->getDB (); + + std::string strSql = str (boost::format ("SELECT * FROM SeedNodes WHERE PublicKey='%s';") + % naNodePublic.humanNodePublic ()); + + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + bResult = db->executeSQL (strSql) && db->startIterRows (); + + if (bResult) + { + std::string strPublicKey; + std::string strSource; + int iNext; + int iScan; + int iFetch; + std::string strSha256; + + if (!db->getNull ("PublicKey") && db->getStr ("PublicKey", strPublicKey)) + { + dstSeedNode.naPublicKey.setNodePublic (strPublicKey); + } + else + { + dstSeedNode.naPublicKey.clear (); + } + + strSource = db->getStrBinary ("Source"); + dstSeedNode.vsSource = static_cast (strSource[0]); + + iNext = db->getInt ("Next"); + dstSeedNode.tpNext = ptFromSeconds (iNext); + iScan = db->getInt ("Scan"); + dstSeedNode.tpScan = ptFromSeconds (iScan); + iFetch = db->getInt ("Fetch"); + dstSeedNode.tpFetch = ptFromSeconds (iFetch); + + if (!db->getNull ("Sha256") && db->getStr ("Sha256", strSha256)) + { + dstSeedNode.iSha256.SetHex (strSha256); + } + else + { + dstSeedNode.iSha256.zero (); + } + + dstSeedNode.strComment = db->getStrBinary ("Comment"); + + db->endIterRows (); + } + + return bResult; + } + + //-------------------------------------------------------------------------- + + // Persist a SeedNode. + // <-- bNext: true, to do fetching if needed. + void setSeedNodes (const seedNode& snSource, bool bNext) + { + Database* db = getApp().getWalletDB ()->getDB (); + + int iNext = iToSeconds (snSource.tpNext); + int iScan = iToSeconds (snSource.tpScan); + int iFetch = iToSeconds (snSource.tpFetch); + + // WriteLog (lsTRACE) << str(boost::format("setSeedNodes: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); + + assert (snSource.naPublicKey.isValid ()); + + std::string strSql = str (boost::format ("REPLACE INTO SeedNodes (PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES ('%s', '%c', %d, %d, %d, '%s', %s);") + % snSource.naPublicKey.humanNodePublic () + % static_cast (snSource.vsSource) + % iNext + % iScan + % iFetch + % snSource.iSha256.GetHex () + % sqlEscape (snSource.strComment) + ); + + { + boost::recursive_mutex::scoped_lock sl (getApp().getWalletDB ()->getDBLock ()); + + if (!db->executeSQL (strSql)) + { + // XXX Check result. + WriteLog (lsTRACE, UniqueNodeList) << "setSeedNodes: failed."; + } + } + + #if 0 + + // YYY When we have a cas schedule lookups similar to this. + if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > snSource.tpNext)) + { + // Schedule earlier wake up. + fetchNext (); + } + + #else + fetchDirty (); + #endif + } + + //-------------------------------------------------------------------------- + + bool validatorsResponse (const boost::system::error_code& err, int iStatus, std::string strResponse) + { + bool bReject = !err && iStatus != 200; + + if (!bReject) + { + WriteLog (lsTRACE, UniqueNodeList) << "Fetch '" VALIDATORS_FILE_NAME "' complete."; + + if (!err) + { + nodeProcess ("network", strResponse, theConfig.VALIDATORS_SITE); + } + else + { + WriteLog (lsWARNING, UniqueNodeList) << "Error: " << err.message (); + } + } + + return bReject; + } + + //-------------------------------------------------------------------------- + + // Process a validators.txt. + // --> strSite: source of validators + // --> strValidators: contents of a validators.txt + // + // VFALCO TODO Can't we name this processValidatorList? + // + void nodeProcess (const std::string& strSite, const std::string& strValidators, const std::string& strSource) + { + Section secValidators = ParseSection (strValidators, true); + + Section::mapped_type* pmtEntries = SectionEntries (secValidators, SECTION_VALIDATORS); + + if (pmtEntries) + { + RippleAddress naInvalid; // Don't want a referrer on added entries. + + // YYY Unspecified might be bootstrap or rpc command + processValidators (strSite, strSource, naInvalid, vsValidator, pmtEntries); + } + else + { + WriteLog (lsWARNING, UniqueNodeList) << boost::str (boost::format ("'%s' missing [" SECTION_VALIDATORS "].") + % theConfig.VALIDATORS_BASE); + } + } + + //-------------------------------------------------------------------------- + +private: + // VFALCO TODO Replace ptime with beast::Time + // Misc persistent information + boost::posix_time::ptime mtpScoreUpdated; + boost::posix_time::ptime mtpFetchUpdated; + + boost::recursive_mutex mUNLLock; + // XXX Make this faster, make this the contents vector unsigned char or raw public key. + // XXX Contents needs to based on score. + boost::unordered_set mUNL; + + boost::posix_time::ptime mtpScoreNext; // When to start scoring. + boost::posix_time::ptime mtpScoreStart; // Time currently started scoring. + DeadlineTimer m_scoreTimer; // Timer to start scoring. + + boost::mutex mFetchLock; + int mFetchActive; // Count of active fetches. + + boost::posix_time::ptime mtpFetchNext; // Time of to start next fetch. + DeadlineTimer m_fetchTimer; // Timer to start fetching. + + std::map m_clusterNodes; +}; + +UniqueNodeList* UniqueNodeList::New () { - boost::recursive_mutex::scoped_lock sl (mUNLLock); - - return mUNL.end () != mUNL.find (naNodePublic.humanNodePublic ()); + return new UniqueNodeListImp (); } - -bool UniqueNodeList::nodeInCluster (const RippleAddress& naNodePublic) -{ - boost::recursive_mutex::scoped_lock sl (mUNLLock); - return sClusterNodes.end () != sClusterNodes.find (naNodePublic); -} - -bool UniqueNodeList::nodeInCluster (const RippleAddress& naNodePublic, std::string& name) -{ - boost::recursive_mutex::scoped_lock sl (mUNLLock); - std::map::iterator it = sClusterNodes.find (naNodePublic); - - if (it == sClusterNodes.end ()) - return false; - - name = it->second; - return true; -} - -IUniqueNodeList* IUniqueNodeList::New (boost::asio::io_service& io_service) -{ - return new UniqueNodeList (io_service); -} - -// vim:ts=4 diff --git a/src/cpp/ripple/ripple_IUniqueNodeList.h b/src/cpp/ripple/ripple_UniqueNodeList.h similarity index 69% rename from src/cpp/ripple/ripple_IUniqueNodeList.h rename to src/cpp/ripple/ripple_UniqueNodeList.h index 827b586776..c77f8a00c0 100644 --- a/src/cpp/ripple/ripple_IUniqueNodeList.h +++ b/src/cpp/ripple/ripple_UniqueNodeList.h @@ -4,13 +4,13 @@ */ //============================================================================== -#ifndef RIPPLE_IUNIQUENODELIST_H -#define RIPPLE_IUNIQUENODELIST_H +#ifndef RIPPLE_UNIQUENODELIST_H_INCLUDED +#define RIPPLE_UNIQUENODELIST_H_INCLUDED -class IUniqueNodeList +class UniqueNodeList { public: - typedef enum + enum ValidatorSource { vsConfig = 'C', // rippled.cfg vsInbound = 'I', @@ -19,23 +19,23 @@ public: vsTold = 'T', vsValidator = 'V', // validators.txt vsWeb = 'W', - } validatorSource; + }; // VFALCO TODO rename this to use the right coding style typedef long score; public: // VFALCO TODO make this not use boost::asio... - static IUniqueNodeList* New (boost::asio::io_service& io_service); + static UniqueNodeList* New (); - virtual ~IUniqueNodeList () { } + virtual ~UniqueNodeList () { } // VFALCO TODO Roll this into the constructor so there is one less state. virtual void start () = 0; // VFALCO TODO rename all these, the "node" prefix is redundant (lol) - virtual void nodeAddPublic (const RippleAddress& naNodePublic, validatorSource vsWhy, const std::string& strComment) = 0; - virtual void nodeAddDomain (std::string strDomain, validatorSource vsWhy, const std::string& strComment = "") = 0; + virtual void nodeAddPublic (const RippleAddress& naNodePublic, ValidatorSource vsWhy, const std::string& strComment) = 0; + virtual void nodeAddDomain (std::string strDomain, ValidatorSource vsWhy, const std::string& strComment = "") = 0; virtual void nodeRemovePublic (const RippleAddress& naNodePublic) = 0; virtual void nodeRemoveDomain (std::string strDomain) = 0; virtual void nodeReset () = 0; @@ -45,6 +45,10 @@ public: virtual bool nodeInUNL (const RippleAddress& naNodePublic) = 0; virtual bool nodeInCluster (const RippleAddress& naNodePublic) = 0; virtual bool nodeInCluster (const RippleAddress& naNodePublic, std::string& name) = 0; + virtual bool nodeUpdate (const RippleAddress& naNodePublic, ClusterNodeStatus const& cnsStatus) = 0; + virtual std::map getClusterStatus () = 0; + virtual uint32 getClusterFee () = 0; + virtual void addClusterStatus (Json::Value&) = 0; virtual void nodeBootstrap () = 0; virtual bool nodeLoad (boost::filesystem::path pConfig) = 0; @@ -52,7 +56,7 @@ public: virtual Json::Value getUnlJson () = 0; - virtual int iSourceScore (validatorSource vsWhy) = 0; + virtual int iSourceScore (ValidatorSource vsWhy) = 0; }; #endif diff --git a/src/cpp/ripple/ripple_Validations.cpp b/src/cpp/ripple/ripple_Validations.cpp index f291741368..016c305a11 100644 --- a/src/cpp/ripple/ripple_Validations.cpp +++ b/src/cpp/ripple/ripple_Validations.cpp @@ -52,10 +52,10 @@ private: RippleAddress signer = val->getSignerPublic (); bool isCurrent = false; - if (theApp->getUNL ().nodeInUNL (signer) || val->isTrusted ()) + if (getApp().getUNL ().nodeInUNL (signer) || val->isTrusted ()) { val->setTrusted (); - uint32 now = theApp->getOPs ().getCloseTimeNC (); + uint32 now = getApp().getOPs ().getCloseTimeNC (); uint32 valClose = val->getSignTime (); if ((now > (valClose - LEDGER_EARLY_INTERVAL)) && (now < (valClose + LEDGER_VAL_INTERVAL))) @@ -104,7 +104,7 @@ private: << " added " << (val->isTrusted () ? "trusted/" : "UNtrusted/") << (isCurrent ? "current" : "stale"); if (val->isTrusted ()) - theApp->getLedgerMaster ().checkAccept (hash); + getApp().getLedgerMaster ().checkAccept (hash); // FIXME: This never forwards untrusted validations return isCurrent; @@ -136,7 +136,7 @@ private: if (set) { - uint32 now = theApp->getOPs ().getNetworkTimeNC (); + uint32 now = getApp().getOPs ().getNetworkTimeNC (); BOOST_FOREACH (u160_val_pair & it, *set) { bool isTrusted = it.second->isTrusted (); @@ -205,6 +205,36 @@ private: return trusted; } + int getFeeAverage (uint256 const& ledger, uint64 ref, uint64& fee) + { + int trusted = 0; + fee = 0; + + boost::mutex::scoped_lock sl (mValidationLock); + VSpointer set = findSet (ledger); + + if (set) + { + BOOST_FOREACH (u160_val_pair & it, *set) + { + if (it.second->isTrusted ()) + { + ++trusted; + if (it.second->isFieldPresent(sfLoadFee)) + fee += it.second->getFieldU32(sfLoadFee); + else + fee += ref; + } + } + } + + if (trusted == 0) + fee = ref; + else + fee /= trusted; + return trusted; + } + int getNodesAfter (uint256 const& ledger) { // Number of trusted nodes that have moved past this ledger @@ -241,7 +271,7 @@ private: std::list getCurrentTrustedValidations () { - uint32 cutoff = theApp->getOPs ().getNetworkTimeNC () - LEDGER_VAL_INTERVAL; + uint32 cutoff = getApp().getOPs ().getNetworkTimeNC () - LEDGER_VAL_INTERVAL; std::list ret; @@ -276,7 +306,7 @@ private: boost::unordered_map getCurrentValidations (uint256 currentLedger, uint256 priorLedger) { - uint32 cutoff = theApp->getOPs ().getNetworkTimeNC () - LEDGER_VAL_INTERVAL; + uint32 cutoff = getApp().getOPs ().getNetworkTimeNC () - LEDGER_VAL_INTERVAL; bool valCurrentLedger = currentLedger.isNonZero (); bool valPriorLedger = priorLedger.isNonZero (); @@ -307,7 +337,7 @@ private: (valPriorLedger && (it->second->getLedgerHash () == priorLedger)))) { countPreferred = true; - WriteLog (lsDEBUG, Validations) << "Counting for " << currentLedger << " not " << it->second->getLedgerHash (); + WriteLog (lsTRACE, Validations) << "Counting for " << currentLedger << " not " << it->second->getLedgerHash (); } currentValidationCount& p = countPreferred ? ret[currentLedger] : ret[it->second->getLedgerHash ()]; @@ -358,13 +388,13 @@ private: return; mWriting = true; - theApp->getJobQueue ().addJob (jtWRITE, "Validations::doWrite", + getApp().getJobQueue ().addJob (jtWRITE, "Validations::doWrite", BIND_TYPE (&Validations::doWrite, this, P_1)); } void doWrite (Job&) { - LoadEvent::autoptr event (theApp->getJobQueue ().getLoadEventAP (jtDISK, "ValidationWrite")); + LoadEvent::autoptr event (getApp().getJobQueue ().getLoadEventAP (jtDISK, "ValidationWrite")); boost::format insVal ("INSERT INTO Validations " "(LedgerHash,NodePubKey,SignTime,RawData) VALUES ('%s','%s','%u',%s);"); @@ -378,8 +408,8 @@ private: mStaleValidations.swap (vector); sl.unlock (); { - Database* db = theApp->getLedgerDB ()->getDB (); - ScopedLock dbl (theApp->getLedgerDB ()->getDBLock ()); + Database* db = getApp().getLedgerDB ()->getDB (); + ScopedLock dbl (getApp().getLedgerDB ()->getDBLock ()); Serializer s (1024); db->executeSQL ("BEGIN TRANSACTION;"); diff --git a/src/cpp/ripple/WSHandler.cpp b/src/cpp/ripple/ripple_WSHandler.cpp similarity index 80% rename from src/cpp/ripple/WSHandler.cpp rename to src/cpp/ripple/ripple_WSHandler.cpp index 3b7d744cc9..c004050fac 100644 --- a/src/cpp/ripple/WSHandler.cpp +++ b/src/cpp/ripple/ripple_WSHandler.cpp @@ -4,5 +4,4 @@ */ //============================================================================== - -SETUP_LOG (WSServerHandlerLog) +SETUP_LOGN (WSServerHandlerLog,"WSServerHandler") diff --git a/src/cpp/ripple/WSHandler.h b/src/cpp/ripple/ripple_WSHandler.h similarity index 91% rename from src/cpp/ripple/WSHandler.h rename to src/cpp/ripple/ripple_WSHandler.h index 1e72241502..583e102962 100644 --- a/src/cpp/ripple/WSHandler.h +++ b/src/cpp/ripple/ripple_WSHandler.h @@ -4,11 +4,8 @@ */ //============================================================================== -#ifndef __WSHANDLER__ -#define __WSHANDLER__ - -extern void initSSLContext (boost::asio::ssl::context& context, - std::string key_file, std::string cert_file, std::string chain_file); +#ifndef RIPPLE_WSHANDLER_H_INCLUDED +#define RIPPLE_WSHANDLER_H_INCLUDED extern bool serverOkay (std::string& reason); @@ -23,7 +20,9 @@ struct WSServerHandlerLog; // This instance dispatches all events. There is no per connection persistence. template -class WSServerHandler : public endpoint_type::handler +class WSServerHandler + : public endpoint_type::handler + , LeakChecked > { public: typedef typename endpoint_type::handler::connection_ptr connection_ptr; @@ -50,8 +49,11 @@ public: { if (theConfig.WEBSOCKET_SECURE != 0) { - initSSLContext (*mCtx, theConfig.WEBSOCKET_SSL_KEY, - theConfig.WEBSOCKET_SSL_CERT, theConfig.WEBSOCKET_SSL_CHAIN); + basio::SslContext::initializeFromFile ( + *mCtx, + theConfig.WEBSOCKET_SSL_KEY, + theConfig.WEBSOCKET_SSL_CERT, + theConfig.WEBSOCKET_SSL_CHAIN); } } @@ -77,7 +79,7 @@ public: } } - static void ssend (connection_ptr cpClient, const std::string& strMessage, bool broadcast) + static void ssendb (connection_ptr cpClient, const std::string& strMessage, bool broadcast) { try { @@ -93,14 +95,14 @@ public: void send (connection_ptr cpClient, message_ptr mpMessage) { - cpClient->get_strand ().post (boost::bind ( + cpClient->get_strand ().post (BIND_TYPE ( &WSServerHandler::ssend, cpClient, mpMessage)); } void send (connection_ptr cpClient, const std::string& strMessage, bool broadcast) { - cpClient->get_strand ().post (boost::bind ( - &WSServerHandler::ssend, cpClient, strMessage, broadcast)); + cpClient->get_strand ().post (BIND_TYPE ( + &WSServerHandler::ssendb, cpClient, strMessage, broadcast)); } void send (connection_ptr cpClient, const Json::Value& jvObj, bool broadcast) @@ -196,7 +198,7 @@ public: ptr->preDestroy (); // Must be done before we return // Must be done without holding the websocket send lock - theApp->getJobQueue ().addJob (jtCLIENT, "WSClient::destroy", + getApp().getJobQueue ().addJob (jtCLIENT, "WSClient::destroy", BIND_TYPE (&WSConnection::destroy, ptr)); } @@ -230,7 +232,7 @@ public: } if (bRunQ) - theApp->getJobQueue ().addJob (jtCLIENT, "WSClient::command", + getApp().getJobQueue ().addJob (jtCLIENT, "WSClient::command", BIND_TYPE (&WSServerHandler::do_messages, this, P_1, cpClient)); } @@ -257,7 +259,7 @@ public: do_message (job, cpClient, ptr, msg); } - theApp->getJobQueue ().addJob (jtCLIENT, "WSClient::more", + getApp().getJobQueue ().addJob (jtCLIENT, "WSClient::more", BIND_TYPE (&WSServerHandler::do_messages, this, P_1, cpClient)); } diff --git a/src/cpp/ripple/rpc.cpp b/src/cpp/ripple/rpc.cpp index df42e86a33..40fb0b32d2 100644 --- a/src/cpp/ripple/rpc.cpp +++ b/src/cpp/ripple/rpc.cpp @@ -5,15 +5,10 @@ //============================================================================== // Used for logging -struct RPC -{ -}; +struct RPC; SETUP_LOG (RPC) -using namespace boost; -using namespace boost::asio; - unsigned int const gMaxHTTPHeaderSize = 0x02000000; std::string gFormatStr ("v1"); diff --git a/test/account_set-test.js b/test/account_set-test.js index 65352c3524..be89b23259 100644 --- a/test/account_set-test.js +++ b/test/account_set-test.js @@ -7,8 +7,7 @@ var Request = require("ripple-lib").Request; var Server = require("./server").Server; var testutils = require("./testutils"); - -var config = require('ripple-lib').config.load(require('./config')); +var config = testutils.init_config(); // How long to wait for server to start. var serverDelay = 1500; diff --git a/test/account_tx-test.js b/test/account_tx-test.js index 6c7a8babec..56b81c64c4 100644 --- a/test/account_tx-test.js +++ b/test/account_tx-test.js @@ -6,9 +6,8 @@ var Remote = require("ripple-lib").Remote; var Transaction = require("ripple-lib").Transaction; var Server = require("./server").Server; -var testutils = require("./testutils"); - -require('ripple-lib').config.load(require('./config')); +var testutils = require("./testutils"); +var config = testutils.init_config(); buster.testRunner.timeout = 350000; //This is a very long test! diff --git a/test/buster.js b/test/buster.js index c445406541..a62cd49dc1 100644 --- a/test/buster.js +++ b/test/buster.js @@ -1,6 +1,6 @@ var config = module.exports; -config["Newcoin tests"] = { +config["Ripple tests"] = { rootPath: "../", environment: "node", tests: [ diff --git a/test/config-example.js b/test/config-example.js index 45bb945967..4191a745ca 100644 --- a/test/config-example.js +++ b/test/config-example.js @@ -7,11 +7,13 @@ var testconfig = require("./testconfig.js"); exports.accounts = testconfig.accounts; -// Where to find the binary. -exports.rippled = path.resolve("build/rippled"); - exports.server_default = "alpha"; +exports.default_server_config = { + // Where to find the binary. + rippled_path: path.resolve(__dirname, "../build/rippled") +}; + // // Configuration for servers. // @@ -33,6 +35,7 @@ exports.servers = { // 'validation_seed' : "shhDFVsmS2GSu5vUyZSPXYfj1r79h", // 'validators' : "n9L8LZZCwsdXzKUN9zoVxs4YznYXZ9hEhsQZY7aVpxtFaSceiyDZ beta", 'local_signing' : false, + 'node_db': 'type=Memory' } }; diff --git a/test/jsonrpc-test.js b/test/jsonrpc-test.js index c19c2636e1..7a6b9d61dd 100644 --- a/test/jsonrpc-test.js +++ b/test/jsonrpc-test.js @@ -10,7 +10,7 @@ var Server = require("./server").Server; var testutils = require("./testutils"); -var config = require('ripple-lib').config.load(require('./config')); +var config = testutils.init_config(); // How long to wait for server to start. var serverDelay = 1500; diff --git a/test/monitor-test.js b/test/monitor-test.js index 342dd7e4fb..e2ef06a738 100644 --- a/test/monitor-test.js +++ b/test/monitor-test.js @@ -6,8 +6,7 @@ var Remote = require("ripple-lib").Remote; var Server = require("./server").Server; var testutils = require("./testutils"); - -require('ripple-lib').config.load(require('./config')); +var config = testutils.init_config(); buster.testRunner.timeout = 5000; diff --git a/test/offer-test.js b/test/offer-test.js index 6d4af9584f..ed015ab7a6 100644 --- a/test/offer-test.js +++ b/test/offer-test.js @@ -7,9 +7,8 @@ var Remote = require("ripple-lib").Remote; var Transaction = require("ripple-lib").Transaction; var Server = require("./server").Server; -var testutils = require("./testutils"); - -require('ripple-lib').config.load(require('./config')); +var testutils = require("./testutils"); +var config = testutils.init_config(); buster.testRunner.timeout = 5000; @@ -67,7 +66,7 @@ buster.testCase("Offer tests", { } ], function (error) { // console.log("result: error=%s", error); - buster.refute(error); + buster.refute(error, self.what || "Unspecifide Error"); done(); }); @@ -95,7 +94,7 @@ buster.testCase("Offer tests", { buster.assert.equals('tesSUCCESS', m.metadata.TransactionResult); buster.assert(final_create); - if (3 === ++dones) + if (3 === ++dones) done(); }) .submit(); @@ -116,7 +115,7 @@ buster.testCase("Offer tests", { buster.assert.equals('tesSUCCESS', m.metadata.TransactionResult); buster.assert(final_create); - if (3 === ++dones) + if (3 === ++dones) done(); }) .submit(); @@ -146,7 +145,7 @@ buster.testCase("Offer tests", { buster.assert.equals('tesSUCCESS', m.metadata.TransactionResult); buster.assert(final_create); - if (3 === ++dones) + if (3 === ++dones) done(); }) .submit(); @@ -161,7 +160,7 @@ buster.testCase("Offer tests", { } ], function (error) { // console.log("result: error=%s", error); - buster.refute(error); + buster.refute(error, self.what); done(); }); @@ -207,21 +206,23 @@ buster.testCase("Offer tests", { function (done) { var self = this; + var alices_initial_balance = 499946999680; + var bobs_initial_balance = 10199999920; + async.waterfall([ function (callback) { self.what = "Create mtgox account."; - - testutils.payment(self.remote, "root", "mtgox", "1149999730", callback); + testutils.payment(self.remote, "root", "mtgox", 1149999730, callback); }, function (callback) { self.what = "Create alice account."; - testutils.payment(self.remote, "root", "alice", "499946999680", callback); + testutils.payment(self.remote, "root", "alice", alices_initial_balance, callback); }, function (callback) { self.what = "Create bob account."; - testutils.payment(self.remote, "root", "bob", "10199999920", callback); + testutils.payment(self.remote, "root", "bob", bobs_initial_balance, callback); }, function (callback) { self.what = "Set transfer rate."; @@ -286,7 +287,7 @@ buster.testCase("Offer tests", { callback); }, function (callback) { - self.what = "Verify balances."; + self.what = "Verify balances. 1"; testutils.verify_balances(self.remote, { @@ -308,12 +309,36 @@ buster.testCase("Offer tests", { .submit(); }, function (callback) { - self.what = "Verify balances."; + self.what = "Verify balances. 2"; + + var alices_fees, alices_num_transactions, alices_tx_fee_units_total, + alices_tx_fee_units_total, alices_final_balance, + + bobs_fees, bobs_num_transactions, bobs_tx_fee_units_total, + bobs_tx_fee_units_total, bobs_final_balance; + + alices_num_transactions = 3; + alices_tx_fee_units_total = alices_num_transactions * Transaction.fee_units["default"] + alices_tx_fees_total = self.remote.fee_tx(alices_tx_fee_units_total); + alices_final_balance = Amount.from_json(alices_initial_balance) + .subtract(alices_tx_fees_total); + + bobs_num_transactions = 2; + bobs_tx_fee_units_total = bobs_num_transactions * Transaction.fee_units["default"] + bobs_tx_fees_total = self.remote.fee_tx(bobs_tx_fee_units_total); + bobs_final_balance = Amount.from_json(bobs_initial_balance) + .subtract(bobs_tx_fees_total); testutils.verify_balances(self.remote, { - "alice" : [ "-50/USD/mtgox", String(499946999680-3*(Transaction.fees['default'].to_number())) ], - "bob" : [ "2710505431213761e-33/USD/mtgox", String(10199999920-2*(Transaction.fees['default'].to_number())) ], + "alice" : [ "-50/USD/mtgox", alices_final_balance.to_json()], + "bob" : [ "2710505431213761e-33/USD/mtgox", + + bobs_final_balance.to_json() + + // bobs_final_balance.to_json() + // String(10199999920-(self.remote.fee_tx(2*(Transaction.fee_units['default'])))).to_number() + ], }, callback); }, @@ -397,8 +422,9 @@ buster.testCase("Offer tests", { testutils.verify_balances(self.remote, { - "alice" : [ "499/USD/mtgox", String(100000000000+4000000000-2*(Transaction.fees['default'].to_number())) ], - "bob" : [ "1/USD/mtgox", String(100000000000-4000000000-2*(Transaction.fees['default'].to_number())) ], + // "bob" : [ "1/USD/mtgox", String(100000000000-4000000000-(Number(self.remote.fee_tx(Transaction.fee_units['default'] * 2).to_json()))) ], + "bob" : [ "1/USD/mtgox", String(100000000000-4000000000-(self.remote.fee_tx(Transaction.fee_units['default'] * 2).to_number())) ], + "alice" : [ "499/USD/mtgox", String(100000000000+4000000000-(self.remote.fee_tx(Transaction.fee_units['default'] * 2).to_number())) ], }, callback); }, @@ -482,8 +508,8 @@ buster.testCase("Offer tests", { testutils.verify_balances(self.remote, { - "alice" : [ "499/USD/mtgox", String(100000000000+3000000000-2*(Transaction.fees['default'].to_number())) ], - "bob" : [ "1/USD/mtgox", String(100000000000-3000000000-2*(Transaction.fees['default'].to_number())) ], + "alice" : [ "499/USD/mtgox", String(100000000000+3000000000-(self.remote.fee_tx(2*(Transaction.fee_units['default'])).to_number())) ], + "bob" : [ "1/USD/mtgox", String(100000000000-3000000000-(self.remote.fee_tx(2*(Transaction.fee_units['default'])).to_number())) ], }, callback); }, @@ -561,7 +587,7 @@ buster.testCase("Offer tests", { self.what = "Create crossing offer."; self.remote.transaction() - .offer_create("bob", "1/USD/mtgox", "3000.0") // + .offer_create("bob", "1/USD/mtgox", "3000.0") // .on('proposed', function (m) { // console.log("PROPOSED: offer_create: %s", JSON.stringify(m)); @@ -574,8 +600,8 @@ buster.testCase("Offer tests", { testutils.verify_balances(self.remote, { - "alice" : [ "499/USD/mtgox", String(100000000000+3000000000-2*(Transaction.fees['default'].to_number())) ], - "bob" : [ "1/USD/mtgox", String(100000000000-3000000000-1*(Transaction.fees['default'].to_number())) ], + "alice" : [ "499/USD/mtgox", String(100000000000+3000000000-(self.remote.fee_tx(2*(Transaction.fee_units['default'])).to_number())) ], + "bob" : [ "1/USD/mtgox", String(100000000000-3000000000-(self.remote.fee_tx(1*(Transaction.fee_units['default'])).to_number())) ], }, callback); }, @@ -676,7 +702,7 @@ buster.testCase("Offer tests", { }, ], function (error) { // console.log("result: error=%s", error); - buster.refute(error); + buster.refute(error, self.what); if (error) done(); }); @@ -771,7 +797,7 @@ buster.testCase("Offer tests", { }, ], function (error) { // console.log("result: error=%s", error); - buster.refute(error); + buster.refute(error, self.what); if (error) done(); }); }, @@ -849,7 +875,7 @@ buster.testCase("Offer tests", { } ], function (error) { // console.log("result: error=%s", error); - buster.refute(error); + buster.refute(error, self.what); done(); }); @@ -947,7 +973,7 @@ buster.testCase("Offer tests", { testutils.verify_balances(self.remote, { - "alice" : [ "0/USD/mtgox", String(10000000000+500-2*(Transaction.fees['default'].to_number())) ], + "alice" : [ "0/USD/mtgox", String(10000000000+500-(self.remote.fee_tx(2*(Transaction.fee_units['default'])).to_number())) ], "bob" : "100/USD/mtgox", }, callback); @@ -1110,7 +1136,7 @@ buster.testCase("Offer tests", { testutils.verify_balances(self.remote, { - "alice" : [ "160/USD/mtgox", String(10000000000+200-2*(Transaction.fees['default'].to_number())) ], + "alice" : [ "160/USD/mtgox", String(10000000000+200-(self.remote.fee_tx(2*(Transaction.fee_units['default'])).to_number())) ], "bob" : "40/USD/mtgox", }, callback); @@ -1152,7 +1178,7 @@ buster.testCase("Offer tests", { testutils.verify_balances(self.remote, { - "alice" : [ "100/USD/mtgox", String(10000000000+200+300-4*(Transaction.fees['default'].to_number())) ], + "alice" : [ "100/USD/mtgox", String(10000000000+200+300-(self.remote.fee_tx(4*(Transaction.fee_units['default'])).to_number())) ], "bob" : "100/USD/mtgox", }, callback); @@ -1456,15 +1482,43 @@ buster.testCase("Offer tests 3", { // Provide micro amounts to compensate for fees to make results round nice. self.what = "Create accounts."; - testutils.create_accounts(self.remote, "root", "350.000020", ["alice", "bob", "mtgox"], callback); + // Alice has 3 entries in the ledger, via trust lines + var max_owner_count = 3; // + // We start off with a + var reserve_amount = self.remote.reserve(max_owner_count); + // console.log("\n"); + // console.log("reserve_amount reserve(max_owner_count=%s): %s", max_owner_count, reserve_amount.to_human()); + + // this.tx_json.Fee = this.remote.fee_tx(this.fee_units()).to_json(); + + // 1 for each trust limit == 3 (alice < mtgox/amazon/bitstamp) + // 1 for payment == 4 + var max_txs_per_user = 4; + + // We don't have access to the tx object[s] created below so we + // just dig into fee_units straight away + var fee_units_for_all_txs = ( Transaction.fee_units["default"] * + max_txs_per_user ); + + starting_xrp = reserve_amount.add(self.remote.fee_tx(fee_units_for_all_txs)) + // console.log("starting_xrp after %s fee units: ", fee_units_for_all_txs, starting_xrp.to_human()); + + starting_xrp = starting_xrp.add(Amount.from_json('100.0')); + // console.log("starting_xrp adding 100 xrp to sell", starting_xrp.to_human()); + + testutils.create_accounts(self.remote, + "root", + starting_xrp.to_json(), + ["alice", "bob", "mtgox", "amazon", "bitstamp"], + callback); }, function (callback) { self.what = "Set limits."; testutils.credit_limits(self.remote, { - "alice" : "1000/USD/mtgox", - "bob" : "1000/USD/mtgox", + "alice" : ["1000/USD/mtgox", "1000/USD/amazon","1000/USD/bitstamp"], + "bob" : ["1000/USD/mtgox", "1000/USD/amazon"], }, callback); }, @@ -1485,7 +1539,6 @@ buster.testCase("Offer tests 3", { .on('proposed', function (m) { // console.log("proposed: offer_create: %s", json.stringify(m)); callback(m.result !== 'tesSUCCESS'); - seq_carol = m.tx_json.sequence; }) .submit(); @@ -1521,14 +1574,14 @@ buster.testCase("Offer tests 3", { testutils.verify_balances(self.remote, { - "alice" : [ "100/USD/mtgox", "250.0" ], - "bob" : "400/USD/mtgox", + "alice" : [ "100/USD/mtgox", "350.0"], + "bob" : ["400/USD/mtgox", ], }, callback); }, ], function (error) { // console.log("result: error=%s", error); - buster.refute(error); + buster.refute(error, self.what); done(); }); @@ -1635,7 +1688,7 @@ buster.testCase("Offer tests 3", { }, ], function (error) { // console.log("result: error=%s", error); - buster.refute(error); + buster.refute(error, self.what); done(); }); @@ -1651,14 +1704,16 @@ buster.testCase("Offer tfSell", { "basic sell" : function (done) { var self = this; - var final_create; + var final_create, seq_carol; async.waterfall([ function (callback) { // Provide micro amounts to compensate for fees to make results round nice. self.what = "Create accounts."; - testutils.create_accounts(self.remote, "root", "350.000020", ["alice", "bob", "mtgox"], callback); + var req_amount = self.remote.reserve(1).add(self.remote.fee_tx(20)).add(100000000); + testutils.create_accounts(self.remote, "root", req_amount.to_json(), + ["alice", "bob", "mtgox"], callback); }, function (callback) { self.what = "Set limits."; @@ -1687,19 +1742,21 @@ buster.testCase("Offer tfSell", { .set_flags('Sell') // Should not matter at all. .on('proposed', function (m) { // console.log("proposed: offer_create: %s", json.stringify(m)); - callback(m.result !== 'tesSUCCESS'); + if (m.result !== 'tesSUCCESS') { + throw new Error("Bob's OfferCreate tx did not succeed: "+m.result); + } else callback(null); seq_carol = m.tx_json.sequence; }) .submit(); }, function (callback) { - // Alice has 350 fees - a reserve of 50 = 250 reserve = 100 available. + // Alice has 350 + fees - a reserve of 50 = 250 reserve = 100 available. // Ask for more than available to prove reserve works. self.what = "Create offer alice."; self.remote.transaction() - .offer_create("alice", "100/USD/mtgox", "100.0") + .offer_create("alice", "200/USD/mtgox", "200.0") .set_flags('Sell') // Should not matter at all. .on('proposed', function (m) { // console.log("proposed: offer_create: %s", json.stringify(m)); @@ -1732,7 +1789,7 @@ buster.testCase("Offer tfSell", { }, ], function (error) { // console.log("result: error=%s", error); - buster.refute(error); + buster.refute(error, self.what); done(); }); @@ -1741,14 +1798,19 @@ buster.testCase("Offer tfSell", { "2x sell exceed limit" : function (done) { var self = this; - var final_create; + var final_create, seq_carol; async.waterfall([ function (callback) { // Provide micro amounts to compensate for fees to make results round nice. self.what = "Create accounts."; - - testutils.create_accounts(self.remote, "root", "350.000020", ["alice", "bob", "mtgox"], callback); + + var starting_xrp = self.amount_for({ + ledger_entries: 1, + default_transactions: 2, + extra: '100.0' + }); + testutils.create_accounts(self.remote, "root", starting_xrp, ["alice", "bob", "mtgox"], callback); }, function (callback) { self.what = "Set limits."; @@ -1779,7 +1841,6 @@ buster.testCase("Offer tfSell", { .on('proposed', function (m) { // console.log("proposed: offer_create: %s", json.stringify(m)); callback(m.result !== 'tesSUCCESS'); - seq_carol = m.tx_json.sequence; }) .submit(); @@ -1797,7 +1858,9 @@ buster.testCase("Offer tfSell", { .set_flags('Sell') .on('proposed', function (m) { // console.log("proposed: offer_create: %s", json.stringify(m)); - callback(m.result !== 'tesSUCCESS'); + if (m.result !== 'tesSUCCESS') { + callback(new Error("Alice's OfferCreate didn't succeed: "+m.result)); + } else callback(null); seq_carol = m.tx_json.sequence; }) @@ -1826,7 +1889,7 @@ buster.testCase("Offer tfSell", { }, ], function (error) { // console.log("result: error=%s", error); - buster.refute(error); + buster.refute(error, self.what); done(); }); @@ -1848,8 +1911,14 @@ buster.testCase("Client Issue #535", { function (callback) { // Provide micro amounts to compensate for fees to make results round nice. self.what = "Create accounts."; + + var starting_xrp = self.amount_for({ + ledger_entries: 1, + default_transactions: 2, + extra: '100.0' + }); - testutils.create_accounts(self.remote, "root", "350.000020", ["alice", "bob", "mtgox"], callback); + testutils.create_accounts(self.remote, "root", starting_xrp, ["alice", "bob", "mtgox"], callback); }, function (callback) { self.what = "Set limits."; @@ -1922,7 +1991,7 @@ buster.testCase("Client Issue #535", { ], function (error) { if (error) console.log("result: %s: error=%s", self.what, error); - buster.refute(error); + buster.refute(error, self.what); done(); }); diff --git a/test/path-test.js b/test/path-test.js index 31aea5f4a6..3616122b3b 100644 --- a/test/path-test.js +++ b/test/path-test.js @@ -6,9 +6,9 @@ var Remote = require("ripple-lib").Remote; var Transaction = require("ripple-lib").Transaction; var Server = require("./server").Server; -var testutils = require("./testutils"); +var testutils = require("./testutils"); +var config = testutils.init_config(); -require('ripple-lib').config.load(require('./config')); buster.testRunner.timeout = 5000; diff --git a/test/remote-test.js b/test/remote-test.js index 841f3df0d0..0b390e3005 100644 --- a/test/remote-test.js +++ b/test/remote-test.js @@ -5,8 +5,7 @@ var Remote = require("ripple-lib").Remote; var Server = require("./server.js").Server; var testutils = require("./testutils.js"); - -var config = require('ripple-lib').config.load(require('./config')); +var config = testutils.init_config(); // How long to wait for server to start. var serverDelay = 1500; // XXX Not implemented. diff --git a/test/runall.sh b/test/runall.sh new file mode 100644 index 0000000000..e0c25f2fe1 --- /dev/null +++ b/test/runall.sh @@ -0,0 +1,5 @@ +#!/bin/bash +for f in test/*.js +do + node $f +done diff --git a/test/send-test.js b/test/send-test.js index dd98c41bc4..15d363ddf5 100644 --- a/test/send-test.js +++ b/test/send-test.js @@ -6,8 +6,7 @@ var Remote = require("ripple-lib").Remote; var Server = require("./server").Server; var testutils = require("./testutils"); - -var config = require('ripple-lib').config.load(require('./config')); +var config = testutils.init_config(); // How long to wait for server to start. var serverDelay = 1500; diff --git a/test/server-test.js b/test/server-test.js index 434563c166..b2800e1d31 100644 --- a/test/server-test.js +++ b/test/server-test.js @@ -1,6 +1,9 @@ -var buster = require("buster"); +var buster = require("buster"); +var extend = require("extend"); +var Server = require("./server").Server; + var testutils = require("./testutils"); -var Server = require("./server").Server; +var config = testutils.init_config(); // How long to wait for server to start. // var serverDelay = 1500; @@ -9,19 +12,22 @@ var alpha; buster.testCase("Standalone server startup", { "server start and stop" : function (done) { - alpha = Server.from_config("alpha", false); //ADD ,true for verbosity + var cfg = extend({}, config.default_server_config, + config.servers.alpha); - alpha - .on('started', function () { - alpha - .on('stopped', function () { - buster.assert(true); + alpha = Server.from_config("alpha", cfg); - done(); - }) - .stop(); + alpha + .on('started', function () { + alpha + .on('stopped', function () { + buster.assert(true); + + done(); }) - .start(); + .stop(); + }) + .start(); } }); diff --git a/test/server.js b/test/server.js index f47189cafd..cbdf68475d 100644 --- a/test/server.js +++ b/test/server.js @@ -22,7 +22,6 @@ var path = require("path"); var util = require("util"); var EventEmitter = require('events').EventEmitter; -var config = require("./config"); var nodeutils = require("./nodeutils"); // Create a server object @@ -58,8 +57,8 @@ var Server = function (name, config, verbose) { util.inherits(Server, EventEmitter); -Server.from_config = function (name, verbose) { - return new Server(name, config.servers[name], verbose); +Server.from_config = function (name, config, verbose) { + return new Server(name, config, verbose); }; Server.prototype.on = function (e, c) { @@ -106,7 +105,7 @@ Server.prototype._serverSpawnSync = function() { // Spawn in standalone mode for now. this.child = child.spawn( - config.rippled, + this.config.rippled_path, args, { cwd: this.serverPath(), @@ -116,7 +115,10 @@ Server.prototype._serverSpawnSync = function() { if (!this.quiet) console.log("server: start %s: %s --conf=%s", - this.child.pid, config.rippled, args.join(" "), this.configPath()); + this.child.pid, + this.config.rippled_path, + args.join(" "), + this.configPath()); // By default, just log exits. this.child.on('exit', function(code, signal) { diff --git a/test/testutils.js b/test/testutils.js index 53bc3c8560..0e06d28992 100644 --- a/test/testutils.js +++ b/test/testutils.js @@ -1,10 +1,10 @@ -var async = require("async"); +var async = require("async"); +var extend = require("extend"); -var Amount = require("ripple-lib").Amount; -var Remote = require("ripple-lib").Remote; -var Server = require("./server").Server; - -var config = require('ripple-lib').config.load(require('./config')); +var Amount = require("ripple-lib").Amount; +var Remote = require("ripple-lib").Remote; +var Server = require("./server").Server; +var Transaction = require("ripple-lib").Transaction; var account_dump = function (remote, account, callback) { var self = this; @@ -64,6 +64,8 @@ var account_dump = function (remote, account, callback) { * @param host {String} Identifier for the host configuration to be used. */ var build_setup = function (opts, host) { + var config = get_config(); + opts = opts || {}; // Normalize options @@ -74,6 +76,18 @@ var build_setup = function (opts, host) { return function (done) { var self = this; + + self.compute_fees_amount_for_txs = function(txs) { + var fee_units = Transaction.fee_units["default"] * txs; + return self.remote.fee_tx(fee_units); + }; + + self.amount_for = function(options) { + var reserve = self.remote.reserve(options.ledger_entries || 0); + var fees = self.compute_fees_amount_for_txs(options.default_transactions || 0) + return reserve.add(fees) + .add(options.extra || 0); + }; host = host || config.server_default; @@ -87,8 +101,12 @@ var build_setup = function (opts, host) { function runServerStep(callback) { if (opts.no_server) return callback(); + var server_config = extend({}, config.default_server_config, + config.servers[host]); + data.server = Server - .from_config(host, !!opts.verbose_server) + .from_config(host, server_config, + !!opts.verbose_server) .on('started', callback) .on('exited', function () { // If know the remote, tell it server is gone. @@ -114,6 +132,7 @@ var build_setup = function (opts, host) { * @param host {String} Identifier for the host configuration to be used. */ var build_teardown = function (host) { + var config = get_config(); return function (done) { host = host || config.server_default; @@ -160,7 +179,9 @@ var create_accounts = function (remote, src, amount, accounts, callback) { .on('proposed', function (m) { // console.log("proposed: %s", JSON.stringify(m)); - callback(m.result != 'tesSUCCESS'); + if (m.result != 'tesSUCCESS') { + callback(new Error("Payment to create account did not succeed.")); + } else callback(null); }) .on('error', function (m) { // console.log("error: %s", JSON.stringify(m)); @@ -203,6 +224,24 @@ var credit_limit = function (remote, src, amount, callback) { } }; +function get_config() { + var cfg = require('./config-example'); + + // See if the person testing wants to override the configuration by creating a + // file called test/config.js. + try { + cfg = extend({}, cfg, require('./config')); + } catch (e) { } + + return cfg; +} + +function init_config() { + var cfg = get_config(); + + return require('ripple-lib').config.load(cfg); +} + var verify_limit = function (remote, src, amount, callback) { assert(4 === arguments.length); @@ -214,6 +253,7 @@ var verify_limit = function (remote, src, amount, callback) { } else { + // console.log("_m", _m.length, _m); // console.log("verify_limit: parsed: %s", JSON.stringify(_m, undefined, 2)); var _account_limit = _m[1]; var _quality_in = _m[2]; @@ -470,6 +510,8 @@ exports.build_teardown = build_teardown; exports.create_accounts = create_accounts; exports.credit_limit = credit_limit; exports.credit_limits = credit_limits; +exports.get_config = get_config; +exports.init_config = init_config; exports.ledger_close = ledger_close; exports.payment = payment; exports.payments = payments; diff --git a/test/websocket-test.js b/test/websocket-test.js index e0a9704984..2415288f02 100644 --- a/test/websocket-test.js +++ b/test/websocket-test.js @@ -1,18 +1,35 @@ -var buster = require("buster"); +var buster = require("buster"); +var extend = require("extend"); -var Server = require("./server").Server; -var Remote = require("ripple-lib").Remote; +var Server = require("./server").Server; +var Remote = require("ripple-lib").Remote; -var config = require('ripple-lib').config.load(require('./config')); +var testutils = require('./testutils'); +var config = testutils.init_config(); buster.testRunner.timeout = 5000; +var server; buster.testCase("WebSocket connection", { 'setUp' : - function (done) { if (config.servers.alpha.no_server) done(); else server = Server.from_config("alpha").on('started', done).start(); }, + function (done) { + var cfg = extend({}, config.default_server_config, + config.servers.alpha); + if (cfg.no_server) { + done(); + } else { + server = Server.from_config("alpha", cfg).on('started', done).start(); + } + }, 'tearDown' : - function (done) { if (config.servers.alpha.no_server) done(); else server.on('stopped', done).stop(); }, + function (done) { + if (config.servers.alpha.no_server) { + done(); + } else { + server.on('stopped', done).stop(); + } + }, "websocket connect and disconnect" : function (done) {