mirror of
https://github.com/XRPLF/rippled.git
synced 2025-11-20 11:05:54 +00:00
Update snappy to 1.1.7
This commit is contained in:
41
src/snappy/snappy/.appveyor.yml
Normal file
41
src/snappy/snappy/.appveyor.yml
Normal file
@@ -0,0 +1,41 @@
|
||||
# Build matrix / environment variables are explained on:
|
||||
# https://www.appveyor.com/docs/appveyor-yml/
|
||||
# This file can be validated on: https://ci.appveyor.com/tools/validate-yaml
|
||||
|
||||
version: "{build}"
|
||||
|
||||
environment:
|
||||
matrix:
|
||||
# AppVeyor currently has no custom job name feature.
|
||||
# http://help.appveyor.com/discussions/questions/1623-can-i-provide-a-friendly-name-for-jobs
|
||||
- JOB: Visual Studio 2017
|
||||
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
|
||||
CMAKE_GENERATOR: Visual Studio 15 2017
|
||||
- JOB: Visual Studio 2015
|
||||
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
|
||||
CMAKE_GENERATOR: Visual Studio 14 2015
|
||||
|
||||
platform:
|
||||
- x86
|
||||
- x64
|
||||
|
||||
configuration:
|
||||
- RelWithDebInfo
|
||||
- Debug
|
||||
|
||||
build:
|
||||
verbosity: minimal
|
||||
|
||||
build_script:
|
||||
- git submodule update --init --recursive
|
||||
- mkdir out
|
||||
- cd out
|
||||
- if "%platform%"=="x64" set CMAKE_GENERATOR=%CMAKE_GENERATOR% Win64
|
||||
- cmake --version
|
||||
- cmake .. -G "%CMAKE_GENERATOR%"
|
||||
-DCMAKE_CONFIGURATION_TYPES="%CONFIGURATION%"
|
||||
- cmake --build . --config %CONFIGURATION%
|
||||
- cd ..
|
||||
|
||||
test_script:
|
||||
- out\%CONFIGURATION%\snappy_unittest
|
||||
59
src/snappy/snappy/.travis.yml
Normal file
59
src/snappy/snappy/.travis.yml
Normal file
@@ -0,0 +1,59 @@
|
||||
# Build matrix / environment variables are explained on:
|
||||
# http://about.travis-ci.org/docs/user/build-configuration/
|
||||
# This file can be validated on: http://lint.travis-ci.org/
|
||||
|
||||
sudo: false
|
||||
dist: trusty
|
||||
language: cpp
|
||||
|
||||
compiler:
|
||||
- gcc
|
||||
- clang
|
||||
os:
|
||||
- linux
|
||||
- osx
|
||||
|
||||
env:
|
||||
- BUILD_TYPE=Debug
|
||||
- BUILD_TYPE=RelWithDebInfo
|
||||
|
||||
matrix:
|
||||
allow_failures:
|
||||
- compiler: clang
|
||||
env: BUILD_TYPE=RelWithDebInfo
|
||||
|
||||
addons:
|
||||
apt:
|
||||
# List of whitelisted in travis packages for ubuntu-trusty can be found here:
|
||||
# https://github.com/travis-ci/apt-package-whitelist/blob/master/ubuntu-trusty
|
||||
# List of whitelisted in travis apt-sources:
|
||||
# https://github.com/travis-ci/apt-source-whitelist/blob/master/ubuntu.json
|
||||
sources:
|
||||
- ubuntu-toolchain-r-test
|
||||
- llvm-toolchain-trusty-4.0
|
||||
packages:
|
||||
- cmake
|
||||
- gcc-6
|
||||
- g++-6
|
||||
- clang-4.0
|
||||
|
||||
install:
|
||||
# Travis doesn't have a nice way to install homebrew packages yet.
|
||||
# https://github.com/travis-ci/travis-ci/issues/5377
|
||||
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew update; fi
|
||||
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install gcc@6; fi
|
||||
# /usr/bin/gcc is stuck to old versions by on both Linux and OSX.
|
||||
- if [ "$CXX" = "g++" ]; then export CXX="g++-6" CC="gcc-6"; fi
|
||||
- echo ${CC}
|
||||
- echo ${CXX}
|
||||
- ${CXX} --version
|
||||
- cmake --version
|
||||
|
||||
before_script:
|
||||
- mkdir -p build && cd build
|
||||
- cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE
|
||||
- cmake --build .
|
||||
- cd ..
|
||||
|
||||
script:
|
||||
- build/snappy_unittest
|
||||
174
src/snappy/snappy/CMakeLists.txt
Normal file
174
src/snappy/snappy/CMakeLists.txt
Normal file
@@ -0,0 +1,174 @@
|
||||
cmake_minimum_required(VERSION 3.1)
|
||||
project(Snappy VERSION 1.1.7 LANGUAGES C CXX)
|
||||
|
||||
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make
|
||||
# it prominent in the GUI.
|
||||
option(BUILD_SHARED_LIBS "Build shared libraries(DLLs)." OFF)
|
||||
|
||||
option(SNAPPY_BUILD_TESTS "Build Snappy's own tests." ON)
|
||||
|
||||
include(TestBigEndian)
|
||||
test_big_endian(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
include(CheckIncludeFile)
|
||||
check_include_file("byteswap.h" HAVE_BYTESWAP_H)
|
||||
check_include_file("stddef.h" HAVE_STDDEF_H)
|
||||
check_include_file("stdint.h" HAVE_STDINT_H)
|
||||
check_include_file("sys/endian.h" HAVE_SYS_ENDIAN_H)
|
||||
check_include_file("sys/mman.h" HAVE_SYS_MMAN_H)
|
||||
check_include_file("sys/resource.h" HAVE_SYS_RESOURCE_H)
|
||||
check_include_file("sys/time.h" HAVE_SYS_TIME_H)
|
||||
check_include_file("sys/uio.h" HAVE_SYS_UIO_H)
|
||||
check_include_file("unistd.h" HAVE_UNISTD_H)
|
||||
check_include_file("windows.h" HAVE_WINDOWS_H)
|
||||
|
||||
include(CheckLibraryExists)
|
||||
check_library_exists(z zlibVersion "" HAVE_LIBZ)
|
||||
check_library_exists(lzo2 lzo1x_1_15_compress "" HAVE_LIBLZO2)
|
||||
|
||||
include(CheckCXXSourceCompiles)
|
||||
check_cxx_source_compiles(
|
||||
"int main(void) { return __builtin_expect(0, 1); }" HAVE_BUILTIN_EXPECT)
|
||||
|
||||
check_cxx_source_compiles(
|
||||
"int main(void) { return __builtin_ctzll(0); }" HAVE_BUILTIN_CTZ)
|
||||
|
||||
include(CheckSymbolExists)
|
||||
check_symbol_exists("mmap" "sys/mman.h" HAVE_FUNC_MMAP)
|
||||
check_symbol_exists("sysconf" "unistd.h" HAVE_FUNC_SYSCONF)
|
||||
|
||||
find_package(GTest QUIET)
|
||||
if(GTEST_FOUND)
|
||||
set(HAVE_GTEST 1)
|
||||
endif(GTEST_FOUND)
|
||||
|
||||
find_package(Gflags QUIET)
|
||||
if(GFLAGS_FOUND)
|
||||
set(HAVE_GFLAGS 1)
|
||||
endif(GFLAGS_FOUND)
|
||||
|
||||
configure_file(
|
||||
"${PROJECT_SOURCE_DIR}/cmake/config.h.in"
|
||||
"${PROJECT_BINARY_DIR}/config.h"
|
||||
)
|
||||
|
||||
# We don't want to define HAVE_ macros in public headers. Instead, we use
|
||||
# CMake's variable substitution with 0/1 variables, which will be seen by the
|
||||
# preprocessor as constants.
|
||||
set(HAVE_STDINT_H_01 ${HAVE_STDINT_H})
|
||||
set(HAVE_STDDEF_H_01 ${HAVE_STDDEF_H})
|
||||
set(HAVE_SYS_UIO_H_01 ${HAVE_SYS_UIO_H})
|
||||
if(NOT HAVE_STDINT_H_01)
|
||||
set(HAVE_STDINT_H_01 0)
|
||||
endif(NOT HAVE_STDINT_H_01)
|
||||
if(NOT HAVE_STDDEF_H_01)
|
||||
set(HAVE_STDDEF_H_01 0)
|
||||
endif(NOT HAVE_STDDEF_H_01)
|
||||
if(NOT HAVE_SYS_UIO_H_01)
|
||||
set(HAVE_SYS_UIO_H_01 0)
|
||||
endif(NOT HAVE_SYS_UIO_H_01)
|
||||
|
||||
configure_file(
|
||||
"${PROJECT_SOURCE_DIR}/snappy-stubs-public.h.in"
|
||||
"${PROJECT_BINARY_DIR}/snappy-stubs-public.h")
|
||||
|
||||
add_library(snappy "")
|
||||
target_sources(snappy
|
||||
PRIVATE
|
||||
"${PROJECT_SOURCE_DIR}/snappy-internal.h"
|
||||
"${PROJECT_SOURCE_DIR}/snappy-stubs-internal.h"
|
||||
"${PROJECT_SOURCE_DIR}/snappy-c.cc"
|
||||
"${PROJECT_SOURCE_DIR}/snappy-sinksource.cc"
|
||||
"${PROJECT_SOURCE_DIR}/snappy-stubs-internal.cc"
|
||||
"${PROJECT_SOURCE_DIR}/snappy.cc"
|
||||
"${PROJECT_BINARY_DIR}/config.h"
|
||||
|
||||
# Only CMake 3.3+ supports PUBLIC sources in targets exported by "install".
|
||||
$<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC>
|
||||
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-c.h>
|
||||
$<INSTALL_INTERFACE:include/snappy-c.h>
|
||||
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-sinksource.h>
|
||||
$<INSTALL_INTERFACE:include/snappy-sinksource.h>
|
||||
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy.h>
|
||||
$<INSTALL_INTERFACE:include/snappy.h>
|
||||
$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/snappy-stubs-public.h>
|
||||
$<INSTALL_INTERFACE:include/snappy-stubs-public.h>
|
||||
)
|
||||
target_include_directories(snappy
|
||||
PUBLIC
|
||||
$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>
|
||||
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>
|
||||
$<INSTALL_INTERFACE:include>
|
||||
)
|
||||
set_target_properties(snappy
|
||||
PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
|
||||
|
||||
target_compile_definitions(snappy PRIVATE -DHAVE_CONFIG_H)
|
||||
if(BUILD_SHARED_LIBS)
|
||||
set_target_properties(snappy PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
|
||||
endif(BUILD_SHARED_LIBS)
|
||||
|
||||
if(SNAPPY_BUILD_TESTS)
|
||||
enable_testing()
|
||||
|
||||
add_executable(snappy_unittest "")
|
||||
target_sources(snappy_unittest
|
||||
PRIVATE
|
||||
"${PROJECT_SOURCE_DIR}/snappy_unittest.cc"
|
||||
"${PROJECT_SOURCE_DIR}/snappy-test.cc"
|
||||
)
|
||||
target_compile_definitions(snappy_unittest PRIVATE -DHAVE_CONFIG_H)
|
||||
target_link_libraries(snappy_unittest snappy ${GFLAGS_LIBRARIES})
|
||||
|
||||
if(HAVE_LIBZ)
|
||||
target_link_libraries(snappy_unittest z)
|
||||
endif(HAVE_LIBZ)
|
||||
if(HAVE_LIBLZO2)
|
||||
target_link_libraries(snappy_unittest lzo2)
|
||||
endif(HAVE_LIBLZO2)
|
||||
|
||||
target_include_directories(snappy_unittest
|
||||
BEFORE PRIVATE
|
||||
"${PROJECT_SOURCE_DIR}"
|
||||
"${GTEST_INCLUDE_DIRS}"
|
||||
"${GFLAGS_INCLUDE_DIRS}"
|
||||
)
|
||||
|
||||
add_test(
|
||||
NAME snappy_unittest
|
||||
WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
|
||||
COMMAND "${PROJECT_BINARY_DIR}/snappy_unittest")
|
||||
endif(SNAPPY_BUILD_TESTS)
|
||||
|
||||
include(GNUInstallDirs)
|
||||
install(TARGETS snappy
|
||||
EXPORT SnappyTargets
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
)
|
||||
install(
|
||||
FILES
|
||||
"${PROJECT_SOURCE_DIR}/snappy-c.h"
|
||||
"${PROJECT_SOURCE_DIR}/snappy-sinksource.h"
|
||||
"${PROJECT_SOURCE_DIR}/snappy.h"
|
||||
"${PROJECT_BINARY_DIR}/snappy-stubs-public.h"
|
||||
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
|
||||
)
|
||||
|
||||
include(CMakePackageConfigHelpers)
|
||||
write_basic_package_version_file(
|
||||
"${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake"
|
||||
COMPATIBILITY SameMajorVersion
|
||||
)
|
||||
install(
|
||||
EXPORT SnappyTargets
|
||||
NAMESPACE Snappy::
|
||||
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy"
|
||||
)
|
||||
install(
|
||||
FILES
|
||||
"${PROJECT_SOURCE_DIR}/cmake/SnappyConfig.cmake"
|
||||
"${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake"
|
||||
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy"
|
||||
)
|
||||
26
src/snappy/snappy/CONTRIBUTING.md
Normal file
26
src/snappy/snappy/CONTRIBUTING.md
Normal file
@@ -0,0 +1,26 @@
|
||||
# How to Contribute
|
||||
|
||||
We'd love to accept your patches and contributions to this project. There are
|
||||
just a few small guidelines you need to follow.
|
||||
|
||||
## Contributor License Agreement
|
||||
|
||||
Contributions to this project must be accompanied by a Contributor License
|
||||
Agreement. You (or your employer) retain the copyright to your contribution,
|
||||
this simply gives us permission to use and redistribute your contributions as
|
||||
part of the project. Head over to <https://cla.developers.google.com/> to see
|
||||
your current agreements on file or to sign a new one.
|
||||
|
||||
You generally only need to submit a CLA once, so if you've already submitted one
|
||||
(even if it was for a different project), you probably don't need to do it
|
||||
again.
|
||||
|
||||
## Code reviews
|
||||
|
||||
All submissions, including submissions by project members, require review. We
|
||||
use GitHub pull requests for this purpose. Consult
|
||||
[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
|
||||
information on using pull requests.
|
||||
|
||||
Please make sure that all the automated checks (CLA, AppVeyor, Travis) pass for
|
||||
your pull requests. Pull requests whose checks fail may be ignored.
|
||||
@@ -29,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
===
|
||||
|
||||
Some of the benchmark data in util/zippy/testdata is licensed differently:
|
||||
Some of the benchmark data in testdata/ is licensed differently:
|
||||
|
||||
- fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and
|
||||
is licensed under the Creative Commons Attribution 3.0 license
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,23 +0,0 @@
|
||||
ACLOCAL_AMFLAGS = -I m4
|
||||
|
||||
# Library.
|
||||
lib_LTLIBRARIES = libsnappy.la
|
||||
libsnappy_la_SOURCES = snappy.cc snappy-sinksource.cc snappy-stubs-internal.cc snappy-c.cc
|
||||
libsnappy_la_LDFLAGS = -version-info $(SNAPPY_LTVERSION)
|
||||
|
||||
include_HEADERS = snappy.h snappy-sinksource.h snappy-stubs-public.h snappy-c.h
|
||||
noinst_HEADERS = snappy-internal.h snappy-stubs-internal.h snappy-test.h
|
||||
|
||||
# Unit tests and benchmarks.
|
||||
snappy_unittest_CPPFLAGS = $(gflags_CFLAGS) $(GTEST_CPPFLAGS)
|
||||
snappy_unittest_SOURCES = snappy_unittest.cc snappy-test.cc
|
||||
snappy_unittest_LDFLAGS = $(GTEST_LDFLAGS)
|
||||
snappy_unittest_LDADD = libsnappy.la $(UNITTEST_LIBS) $(gflags_LIBS) $(GTEST_LIBS)
|
||||
TESTS = snappy_unittest
|
||||
noinst_PROGRAMS = $(TESTS)
|
||||
|
||||
EXTRA_DIST = autogen.sh testdata/alice29.txt testdata/asyoulik.txt testdata/baddata1.snappy testdata/baddata2.snappy testdata/baddata3.snappy testdata/geo.protodata testdata/fireworks.jpeg testdata/html testdata/html_x_4 testdata/kppkn.gtb testdata/lcet10.txt testdata/paper-100k.pdf testdata/plrabn12.txt testdata/urls.10K
|
||||
dist_doc_DATA = ChangeLog COPYING INSTALL NEWS README format_description.txt framing_format.txt
|
||||
|
||||
libtool: $(LIBTOOL_DEPS)
|
||||
$(SHELL) ./config.status --recheck
|
||||
@@ -1,3 +1,55 @@
|
||||
Snappy v1.1.7, August 24th 2017:
|
||||
|
||||
* Improved CMake build support for 64-bit Linux distributions.
|
||||
|
||||
* MSVC builds now use MSVC-specific intrinsics that map to clzll.
|
||||
|
||||
* ARM64 (AArch64) builds use the code paths optimized for 64-bit processors.
|
||||
|
||||
Snappy v1.1.6, July 12th 2017:
|
||||
|
||||
This is a re-release of v1.1.5 with proper SONAME / SOVERSION values.
|
||||
|
||||
Snappy v1.1.5, June 28th 2017:
|
||||
|
||||
This release has broken SONAME / SOVERSION values. Users of snappy as a shared
|
||||
library should avoid 1.1.5 and use 1.1.6 instead. SONAME / SOVERSION errors will
|
||||
manifest as the dynamic library loader complaining that it cannot find snappy's
|
||||
shared library file (libsnappy.so / libsnappy.dylib), or that the library it
|
||||
found does not have the required version. 1.1.6 has the same code as 1.1.5, but
|
||||
carries build configuration fixes for the issues above.
|
||||
|
||||
* Add CMake build support. The autoconf build support is now deprecated, and
|
||||
will be removed in the next release.
|
||||
|
||||
* Add AppVeyor configuration, for Windows CI coverage.
|
||||
|
||||
* Small performance improvement on little-endian PowerPC.
|
||||
|
||||
* Small performance improvement on LLVM with position-independent executables.
|
||||
|
||||
* Fix a few issues with various build environments.
|
||||
|
||||
Snappy v1.1.4, January 25th 2017:
|
||||
|
||||
* Fix a 1% performance regression when snappy is used in PIE executables.
|
||||
|
||||
* Improve compression performance by 5%.
|
||||
|
||||
* Improve decompression performance by 20%.
|
||||
|
||||
Snappy v1.1.3, July 6th 2015:
|
||||
|
||||
This is the first release to be done from GitHub, which means that
|
||||
some minor things like the ChangeLog format has changed (git log
|
||||
format instead of svn log).
|
||||
|
||||
* Add support for Uncompress() from a Source to a Sink.
|
||||
|
||||
* Various minor changes to improve MSVC support; in particular,
|
||||
the unit tests now compile and run under MSVC.
|
||||
|
||||
|
||||
Snappy v1.1.2, February 28th 2014:
|
||||
|
||||
This is a maintenance release with no changes to the actual library
|
||||
|
||||
@@ -34,7 +34,7 @@ Snappy is intended to be fast. On a single core of a Core i7 processor
|
||||
in 64-bit mode, it compresses at about 250 MB/sec or more and decompresses at
|
||||
about 500 MB/sec or more. (These numbers are for the slowest inputs in our
|
||||
benchmark suite; others are much faster.) In our tests, Snappy usually
|
||||
is faster than algorithms in the same class (e.g. LZO, LZF, FastLZ, QuickLZ,
|
||||
is faster than algorithms in the same class (e.g. LZO, LZF, QuickLZ,
|
||||
etc.) while achieving comparable compression ratios.
|
||||
|
||||
Typical compression ratios (based on the benchmark suite) are about 1.5-1.7x
|
||||
@@ -62,12 +62,22 @@ Performance optimizations, whether for 64-bit x86 or other platforms,
|
||||
are of course most welcome; see "Contact", below.
|
||||
|
||||
|
||||
Building
|
||||
========
|
||||
|
||||
CMake is supported and autotools will soon be deprecated.
|
||||
You need CMake 3.4 or above to build:
|
||||
|
||||
mkdir build
|
||||
cd build && cmake ../ && make
|
||||
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
Note that Snappy, both the implementation and the main interface,
|
||||
is written in C++. However, several third-party bindings to other languages
|
||||
are available; see the Google Code page at http://code.google.com/p/snappy/
|
||||
are available; see the home page at http://google.github.io/snappy/
|
||||
for more information. Also, if you want to use Snappy from C code, you can
|
||||
use the included C bindings in snappy-c.h.
|
||||
|
||||
@@ -102,12 +112,12 @@ tests to verify you have not broken anything. Note that if you have the
|
||||
Google Test library installed, unit test behavior (especially failures) will be
|
||||
significantly more user-friendly. You can find Google Test at
|
||||
|
||||
http://code.google.com/p/googletest/
|
||||
http://github.com/google/googletest
|
||||
|
||||
You probably also want the gflags library for handling of command-line flags;
|
||||
you can find it at
|
||||
|
||||
http://code.google.com/p/google-gflags/
|
||||
http://gflags.github.io/gflags/
|
||||
|
||||
In addition to the unit tests, snappy contains microbenchmarks used to
|
||||
tune compression and decompression performance. These are automatically run
|
||||
@@ -116,7 +126,7 @@ before the unit tests, but you can disable them using the flag
|
||||
need to edit the source).
|
||||
|
||||
Finally, snappy can benchmark Snappy against a few other compression libraries
|
||||
(zlib, LZO, LZF, FastLZ and QuickLZ), if they were detected at configure time.
|
||||
(zlib, LZO, LZF, and QuickLZ), if they were detected at configure time.
|
||||
To benchmark using a given file, give the compression algorithm you want to test
|
||||
Snappy against (e.g. --zlib) and then a list of one or more file names on the
|
||||
command line. The testdata/ directory contains the files used by the
|
||||
@@ -129,7 +139,11 @@ test.)
|
||||
Contact
|
||||
=======
|
||||
|
||||
Snappy is distributed through Google Code. For the latest version, a bug tracker,
|
||||
Snappy is distributed through GitHub. For the latest version, a bug tracker,
|
||||
and other information, see
|
||||
|
||||
http://code.google.com/p/snappy/
|
||||
http://google.github.io/snappy/
|
||||
|
||||
or the repository at
|
||||
|
||||
https://github.com/google/snappy
|
||||
@@ -1,7 +0,0 @@
|
||||
#! /bin/sh -e
|
||||
rm -rf autom4te.cache
|
||||
aclocal -I m4
|
||||
autoheader
|
||||
libtoolize --copy
|
||||
automake --add-missing --copy
|
||||
autoconf
|
||||
1
src/snappy/snappy/cmake/SnappyConfig.cmake
Normal file
1
src/snappy/snappy/cmake/SnappyConfig.cmake
Normal file
@@ -0,0 +1 @@
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/SnappyTargets.cmake")
|
||||
62
src/snappy/snappy/cmake/config.h.in
Normal file
62
src/snappy/snappy/cmake/config.h.in
Normal file
@@ -0,0 +1,62 @@
|
||||
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
|
||||
#define THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
|
||||
|
||||
/* Define to 1 if the compiler supports __builtin_ctz and friends. */
|
||||
#cmakedefine HAVE_BUILTIN_CTZ 1
|
||||
|
||||
/* Define to 1 if the compiler supports __builtin_expect. */
|
||||
#cmakedefine HAVE_BUILTIN_EXPECT 1
|
||||
|
||||
/* Define to 1 if you have the <byteswap.h> header file. */
|
||||
#cmakedefine HAVE_BYTESWAP_H 1
|
||||
|
||||
/* Define to 1 if you have a definition for mmap() in <sys/mman.h>. */
|
||||
#cmakedefine HAVE_FUNC_MMAP 1
|
||||
|
||||
/* Define to 1 if you have a definition for sysconf() in <unistd.h>. */
|
||||
#cmakedefine HAVE_FUNC_SYSCONF 1
|
||||
|
||||
/* Define to 1 to use the gflags package for command-line parsing. */
|
||||
#cmakedefine HAVE_GFLAGS 1
|
||||
|
||||
/* Define to 1 if you have Google Test. */
|
||||
#cmakedefine HAVE_GTEST 1
|
||||
|
||||
/* Define to 1 if you have the `lzo2' library (-llzo2). */
|
||||
#cmakedefine HAVE_LIBLZO2 1
|
||||
|
||||
/* Define to 1 if you have the `z' library (-lz). */
|
||||
#cmakedefine HAVE_LIBZ 1
|
||||
|
||||
/* Define to 1 if you have the <stddef.h> header file. */
|
||||
#cmakedefine HAVE_STDDEF_H 1
|
||||
|
||||
/* Define to 1 if you have the <stdint.h> header file. */
|
||||
#cmakedefine HAVE_STDINT_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/endian.h> header file. */
|
||||
#cmakedefine HAVE_SYS_ENDIAN_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/mman.h> header file. */
|
||||
#cmakedefine HAVE_SYS_MMAN_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/resource.h> header file. */
|
||||
#cmakedefine HAVE_SYS_RESOURCE_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/time.h> header file. */
|
||||
#cmakedefine HAVE_SYS_TIME_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/uio.h> header file. */
|
||||
#cmakedefine HAVE_SYS_UIO_H 1
|
||||
|
||||
/* Define to 1 if you have the <unistd.h> header file. */
|
||||
#cmakedefine HAVE_UNISTD_H 1
|
||||
|
||||
/* Define to 1 if you have the <windows.h> header file. */
|
||||
#cmakedefine HAVE_WINDOWS_H 1
|
||||
|
||||
/* Define to 1 if your processor stores words with the most significant byte
|
||||
first (like Motorola and SPARC, unlike Intel and VAX). */
|
||||
#cmakedefine SNAPPY_IS_BIG_ENDIAN 1
|
||||
|
||||
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
|
||||
@@ -1,133 +0,0 @@
|
||||
m4_define([snappy_major], [1])
|
||||
m4_define([snappy_minor], [1])
|
||||
m4_define([snappy_patchlevel], [2])
|
||||
|
||||
# Libtool shared library interface versions (current:revision:age)
|
||||
# Update this value for every release! (A:B:C will map to foo.so.(A-C).C.B)
|
||||
# http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
|
||||
m4_define([snappy_ltversion], [3:1:2])
|
||||
|
||||
AC_INIT([snappy], [snappy_major.snappy_minor.snappy_patchlevel])
|
||||
AC_CONFIG_MACRO_DIR([m4])
|
||||
|
||||
# These are flags passed to automake (though they look like gcc flags!)
|
||||
AM_INIT_AUTOMAKE([-Wall])
|
||||
|
||||
LT_INIT
|
||||
AC_SUBST([LIBTOOL_DEPS])
|
||||
AC_PROG_CXX
|
||||
AC_LANG([C++])
|
||||
AC_C_BIGENDIAN
|
||||
AC_TYPE_SIZE_T
|
||||
AC_TYPE_SSIZE_T
|
||||
AC_CHECK_HEADERS([stdint.h stddef.h sys/mman.h sys/resource.h windows.h byteswap.h sys/byteswap.h sys/endian.h sys/time.h])
|
||||
|
||||
# Don't use AC_FUNC_MMAP, as it checks for mappings of already-mapped memory,
|
||||
# which we don't need (and does not exist on Windows).
|
||||
AC_CHECK_FUNC([mmap])
|
||||
|
||||
GTEST_LIB_CHECK([], [true], [true # Ignore; we can live without it.])
|
||||
|
||||
AC_ARG_WITH([gflags],
|
||||
[AS_HELP_STRING(
|
||||
[--with-gflags],
|
||||
[use Google Flags package to enhance the unit test @<:@default=check@:>@])],
|
||||
[],
|
||||
[with_gflags=check])
|
||||
|
||||
if test "x$with_gflags" != "xno"; then
|
||||
PKG_CHECK_MODULES(
|
||||
[gflags],
|
||||
[libgflags],
|
||||
[AC_DEFINE([HAVE_GFLAGS], [1], [Use the gflags package for command-line parsing.])],
|
||||
[if test "x$with_gflags" != "xcheck"; then
|
||||
AC_MSG_FAILURE([--with-gflags was given, but test for gflags failed])
|
||||
fi])
|
||||
fi
|
||||
|
||||
# See if we have __builtin_expect.
|
||||
# TODO: Use AC_CACHE.
|
||||
AC_MSG_CHECKING([if the compiler supports __builtin_expect])
|
||||
|
||||
AC_TRY_COMPILE(, [
|
||||
return __builtin_expect(1, 1) ? 1 : 0
|
||||
], [
|
||||
snappy_have_builtin_expect=yes
|
||||
AC_MSG_RESULT([yes])
|
||||
], [
|
||||
snappy_have_builtin_expect=no
|
||||
AC_MSG_RESULT([no])
|
||||
])
|
||||
if test x$snappy_have_builtin_expect = xyes ; then
|
||||
AC_DEFINE([HAVE_BUILTIN_EXPECT], [1], [Define to 1 if the compiler supports __builtin_expect.])
|
||||
fi
|
||||
|
||||
# See if we have working count-trailing-zeros intrinsics.
|
||||
# TODO: Use AC_CACHE.
|
||||
AC_MSG_CHECKING([if the compiler supports __builtin_ctzll])
|
||||
|
||||
AC_TRY_COMPILE(, [
|
||||
return (__builtin_ctzll(0x100000000LL) == 32) ? 1 : 0
|
||||
], [
|
||||
snappy_have_builtin_ctz=yes
|
||||
AC_MSG_RESULT([yes])
|
||||
], [
|
||||
snappy_have_builtin_ctz=no
|
||||
AC_MSG_RESULT([no])
|
||||
])
|
||||
if test x$snappy_have_builtin_ctz = xyes ; then
|
||||
AC_DEFINE([HAVE_BUILTIN_CTZ], [1], [Define to 1 if the compiler supports __builtin_ctz and friends.])
|
||||
fi
|
||||
|
||||
# Other compression libraries; the unit test can use these for comparison
|
||||
# if they are available. If they are not found, just ignore.
|
||||
UNITTEST_LIBS=""
|
||||
AC_DEFUN([CHECK_EXT_COMPRESSION_LIB], [
|
||||
AH_CHECK_LIB([$1])
|
||||
AC_CHECK_LIB(
|
||||
[$1],
|
||||
[$2],
|
||||
[
|
||||
AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_LIB$1))
|
||||
UNITTEST_LIBS="-l$1 $UNITTEST_LIBS"
|
||||
],
|
||||
[true]
|
||||
)
|
||||
])
|
||||
CHECK_EXT_COMPRESSION_LIB([z], [zlibVersion])
|
||||
CHECK_EXT_COMPRESSION_LIB([lzo2], [lzo1x_1_15_compress])
|
||||
CHECK_EXT_COMPRESSION_LIB([lzf], [lzf_compress])
|
||||
CHECK_EXT_COMPRESSION_LIB([fastlz], [fastlz_compress])
|
||||
CHECK_EXT_COMPRESSION_LIB([quicklz], [qlz_compress])
|
||||
AC_SUBST([UNITTEST_LIBS])
|
||||
|
||||
# These are used by snappy-stubs-public.h.in.
|
||||
if test "$ac_cv_header_stdint_h" = "yes"; then
|
||||
AC_SUBST([ac_cv_have_stdint_h], [1])
|
||||
else
|
||||
AC_SUBST([ac_cv_have_stdint_h], [0])
|
||||
fi
|
||||
if test "$ac_cv_header_stddef_h" = "yes"; then
|
||||
AC_SUBST([ac_cv_have_stddef_h], [1])
|
||||
else
|
||||
AC_SUBST([ac_cv_have_stddef_h], [0])
|
||||
fi
|
||||
if test "$ac_cv_header_sys_uio_h" = "yes"; then
|
||||
AC_SUBST([ac_cv_have_sys_uio_h], [1])
|
||||
else
|
||||
AC_SUBST([ac_cv_have_sys_uio_h], [0])
|
||||
fi
|
||||
|
||||
# Export the version to snappy-stubs-public.h.
|
||||
SNAPPY_MAJOR="snappy_major"
|
||||
SNAPPY_MINOR="snappy_minor"
|
||||
SNAPPY_PATCHLEVEL="snappy_patchlevel"
|
||||
|
||||
AC_SUBST([SNAPPY_MAJOR])
|
||||
AC_SUBST([SNAPPY_MINOR])
|
||||
AC_SUBST([SNAPPY_PATCHLEVEL])
|
||||
AC_SUBST([SNAPPY_LTVERSION], snappy_ltversion)
|
||||
|
||||
AC_CONFIG_HEADERS([config.h])
|
||||
AC_CONFIG_FILES([Makefile snappy-stubs-public.h])
|
||||
AC_OUTPUT
|
||||
@@ -1,74 +0,0 @@
|
||||
dnl GTEST_LIB_CHECK([minimum version [,
|
||||
dnl action if found [,action if not found]]])
|
||||
dnl
|
||||
dnl Check for the presence of the Google Test library, optionally at a minimum
|
||||
dnl version, and indicate a viable version with the HAVE_GTEST flag. It defines
|
||||
dnl standard variables for substitution including GTEST_CPPFLAGS,
|
||||
dnl GTEST_CXXFLAGS, GTEST_LDFLAGS, and GTEST_LIBS. It also defines
|
||||
dnl GTEST_VERSION as the version of Google Test found. Finally, it provides
|
||||
dnl optional custom action slots in the event GTEST is found or not.
|
||||
AC_DEFUN([GTEST_LIB_CHECK],
|
||||
[
|
||||
dnl Provide a flag to enable or disable Google Test usage.
|
||||
AC_ARG_ENABLE([gtest],
|
||||
[AS_HELP_STRING([--enable-gtest],
|
||||
[Enable tests using the Google C++ Testing Framework.
|
||||
(Default is enabled.)])],
|
||||
[],
|
||||
[enable_gtest=])
|
||||
AC_ARG_VAR([GTEST_CONFIG],
|
||||
[The exact path of Google Test's 'gtest-config' script.])
|
||||
AC_ARG_VAR([GTEST_CPPFLAGS],
|
||||
[C-like preprocessor flags for Google Test.])
|
||||
AC_ARG_VAR([GTEST_CXXFLAGS],
|
||||
[C++ compile flags for Google Test.])
|
||||
AC_ARG_VAR([GTEST_LDFLAGS],
|
||||
[Linker path and option flags for Google Test.])
|
||||
AC_ARG_VAR([GTEST_LIBS],
|
||||
[Library linking flags for Google Test.])
|
||||
AC_ARG_VAR([GTEST_VERSION],
|
||||
[The version of Google Test available.])
|
||||
HAVE_GTEST="no"
|
||||
AS_IF([test "x${enable_gtest}" != "xno"],
|
||||
[AC_MSG_CHECKING([for 'gtest-config'])
|
||||
AS_IF([test "x${enable_gtest}" = "xyes"],
|
||||
[AS_IF([test -x "${enable_gtest}/scripts/gtest-config"],
|
||||
[GTEST_CONFIG="${enable_gtest}/scripts/gtest-config"],
|
||||
[GTEST_CONFIG="${enable_gtest}/bin/gtest-config"])
|
||||
AS_IF([test -x "${GTEST_CONFIG}"], [],
|
||||
[AC_MSG_RESULT([no])
|
||||
AC_MSG_ERROR([dnl
|
||||
Unable to locate either a built or installed Google Test.
|
||||
The specific location '${enable_gtest}' was provided for a built or installed
|
||||
Google Test, but no 'gtest-config' script could be found at this location.])
|
||||
])],
|
||||
[AC_PATH_PROG([GTEST_CONFIG], [gtest-config])])
|
||||
AS_IF([test -x "${GTEST_CONFIG}"],
|
||||
[AC_MSG_RESULT([${GTEST_CONFIG}])
|
||||
m4_ifval([$1],
|
||||
[_gtest_min_version="--min-version=$1"
|
||||
AC_MSG_CHECKING([for Google Test at least version >= $1])],
|
||||
[_gtest_min_version="--min-version=0"
|
||||
AC_MSG_CHECKING([for Google Test])])
|
||||
AS_IF([${GTEST_CONFIG} ${_gtest_min_version}],
|
||||
[AC_MSG_RESULT([yes])
|
||||
HAVE_GTEST='yes'],
|
||||
[AC_MSG_RESULT([no])])],
|
||||
[AC_MSG_RESULT([no])])
|
||||
AS_IF([test "x${HAVE_GTEST}" = "xyes"],
|
||||
[GTEST_CPPFLAGS=`${GTEST_CONFIG} --cppflags`
|
||||
GTEST_CXXFLAGS=`${GTEST_CONFIG} --cxxflags`
|
||||
GTEST_LDFLAGS=`${GTEST_CONFIG} --ldflags`
|
||||
GTEST_LIBS=`${GTEST_CONFIG} --libs`
|
||||
GTEST_VERSION=`${GTEST_CONFIG} --version`
|
||||
AC_DEFINE([HAVE_GTEST],[1],[Defined when Google Test is available.])],
|
||||
[AS_IF([test "x${enable_gtest}" = "xyes"],
|
||||
[AC_MSG_ERROR([dnl
|
||||
Google Test was enabled, but no viable version could be found.])
|
||||
])])])
|
||||
AC_SUBST([HAVE_GTEST])
|
||||
AM_CONDITIONAL([HAVE_GTEST],[test "x$HAVE_GTEST" = "xyes"])
|
||||
AS_IF([test "x$HAVE_GTEST" = "xyes"],
|
||||
[m4_ifval([$2], [$2])],
|
||||
[m4_ifval([$3], [$3])])
|
||||
])
|
||||
@@ -30,8 +30,8 @@
|
||||
* Plain C interface (a wrapper around the C++ implementation).
|
||||
*/
|
||||
|
||||
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_
|
||||
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_
|
||||
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_
|
||||
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
@@ -135,4 +135,4 @@ snappy_status snappy_validate_compressed_buffer(const char* compressed,
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
#endif /* UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_ */
|
||||
#endif /* THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_ */
|
||||
|
||||
@@ -28,8 +28,8 @@
|
||||
//
|
||||
// Internals shared between the Snappy implementation and its unittest.
|
||||
|
||||
#ifndef UTIL_SNAPPY_SNAPPY_INTERNAL_H_
|
||||
#define UTIL_SNAPPY_SNAPPY_INTERNAL_H_
|
||||
#ifndef THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_
|
||||
#define THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_
|
||||
|
||||
#include "snappy-stubs-internal.h"
|
||||
|
||||
@@ -50,7 +50,9 @@ class WorkingMemory {
|
||||
uint16 small_table_[1<<10]; // 2KB
|
||||
uint16* large_table_; // Allocated only when needed
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(WorkingMemory);
|
||||
// No copying
|
||||
WorkingMemory(const WorkingMemory&);
|
||||
void operator=(const WorkingMemory&);
|
||||
};
|
||||
|
||||
// Flat array compression that does not emit the "uncompressed length"
|
||||
@@ -70,55 +72,70 @@ char* CompressFragment(const char* input,
|
||||
uint16* table,
|
||||
const int table_size);
|
||||
|
||||
// Return the largest n such that
|
||||
// Find the largest n such that
|
||||
//
|
||||
// s1[0,n-1] == s2[0,n-1]
|
||||
// and n <= (s2_limit - s2).
|
||||
//
|
||||
// Return make_pair(n, n < 8).
|
||||
// Does not read *s2_limit or beyond.
|
||||
// Does not read *(s1 + (s2_limit - s2)) or beyond.
|
||||
// Requires that s2_limit >= s2.
|
||||
//
|
||||
// Separate implementation for x86_64, for speed. Uses the fact that
|
||||
// x86_64 is little endian.
|
||||
#if defined(ARCH_K8)
|
||||
static inline int FindMatchLength(const char* s1,
|
||||
// Separate implementation for 64-bit, little-endian cpus.
|
||||
#if !defined(SNAPPY_IS_BIG_ENDIAN) && \
|
||||
(defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM))
|
||||
static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
|
||||
const char* s2,
|
||||
const char* s2_limit) {
|
||||
assert(s2_limit >= s2);
|
||||
int matched = 0;
|
||||
size_t matched = 0;
|
||||
|
||||
// This block isn't necessary for correctness; we could just start looping
|
||||
// immediately. As an optimization though, it is useful. It creates some not
|
||||
// uncommon code paths that determine, without extra effort, whether the match
|
||||
// length is less than 8. In short, we are hoping to avoid a conditional
|
||||
// branch, and perhaps get better code layout from the C++ compiler.
|
||||
if (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
|
||||
uint64 a1 = UNALIGNED_LOAD64(s1);
|
||||
uint64 a2 = UNALIGNED_LOAD64(s2);
|
||||
if (a1 != a2) {
|
||||
return std::pair<size_t, bool>(Bits::FindLSBSetNonZero64(a1 ^ a2) >> 3,
|
||||
true);
|
||||
} else {
|
||||
matched = 8;
|
||||
s2 += 8;
|
||||
}
|
||||
}
|
||||
|
||||
// Find out how long the match is. We loop over the data 64 bits at a
|
||||
// time until we find a 64-bit block that doesn't match; then we find
|
||||
// the first non-matching bit and use that to calculate the total
|
||||
// length of the match.
|
||||
while (PREDICT_TRUE(s2 <= s2_limit - 8)) {
|
||||
if (PREDICT_FALSE(UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched))) {
|
||||
while (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
|
||||
if (UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched)) {
|
||||
s2 += 8;
|
||||
matched += 8;
|
||||
} else {
|
||||
// On current (mid-2008) Opteron models there is a 3% more
|
||||
// efficient code sequence to find the first non-matching byte.
|
||||
// However, what follows is ~10% better on Intel Core 2 and newer,
|
||||
// and we expect AMD's bsf instruction to improve.
|
||||
uint64 x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched);
|
||||
int matching_bits = Bits::FindLSBSetNonZero64(x);
|
||||
matched += matching_bits >> 3;
|
||||
return matched;
|
||||
assert(matched >= 8);
|
||||
return std::pair<size_t, bool>(matched, false);
|
||||
}
|
||||
}
|
||||
while (PREDICT_TRUE(s2 < s2_limit)) {
|
||||
if (PREDICT_TRUE(s1[matched] == *s2)) {
|
||||
while (SNAPPY_PREDICT_TRUE(s2 < s2_limit)) {
|
||||
if (s1[matched] == *s2) {
|
||||
++s2;
|
||||
++matched;
|
||||
} else {
|
||||
return matched;
|
||||
return std::pair<size_t, bool>(matched, matched < 8);
|
||||
}
|
||||
}
|
||||
return matched;
|
||||
return std::pair<size_t, bool>(matched, matched < 8);
|
||||
}
|
||||
#else
|
||||
static inline int FindMatchLength(const char* s1,
|
||||
static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
|
||||
const char* s2,
|
||||
const char* s2_limit) {
|
||||
// Implementation based on the x86-64 version, above.
|
||||
@@ -140,11 +157,68 @@ static inline int FindMatchLength(const char* s1,
|
||||
++matched;
|
||||
}
|
||||
}
|
||||
return matched;
|
||||
return std::pair<size_t, bool>(matched, matched < 8);
|
||||
}
|
||||
#endif
|
||||
|
||||
// Lookup tables for decompression code. Give --snappy_dump_decompression_table
|
||||
// to the unit test to recompute char_table.
|
||||
|
||||
enum {
|
||||
LITERAL = 0,
|
||||
COPY_1_BYTE_OFFSET = 1, // 3 bit length + 3 bits of offset in opcode
|
||||
COPY_2_BYTE_OFFSET = 2,
|
||||
COPY_4_BYTE_OFFSET = 3
|
||||
};
|
||||
static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset.
|
||||
|
||||
// Data stored per entry in lookup table:
|
||||
// Range Bits-used Description
|
||||
// ------------------------------------
|
||||
// 1..64 0..7 Literal/copy length encoded in opcode byte
|
||||
// 0..7 8..10 Copy offset encoded in opcode byte / 256
|
||||
// 0..4 11..13 Extra bytes after opcode
|
||||
//
|
||||
// We use eight bits for the length even though 7 would have sufficed
|
||||
// because of efficiency reasons:
|
||||
// (1) Extracting a byte is faster than a bit-field
|
||||
// (2) It properly aligns copy offset so we do not need a <<8
|
||||
static const uint16 char_table[256] = {
|
||||
0x0001, 0x0804, 0x1001, 0x2001, 0x0002, 0x0805, 0x1002, 0x2002,
|
||||
0x0003, 0x0806, 0x1003, 0x2003, 0x0004, 0x0807, 0x1004, 0x2004,
|
||||
0x0005, 0x0808, 0x1005, 0x2005, 0x0006, 0x0809, 0x1006, 0x2006,
|
||||
0x0007, 0x080a, 0x1007, 0x2007, 0x0008, 0x080b, 0x1008, 0x2008,
|
||||
0x0009, 0x0904, 0x1009, 0x2009, 0x000a, 0x0905, 0x100a, 0x200a,
|
||||
0x000b, 0x0906, 0x100b, 0x200b, 0x000c, 0x0907, 0x100c, 0x200c,
|
||||
0x000d, 0x0908, 0x100d, 0x200d, 0x000e, 0x0909, 0x100e, 0x200e,
|
||||
0x000f, 0x090a, 0x100f, 0x200f, 0x0010, 0x090b, 0x1010, 0x2010,
|
||||
0x0011, 0x0a04, 0x1011, 0x2011, 0x0012, 0x0a05, 0x1012, 0x2012,
|
||||
0x0013, 0x0a06, 0x1013, 0x2013, 0x0014, 0x0a07, 0x1014, 0x2014,
|
||||
0x0015, 0x0a08, 0x1015, 0x2015, 0x0016, 0x0a09, 0x1016, 0x2016,
|
||||
0x0017, 0x0a0a, 0x1017, 0x2017, 0x0018, 0x0a0b, 0x1018, 0x2018,
|
||||
0x0019, 0x0b04, 0x1019, 0x2019, 0x001a, 0x0b05, 0x101a, 0x201a,
|
||||
0x001b, 0x0b06, 0x101b, 0x201b, 0x001c, 0x0b07, 0x101c, 0x201c,
|
||||
0x001d, 0x0b08, 0x101d, 0x201d, 0x001e, 0x0b09, 0x101e, 0x201e,
|
||||
0x001f, 0x0b0a, 0x101f, 0x201f, 0x0020, 0x0b0b, 0x1020, 0x2020,
|
||||
0x0021, 0x0c04, 0x1021, 0x2021, 0x0022, 0x0c05, 0x1022, 0x2022,
|
||||
0x0023, 0x0c06, 0x1023, 0x2023, 0x0024, 0x0c07, 0x1024, 0x2024,
|
||||
0x0025, 0x0c08, 0x1025, 0x2025, 0x0026, 0x0c09, 0x1026, 0x2026,
|
||||
0x0027, 0x0c0a, 0x1027, 0x2027, 0x0028, 0x0c0b, 0x1028, 0x2028,
|
||||
0x0029, 0x0d04, 0x1029, 0x2029, 0x002a, 0x0d05, 0x102a, 0x202a,
|
||||
0x002b, 0x0d06, 0x102b, 0x202b, 0x002c, 0x0d07, 0x102c, 0x202c,
|
||||
0x002d, 0x0d08, 0x102d, 0x202d, 0x002e, 0x0d09, 0x102e, 0x202e,
|
||||
0x002f, 0x0d0a, 0x102f, 0x202f, 0x0030, 0x0d0b, 0x1030, 0x2030,
|
||||
0x0031, 0x0e04, 0x1031, 0x2031, 0x0032, 0x0e05, 0x1032, 0x2032,
|
||||
0x0033, 0x0e06, 0x1033, 0x2033, 0x0034, 0x0e07, 0x1034, 0x2034,
|
||||
0x0035, 0x0e08, 0x1035, 0x2035, 0x0036, 0x0e09, 0x1036, 0x2036,
|
||||
0x0037, 0x0e0a, 0x1037, 0x2037, 0x0038, 0x0e0b, 0x1038, 0x2038,
|
||||
0x0039, 0x0f04, 0x1039, 0x2039, 0x003a, 0x0f05, 0x103a, 0x203a,
|
||||
0x003b, 0x0f06, 0x103b, 0x203b, 0x003c, 0x0f07, 0x103c, 0x203c,
|
||||
0x0801, 0x0f08, 0x103d, 0x203d, 0x1001, 0x0f09, 0x103e, 0x203e,
|
||||
0x1801, 0x0f0a, 0x103f, 0x203f, 0x2001, 0x0f0b, 0x1040, 0x2040
|
||||
};
|
||||
|
||||
} // end namespace internal
|
||||
} // end namespace snappy
|
||||
|
||||
#endif // UTIL_SNAPPY_SNAPPY_INTERNAL_H_
|
||||
#endif // THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_
|
||||
|
||||
@@ -40,6 +40,21 @@ char* Sink::GetAppendBuffer(size_t length, char* scratch) {
|
||||
return scratch;
|
||||
}
|
||||
|
||||
char* Sink::GetAppendBufferVariable(
|
||||
size_t min_size, size_t desired_size_hint, char* scratch,
|
||||
size_t scratch_size, size_t* allocated_size) {
|
||||
*allocated_size = scratch_size;
|
||||
return scratch;
|
||||
}
|
||||
|
||||
void Sink::AppendAndTakeOwnership(
|
||||
char* bytes, size_t n,
|
||||
void (*deleter)(void*, const char*, size_t),
|
||||
void *deleter_arg) {
|
||||
Append(bytes, n);
|
||||
(*deleter)(deleter_arg, bytes, n);
|
||||
}
|
||||
|
||||
ByteArraySource::~ByteArraySource() { }
|
||||
|
||||
size_t ByteArraySource::Available() const { return left_; }
|
||||
@@ -68,4 +83,22 @@ char* UncheckedByteArraySink::GetAppendBuffer(size_t len, char* scratch) {
|
||||
return dest_;
|
||||
}
|
||||
|
||||
void UncheckedByteArraySink::AppendAndTakeOwnership(
|
||||
char* data, size_t n,
|
||||
void (*deleter)(void*, const char*, size_t),
|
||||
void *deleter_arg) {
|
||||
if (data != dest_) {
|
||||
memcpy(dest_, data, n);
|
||||
(*deleter)(deleter_arg, data, n);
|
||||
}
|
||||
dest_ += n;
|
||||
}
|
||||
|
||||
char* UncheckedByteArraySink::GetAppendBufferVariable(
|
||||
size_t min_size, size_t desired_size_hint, char* scratch,
|
||||
size_t scratch_size, size_t* allocated_size) {
|
||||
*allocated_size = desired_size_hint;
|
||||
return dest_;
|
||||
}
|
||||
|
||||
} // namespace snappy
|
||||
|
||||
@@ -26,12 +26,11 @@
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
#ifndef UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_
|
||||
#define UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_
|
||||
#ifndef THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
|
||||
#define THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
|
||||
namespace snappy {
|
||||
|
||||
// A Sink is an interface that consumes a sequence of bytes.
|
||||
@@ -60,6 +59,47 @@ class Sink {
|
||||
// The default implementation always returns the scratch buffer.
|
||||
virtual char* GetAppendBuffer(size_t length, char* scratch);
|
||||
|
||||
// For higher performance, Sink implementations can provide custom
|
||||
// AppendAndTakeOwnership() and GetAppendBufferVariable() methods.
|
||||
// These methods can reduce the number of copies done during
|
||||
// compression/decompression.
|
||||
|
||||
// Append "bytes[0,n-1] to the sink. Takes ownership of "bytes"
|
||||
// and calls the deleter function as (*deleter)(deleter_arg, bytes, n)
|
||||
// to free the buffer. deleter function must be non NULL.
|
||||
//
|
||||
// The default implementation just calls Append and frees "bytes".
|
||||
// Other implementations may avoid a copy while appending the buffer.
|
||||
virtual void AppendAndTakeOwnership(
|
||||
char* bytes, size_t n, void (*deleter)(void*, const char*, size_t),
|
||||
void *deleter_arg);
|
||||
|
||||
// Returns a writable buffer for appending and writes the buffer's capacity to
|
||||
// *allocated_size. Guarantees *allocated_size >= min_size.
|
||||
// May return a pointer to the caller-owned scratch buffer which must have
|
||||
// scratch_size >= min_size.
|
||||
//
|
||||
// The returned buffer is only valid until the next operation
|
||||
// on this ByteSink.
|
||||
//
|
||||
// After writing at most *allocated_size bytes, call Append() with the
|
||||
// pointer returned from this function and the number of bytes written.
|
||||
// Many Append() implementations will avoid copying bytes if this function
|
||||
// returned an internal buffer.
|
||||
//
|
||||
// If the sink implementation allocates or reallocates an internal buffer,
|
||||
// it should use the desired_size_hint if appropriate. If a caller cannot
|
||||
// provide a reasonable guess at the desired capacity, it should set
|
||||
// desired_size_hint = 0.
|
||||
//
|
||||
// If a non-scratch buffer is returned, the caller may only pass
|
||||
// a prefix to it to Append(). That is, it is not correct to pass an
|
||||
// interior pointer to Append().
|
||||
//
|
||||
// The default implementation always returns the scratch buffer.
|
||||
virtual char* GetAppendBufferVariable(
|
||||
size_t min_size, size_t desired_size_hint, char* scratch,
|
||||
size_t scratch_size, size_t* allocated_size);
|
||||
|
||||
private:
|
||||
// No copying
|
||||
@@ -122,6 +162,12 @@ class UncheckedByteArraySink : public Sink {
|
||||
virtual ~UncheckedByteArraySink();
|
||||
virtual void Append(const char* data, size_t n);
|
||||
virtual char* GetAppendBuffer(size_t len, char* scratch);
|
||||
virtual char* GetAppendBufferVariable(
|
||||
size_t min_size, size_t desired_size_hint, char* scratch,
|
||||
size_t scratch_size, size_t* allocated_size);
|
||||
virtual void AppendAndTakeOwnership(
|
||||
char* bytes, size_t n, void (*deleter)(void*, const char*, size_t),
|
||||
void *deleter_arg);
|
||||
|
||||
// Return the current output pointer so that a caller can see how
|
||||
// many bytes were produced.
|
||||
@@ -131,7 +177,6 @@ class UncheckedByteArraySink : public Sink {
|
||||
char* dest_;
|
||||
};
|
||||
|
||||
} // namespace snappy
|
||||
|
||||
}
|
||||
|
||||
#endif // UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_
|
||||
#endif // THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
|
||||
|
||||
@@ -28,8 +28,8 @@
|
||||
//
|
||||
// Various stubs for the open-source version of Snappy.
|
||||
|
||||
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
|
||||
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
|
||||
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
|
||||
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
|
||||
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
@@ -45,6 +45,14 @@
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_UNISTD_H
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#include <intrin.h>
|
||||
#endif // defined(_MSC_VER)
|
||||
|
||||
#include "snappy-stubs-public.h"
|
||||
|
||||
#if defined(__x86_64__)
|
||||
@@ -52,6 +60,14 @@
|
||||
// Enable 64-bit optimized versions of some routines.
|
||||
#define ARCH_K8 1
|
||||
|
||||
#elif defined(__ppc64__)
|
||||
|
||||
#define ARCH_PPC 1
|
||||
|
||||
#elif defined(__aarch64__)
|
||||
|
||||
#define ARCH_ARM 1
|
||||
|
||||
#endif
|
||||
|
||||
// Needed by OS X, among others.
|
||||
@@ -59,10 +75,6 @@
|
||||
#define MAP_ANONYMOUS MAP_ANON
|
||||
#endif
|
||||
|
||||
// Pull in std::min, std::ostream, and the likes. This is safe because this
|
||||
// header file is never used from any public header files.
|
||||
using namespace std;
|
||||
|
||||
// The size of an array, if known at compile-time.
|
||||
// Will give unexpected results if used on a pointer.
|
||||
// We undefine it first, since some compilers already have a definition.
|
||||
@@ -73,11 +85,11 @@ using namespace std;
|
||||
|
||||
// Static prediction hints.
|
||||
#ifdef HAVE_BUILTIN_EXPECT
|
||||
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
|
||||
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
|
||||
#define SNAPPY_PREDICT_FALSE(x) (__builtin_expect(x, 0))
|
||||
#define SNAPPY_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
|
||||
#else
|
||||
#define PREDICT_FALSE(x) x
|
||||
#define PREDICT_TRUE(x) x
|
||||
#define SNAPPY_PREDICT_FALSE(x) x
|
||||
#define SNAPPY_PREDICT_TRUE(x) x
|
||||
#endif
|
||||
|
||||
// This is only used for recomputing the tag byte table used during
|
||||
@@ -96,9 +108,10 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
|
||||
|
||||
// Potentially unaligned loads and stores.
|
||||
|
||||
// x86 and PowerPC can simply do these loads and stores native.
|
||||
// x86, PowerPC, and ARM64 can simply do these loads and stores native.
|
||||
|
||||
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__)
|
||||
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || \
|
||||
defined(__aarch64__)
|
||||
|
||||
#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p))
|
||||
#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p))
|
||||
@@ -116,6 +129,15 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
|
||||
// sub-architectures.
|
||||
//
|
||||
// This is a mess, but there's not much we can do about it.
|
||||
//
|
||||
// To further complicate matters, only LDR instructions (single reads) are
|
||||
// allowed to be unaligned, not LDRD (two reads) or LDM (many reads). Unless we
|
||||
// explicitly tell the compiler that these accesses can be unaligned, it can and
|
||||
// will combine accesses. On armcc, the way to signal this is done by accessing
|
||||
// through the type (uint32 __packed *), but GCC has no such attribute
|
||||
// (it ignores __attribute__((packed)) on individual variables). However,
|
||||
// we can tell it that a _struct_ is unaligned, which has the same effect,
|
||||
// so we do that.
|
||||
|
||||
#elif defined(__arm__) && \
|
||||
!defined(__ARM_ARCH_4__) && \
|
||||
@@ -131,11 +153,39 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
|
||||
!defined(__ARM_ARCH_6ZK__) && \
|
||||
!defined(__ARM_ARCH_6T2__)
|
||||
|
||||
#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p))
|
||||
#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p))
|
||||
#if __GNUC__
|
||||
#define ATTRIBUTE_PACKED __attribute__((__packed__))
|
||||
#else
|
||||
#define ATTRIBUTE_PACKED
|
||||
#endif
|
||||
|
||||
#define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast<uint16 *>(_p) = (_val))
|
||||
#define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast<uint32 *>(_p) = (_val))
|
||||
namespace base {
|
||||
namespace internal {
|
||||
|
||||
struct Unaligned16Struct {
|
||||
uint16 value;
|
||||
uint8 dummy; // To make the size non-power-of-two.
|
||||
} ATTRIBUTE_PACKED;
|
||||
|
||||
struct Unaligned32Struct {
|
||||
uint32 value;
|
||||
uint8 dummy; // To make the size non-power-of-two.
|
||||
} ATTRIBUTE_PACKED;
|
||||
|
||||
} // namespace internal
|
||||
} // namespace base
|
||||
|
||||
#define UNALIGNED_LOAD16(_p) \
|
||||
((reinterpret_cast<const ::snappy::base::internal::Unaligned16Struct *>(_p))->value)
|
||||
#define UNALIGNED_LOAD32(_p) \
|
||||
((reinterpret_cast<const ::snappy::base::internal::Unaligned32Struct *>(_p))->value)
|
||||
|
||||
#define UNALIGNED_STORE16(_p, _val) \
|
||||
((reinterpret_cast< ::snappy::base::internal::Unaligned16Struct *>(_p))->value = \
|
||||
(_val))
|
||||
#define UNALIGNED_STORE32(_p, _val) \
|
||||
((reinterpret_cast< ::snappy::base::internal::Unaligned32Struct *>(_p))->value = \
|
||||
(_val))
|
||||
|
||||
// TODO(user): NEON supports unaligned 64-bit loads and stores.
|
||||
// See if that would be more efficient on platforms supporting it,
|
||||
@@ -188,22 +238,8 @@ inline void UNALIGNED_STORE64(void *p, uint64 v) {
|
||||
|
||||
#endif
|
||||
|
||||
// This can be more efficient than UNALIGNED_LOAD64 + UNALIGNED_STORE64
|
||||
// on some platforms, in particular ARM.
|
||||
inline void UnalignedCopy64(const void *src, void *dst) {
|
||||
if (sizeof(void *) == 8) {
|
||||
UNALIGNED_STORE64(dst, UNALIGNED_LOAD64(src));
|
||||
} else {
|
||||
const char *src_char = reinterpret_cast<const char *>(src);
|
||||
char *dst_char = reinterpret_cast<char *>(dst);
|
||||
|
||||
UNALIGNED_STORE32(dst_char, UNALIGNED_LOAD32(src_char));
|
||||
UNALIGNED_STORE32(dst_char + 4, UNALIGNED_LOAD32(src_char + 4));
|
||||
}
|
||||
}
|
||||
|
||||
// The following guarantees declaration of the byte swap functions.
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
#if defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
#ifdef HAVE_SYS_BYTEORDER_H
|
||||
#include <sys/byteorder.h>
|
||||
@@ -260,7 +296,7 @@ inline uint64 bswap_64(uint64 x) {
|
||||
|
||||
#endif
|
||||
|
||||
#endif // WORDS_BIGENDIAN
|
||||
#endif // defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
// Convert to little-endian storage, opposite of network format.
|
||||
// Convert x from host to little endian: x = LittleEndian.FromHost(x);
|
||||
@@ -274,7 +310,7 @@ inline uint64 bswap_64(uint64 x) {
|
||||
class LittleEndian {
|
||||
public:
|
||||
// Conversion functions.
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
#if defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
static uint16 FromHost16(uint16 x) { return bswap_16(x); }
|
||||
static uint16 ToHost16(uint16 x) { return bswap_16(x); }
|
||||
@@ -284,7 +320,7 @@ class LittleEndian {
|
||||
|
||||
static bool IsLittleEndian() { return false; }
|
||||
|
||||
#else // !defined(WORDS_BIGENDIAN)
|
||||
#else // !defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
static uint16 FromHost16(uint16 x) { return x; }
|
||||
static uint16 ToHost16(uint16 x) { return x; }
|
||||
@@ -294,7 +330,7 @@ class LittleEndian {
|
||||
|
||||
static bool IsLittleEndian() { return true; }
|
||||
|
||||
#endif // !defined(WORDS_BIGENDIAN)
|
||||
#endif // !defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
// Functions to do unaligned loads and stores in little-endian order.
|
||||
static uint16 Load16(const void *p) {
|
||||
@@ -324,10 +360,15 @@ class Bits {
|
||||
// undefined value if n == 0. FindLSBSetNonZero() is similar to ffs() except
|
||||
// that it's 0-indexed.
|
||||
static int FindLSBSetNonZero(uint32 n);
|
||||
|
||||
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
static int FindLSBSetNonZero64(uint64 n);
|
||||
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
|
||||
private:
|
||||
DISALLOW_COPY_AND_ASSIGN(Bits);
|
||||
// No copying
|
||||
Bits(const Bits&);
|
||||
void operator=(const Bits&);
|
||||
};
|
||||
|
||||
#ifdef HAVE_BUILTIN_CTZ
|
||||
@@ -340,9 +381,36 @@ inline int Bits::FindLSBSetNonZero(uint32 n) {
|
||||
return __builtin_ctz(n);
|
||||
}
|
||||
|
||||
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
||||
return __builtin_ctzll(n);
|
||||
}
|
||||
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
|
||||
inline int Bits::Log2Floor(uint32 n) {
|
||||
unsigned long where;
|
||||
if (_BitScanReverse(&where, n)) {
|
||||
return where;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
inline int Bits::FindLSBSetNonZero(uint32 n) {
|
||||
unsigned long where;
|
||||
if (_BitScanForward(&where, n)) return static_cast<int>(where);
|
||||
return 32;
|
||||
}
|
||||
|
||||
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
||||
unsigned long where;
|
||||
if (_BitScanForward64(&where, n)) return static_cast<int>(where);
|
||||
return 64;
|
||||
}
|
||||
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
|
||||
#else // Portable versions.
|
||||
|
||||
@@ -376,6 +444,7 @@ inline int Bits::FindLSBSetNonZero(uint32 n) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
// FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero().
|
||||
inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
||||
const uint32 bottombits = static_cast<uint32>(n);
|
||||
@@ -386,6 +455,7 @@ inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
||||
return FindLSBSetNonZero(bottombits);
|
||||
}
|
||||
}
|
||||
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
|
||||
#endif // End portable versions.
|
||||
|
||||
@@ -488,4 +558,4 @@ inline char* string_as_array(string* str) {
|
||||
|
||||
} // namespace snappy
|
||||
|
||||
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
|
||||
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
|
||||
|
||||
@@ -33,24 +33,24 @@
|
||||
// which is a public header. Instead, snappy-stubs-public.h is generated by
|
||||
// from snappy-stubs-public.h.in at configure time.
|
||||
|
||||
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
||||
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
||||
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
||||
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
||||
|
||||
#if @ac_cv_have_stdint_h@
|
||||
#if ${HAVE_STDINT_H_01} // HAVE_STDINT_H
|
||||
#include <stdint.h>
|
||||
#endif
|
||||
#endif // HAVE_STDDEF_H
|
||||
|
||||
#if @ac_cv_have_stddef_h@
|
||||
#if ${HAVE_STDDEF_H_01} // HAVE_STDDEF_H
|
||||
#include <stddef.h>
|
||||
#endif
|
||||
#endif // HAVE_STDDEF_H
|
||||
|
||||
#if @ac_cv_have_sys_uio_h@
|
||||
#if ${HAVE_SYS_UIO_H_01} // HAVE_SYS_UIO_H
|
||||
#include <sys/uio.h>
|
||||
#endif
|
||||
#endif // HAVE_SYS_UIO_H
|
||||
|
||||
#define SNAPPY_MAJOR @SNAPPY_MAJOR@
|
||||
#define SNAPPY_MINOR @SNAPPY_MINOR@
|
||||
#define SNAPPY_PATCHLEVEL @SNAPPY_PATCHLEVEL@
|
||||
#define SNAPPY_MAJOR ${SNAPPY_MAJOR}
|
||||
#define SNAPPY_MINOR ${SNAPPY_MINOR}
|
||||
#define SNAPPY_PATCHLEVEL ${SNAPPY_PATCHLEVEL}
|
||||
#define SNAPPY_VERSION \
|
||||
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
|
||||
|
||||
@@ -58,7 +58,7 @@
|
||||
|
||||
namespace snappy {
|
||||
|
||||
#if @ac_cv_have_stdint_h@
|
||||
#if ${HAVE_STDINT_H_01} // HAVE_STDINT_H
|
||||
typedef int8_t int8;
|
||||
typedef uint8_t uint8;
|
||||
typedef int16_t int16;
|
||||
@@ -76,23 +76,19 @@ typedef int int32;
|
||||
typedef unsigned int uint32;
|
||||
typedef long long int64;
|
||||
typedef unsigned long long uint64;
|
||||
#endif
|
||||
#endif // HAVE_STDINT_H
|
||||
|
||||
typedef std::string string;
|
||||
|
||||
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
|
||||
TypeName(const TypeName&); \
|
||||
void operator=(const TypeName&)
|
||||
|
||||
#if !@ac_cv_have_sys_uio_h@
|
||||
#if !${HAVE_SYS_UIO_H_01} // !HAVE_SYS_UIO_H
|
||||
// Windows does not have an iovec type, yet the concept is universally useful.
|
||||
// It is simple to define it ourselves, so we put it inside our own namespace.
|
||||
struct iovec {
|
||||
void* iov_base;
|
||||
size_t iov_len;
|
||||
};
|
||||
#endif
|
||||
#endif // !HAVE_SYS_UIO_H
|
||||
|
||||
} // namespace snappy
|
||||
|
||||
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
||||
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
||||
|
||||
@@ -28,13 +28,19 @@
|
||||
//
|
||||
// Various stubs for the unit tests for the open-source version of Snappy.
|
||||
|
||||
#include "snappy-test.h"
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_WINDOWS_H
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
// Needed to be able to use std::max without workarounds in the source code.
|
||||
// https://support.microsoft.com/en-us/help/143208/prb-using-stl-in-windows-program-can-cause-min-max-conflicts
|
||||
#define NOMINMAX
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
#include "snappy-test.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
DEFINE_bool(run_microbenchmarks, true,
|
||||
@@ -198,7 +204,7 @@ void Benchmark::Run() {
|
||||
if (benchmark_real_time_us > 0) {
|
||||
num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us;
|
||||
}
|
||||
num_iterations = max(num_iterations, kCalibrateIterations);
|
||||
num_iterations = std::max(num_iterations, kCalibrateIterations);
|
||||
BenchmarkRun benchmark_runs[kNumRuns];
|
||||
|
||||
for (int run = 0; run < kNumRuns; ++run) {
|
||||
@@ -214,7 +220,7 @@ void Benchmark::Run() {
|
||||
string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num);
|
||||
string human_readable_speed;
|
||||
|
||||
nth_element(benchmark_runs,
|
||||
std::nth_element(benchmark_runs,
|
||||
benchmark_runs + kMedianPos,
|
||||
benchmark_runs + kNumRuns,
|
||||
BenchmarkCompareCPUTime());
|
||||
@@ -520,8 +526,8 @@ int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen,
|
||||
LOG(WARNING)
|
||||
<< "UncompressChunkOrAll: Received some extra data, bytes total: "
|
||||
<< uncomp_stream_.avail_in << " bytes: "
|
||||
<< string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
|
||||
min(int(uncomp_stream_.avail_in), 20));
|
||||
<< std::string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
|
||||
std::min(int(uncomp_stream_.avail_in), 20));
|
||||
UncompressErrorInit();
|
||||
return Z_DATA_ERROR; // what's the extra data for?
|
||||
} else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
|
||||
|
||||
@@ -28,8 +28,8 @@
|
||||
//
|
||||
// Various stubs for the unit tests for the open-source version of Snappy.
|
||||
|
||||
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
|
||||
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
|
||||
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
|
||||
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
@@ -52,7 +52,6 @@
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_WINDOWS_H
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
@@ -111,35 +110,18 @@
|
||||
#include "lzo/lzo1x.h"
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_LIBLZF
|
||||
extern "C" {
|
||||
#include "lzf.h"
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_LIBFASTLZ
|
||||
#include "fastlz.h"
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_LIBQUICKLZ
|
||||
#include "quicklz.h"
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
namespace File {
|
||||
void Init() { }
|
||||
} // namespace File
|
||||
|
||||
namespace file {
|
||||
int Defaults() { }
|
||||
int Defaults() { return 0; }
|
||||
|
||||
class DummyStatus {
|
||||
public:
|
||||
void CheckSuccess() { }
|
||||
};
|
||||
|
||||
DummyStatus GetContents(const string& filename, string* data, int unused) {
|
||||
DummyStatus GetContents(
|
||||
const std::string& filename, std::string* data, int unused) {
|
||||
FILE* fp = fopen(filename.c_str(), "rb");
|
||||
if (fp == NULL) {
|
||||
perror(filename.c_str());
|
||||
@@ -154,15 +136,16 @@ namespace file {
|
||||
perror("fread");
|
||||
exit(1);
|
||||
}
|
||||
data->append(string(buf, ret));
|
||||
data->append(std::string(buf, ret));
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
|
||||
return DummyStatus();
|
||||
}
|
||||
|
||||
DummyStatus SetContents(const string& filename,
|
||||
const string& str,
|
||||
int unused) {
|
||||
inline DummyStatus SetContents(
|
||||
const std::string& filename, const std::string& str, int unused) {
|
||||
FILE* fp = fopen(filename.c_str(), "wb");
|
||||
if (fp == NULL) {
|
||||
perror(filename.c_str());
|
||||
@@ -176,6 +159,8 @@ namespace file {
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
|
||||
return DummyStatus();
|
||||
}
|
||||
} // namespace file
|
||||
|
||||
@@ -193,6 +178,7 @@ void Test_Snappy_RandomData();
|
||||
void Test_Snappy_FourByteOffset();
|
||||
void Test_SnappyCorruption_TruncatedVarint();
|
||||
void Test_SnappyCorruption_UnterminatedVarint();
|
||||
void Test_SnappyCorruption_OverflowingVarint();
|
||||
void Test_Snappy_ReadPastEndOfBuffer();
|
||||
void Test_Snappy_FindMatchLength();
|
||||
void Test_Snappy_FindMatchLengthRandom();
|
||||
@@ -463,7 +449,7 @@ class ZLib {
|
||||
|
||||
DECLARE_bool(run_microbenchmarks);
|
||||
|
||||
static void RunSpecifiedBenchmarks() {
|
||||
static inline void RunSpecifiedBenchmarks() {
|
||||
if (!FLAGS_run_microbenchmarks) {
|
||||
return;
|
||||
}
|
||||
@@ -497,6 +483,7 @@ static inline int RUN_ALL_TESTS() {
|
||||
snappy::Test_Snappy_FourByteOffset();
|
||||
snappy::Test_SnappyCorruption_TruncatedVarint();
|
||||
snappy::Test_SnappyCorruption_UnterminatedVarint();
|
||||
snappy::Test_SnappyCorruption_OverflowingVarint();
|
||||
snappy::Test_Snappy_ReadPastEndOfBuffer();
|
||||
snappy::Test_Snappy_FindMatchLength();
|
||||
snappy::Test_Snappy_FindMatchLengthRandom();
|
||||
@@ -510,10 +497,6 @@ static inline int RUN_ALL_TESTS() {
|
||||
// For main().
|
||||
namespace snappy {
|
||||
|
||||
static void CompressFile(const char* fname);
|
||||
static void UncompressFile(const char* fname);
|
||||
static void MeasureFile(const char* fname);
|
||||
|
||||
// Logging.
|
||||
|
||||
#define LOG(level) LogMessage()
|
||||
@@ -524,15 +507,15 @@ class LogMessage {
|
||||
public:
|
||||
LogMessage() { }
|
||||
~LogMessage() {
|
||||
cerr << endl;
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
|
||||
LogMessage& operator<<(const std::string& msg) {
|
||||
cerr << msg;
|
||||
std::cerr << msg;
|
||||
return *this;
|
||||
}
|
||||
LogMessage& operator<<(int x) {
|
||||
cerr << x;
|
||||
std::cerr << x;
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
@@ -541,18 +524,29 @@ class LogMessage {
|
||||
// and ones that are always active.
|
||||
|
||||
#define CRASH_UNLESS(condition) \
|
||||
PREDICT_TRUE(condition) ? (void)0 : \
|
||||
SNAPPY_PREDICT_TRUE(condition) ? (void)0 : \
|
||||
snappy::LogMessageVoidify() & snappy::LogMessageCrash()
|
||||
|
||||
#ifdef _MSC_VER
|
||||
// ~LogMessageCrash calls abort() and therefore never exits. This is by design
|
||||
// so temporarily disable warning C4722.
|
||||
#pragma warning(push)
|
||||
#pragma warning(disable:4722)
|
||||
#endif
|
||||
|
||||
class LogMessageCrash : public LogMessage {
|
||||
public:
|
||||
LogMessageCrash() { }
|
||||
~LogMessageCrash() {
|
||||
cerr << endl;
|
||||
std::cerr << std::endl;
|
||||
abort();
|
||||
}
|
||||
};
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
|
||||
// This class is used to explicitly ignore values in the conditional
|
||||
// logging macros. This avoids compiler warnings like "value computed
|
||||
// is not used" and "statement has no effect".
|
||||
@@ -572,11 +566,8 @@ class LogMessageVoidify {
|
||||
#define CHECK_NE(a, b) CRASH_UNLESS((a) != (b))
|
||||
#define CHECK_LT(a, b) CRASH_UNLESS((a) < (b))
|
||||
#define CHECK_GT(a, b) CRASH_UNLESS((a) > (b))
|
||||
#define CHECK_OK(cond) (cond).CheckSuccess()
|
||||
|
||||
} // namespace
|
||||
} // namespace snappy
|
||||
|
||||
using snappy::CompressFile;
|
||||
using snappy::UncompressFile;
|
||||
using snappy::MeasureFile;
|
||||
|
||||
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
|
||||
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -36,8 +36,8 @@
|
||||
// using BMDiff and then compressing the output of BMDiff with
|
||||
// Snappy.
|
||||
|
||||
#ifndef UTIL_SNAPPY_SNAPPY_H__
|
||||
#define UTIL_SNAPPY_SNAPPY_H__
|
||||
#ifndef THIRD_PARTY_SNAPPY_SNAPPY_H__
|
||||
#define THIRD_PARTY_SNAPPY_SNAPPY_H__
|
||||
|
||||
#include <stddef.h>
|
||||
#include <string>
|
||||
@@ -84,6 +84,18 @@ namespace snappy {
|
||||
bool Uncompress(const char* compressed, size_t compressed_length,
|
||||
string* uncompressed);
|
||||
|
||||
// Decompresses "compressed" to "*uncompressed".
|
||||
//
|
||||
// returns false if the message is corrupted and could not be decompressed
|
||||
bool Uncompress(Source* compressed, Sink* uncompressed);
|
||||
|
||||
// This routine uncompresses as much of the "compressed" as possible
|
||||
// into sink. It returns the number of valid bytes added to sink
|
||||
// (extra invalid bytes may have been added due to errors; the caller
|
||||
// should ignore those). The emitted data typically has length
|
||||
// GetUncompressedLength(), but may be shorter if an error is
|
||||
// encountered.
|
||||
size_t UncompressAsMuchAsPossible(Source* compressed, Sink* uncompressed);
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// Lower-level character array based routines. May be useful for
|
||||
@@ -164,6 +176,14 @@ namespace snappy {
|
||||
bool IsValidCompressedBuffer(const char* compressed,
|
||||
size_t compressed_length);
|
||||
|
||||
// Returns true iff the contents of "compressed" can be uncompressed
|
||||
// successfully. Does not return the uncompressed data. Takes
|
||||
// time proportional to *compressed length, but is usually at least
|
||||
// a factor of four faster than actual decompression.
|
||||
// On success, consumes all of *compressed. On failure, consumes an
|
||||
// unspecified prefix of *compressed.
|
||||
bool IsValidCompressed(Source* compressed);
|
||||
|
||||
// The size of a compression block. Note that many parts of the compression
|
||||
// code assumes that kBlockSize <= 65536; in particular, the hash table
|
||||
// can only store 16-bit offsets, and EmitCopy() also assumes the offset
|
||||
@@ -180,5 +200,4 @@ namespace snappy {
|
||||
static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
|
||||
} // end namespace snappy
|
||||
|
||||
|
||||
#endif // UTIL_SNAPPY_SNAPPY_H__
|
||||
#endif // THIRD_PARTY_SNAPPY_SNAPPY_H__
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "snappy.h"
|
||||
@@ -50,25 +51,19 @@ DEFINE_bool(zlib, false,
|
||||
"Run zlib compression (http://www.zlib.net)");
|
||||
DEFINE_bool(lzo, false,
|
||||
"Run LZO compression (http://www.oberhumer.com/opensource/lzo/)");
|
||||
DEFINE_bool(quicklz, false,
|
||||
"Run quickLZ compression (http://www.quicklz.com/)");
|
||||
DEFINE_bool(liblzf, false,
|
||||
"Run libLZF compression "
|
||||
"(http://www.goof.com/pcg/marc/liblzf.html)");
|
||||
DEFINE_bool(fastlz, false,
|
||||
"Run FastLZ compression (http://www.fastlz.org/");
|
||||
DEFINE_bool(snappy, true, "Run snappy compression");
|
||||
|
||||
|
||||
DEFINE_bool(write_compressed, false,
|
||||
"Write compressed versions of each file to <file>.comp");
|
||||
DEFINE_bool(write_uncompressed, false,
|
||||
"Write uncompressed versions of each file to <file>.uncomp");
|
||||
|
||||
DEFINE_bool(snappy_dump_decompression_table, false,
|
||||
"If true, we print the decompression table during tests.");
|
||||
|
||||
namespace snappy {
|
||||
|
||||
|
||||
#ifdef HAVE_FUNC_MMAP
|
||||
#if defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF)
|
||||
|
||||
// To test against code that reads beyond its input, this class copies a
|
||||
// string to a newly allocated group of pages, the last of which
|
||||
@@ -79,7 +74,7 @@ namespace snappy {
|
||||
class DataEndingAtUnreadablePage {
|
||||
public:
|
||||
explicit DataEndingAtUnreadablePage(const string& s) {
|
||||
const size_t page_size = getpagesize();
|
||||
const size_t page_size = sysconf(_SC_PAGESIZE);
|
||||
const size_t size = s.size();
|
||||
// Round up space for string to a multiple of page_size.
|
||||
size_t space_for_string = (size + page_size - 1) & ~(page_size - 1);
|
||||
@@ -97,8 +92,9 @@ class DataEndingAtUnreadablePage {
|
||||
}
|
||||
|
||||
~DataEndingAtUnreadablePage() {
|
||||
const size_t page_size = sysconf(_SC_PAGESIZE);
|
||||
// Undo the mprotect.
|
||||
CHECK_EQ(0, mprotect(protected_page_, getpagesize(), PROT_READ|PROT_WRITE));
|
||||
CHECK_EQ(0, mprotect(protected_page_, page_size, PROT_READ|PROT_WRITE));
|
||||
CHECK_EQ(0, munmap(mem_, alloc_size_));
|
||||
}
|
||||
|
||||
@@ -113,7 +109,7 @@ class DataEndingAtUnreadablePage {
|
||||
size_t size_;
|
||||
};
|
||||
|
||||
#else // HAVE_FUNC_MMAP
|
||||
#else // defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF)
|
||||
|
||||
// Fallback for systems without mmap.
|
||||
typedef string DataEndingAtUnreadablePage;
|
||||
@@ -121,11 +117,11 @@ typedef string DataEndingAtUnreadablePage;
|
||||
#endif
|
||||
|
||||
enum CompressorType {
|
||||
ZLIB, LZO, LIBLZF, QUICKLZ, FASTLZ, SNAPPY
|
||||
ZLIB, LZO, SNAPPY
|
||||
};
|
||||
|
||||
const char* names[] = {
|
||||
"ZLIB", "LZO", "LIBLZF", "QUICKLZ", "FASTLZ", "SNAPPY"
|
||||
"ZLIB", "LZO", "SNAPPY"
|
||||
};
|
||||
|
||||
static size_t MinimumRequiredOutputSpace(size_t input_size,
|
||||
@@ -141,26 +137,12 @@ static size_t MinimumRequiredOutputSpace(size_t input_size,
|
||||
return input_size + input_size/64 + 16 + 3;
|
||||
#endif // LZO_VERSION
|
||||
|
||||
#ifdef LZF_VERSION
|
||||
case LIBLZF:
|
||||
return input_size;
|
||||
#endif // LZF_VERSION
|
||||
|
||||
#ifdef QLZ_VERSION_MAJOR
|
||||
case QUICKLZ:
|
||||
return input_size + 36000; // 36000 is used for scratch.
|
||||
#endif // QLZ_VERSION_MAJOR
|
||||
|
||||
#ifdef FASTLZ_VERSION
|
||||
case FASTLZ:
|
||||
return max(static_cast<int>(ceil(input_size * 1.05)), 66);
|
||||
#endif // FASTLZ_VERSION
|
||||
|
||||
case SNAPPY:
|
||||
return snappy::MaxCompressedLength(input_size);
|
||||
|
||||
default:
|
||||
LOG(FATAL) << "Unknown compression type number " << comp;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,58 +196,6 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
|
||||
}
|
||||
#endif // LZO_VERSION
|
||||
|
||||
#ifdef LZF_VERSION
|
||||
case LIBLZF: {
|
||||
int destlen = lzf_compress(input,
|
||||
input_size,
|
||||
string_as_array(compressed),
|
||||
input_size);
|
||||
if (destlen == 0) {
|
||||
// lzf *can* cause lots of blowup when compressing, so they
|
||||
// recommend to limit outsize to insize, and just not compress
|
||||
// if it's bigger. Ideally, we'd just swap input and output.
|
||||
compressed->assign(input, input_size);
|
||||
destlen = input_size;
|
||||
}
|
||||
if (!compressed_is_preallocated) {
|
||||
compressed->resize(destlen);
|
||||
}
|
||||
break;
|
||||
}
|
||||
#endif // LZF_VERSION
|
||||
|
||||
#ifdef QLZ_VERSION_MAJOR
|
||||
case QUICKLZ: {
|
||||
qlz_state_compress *state_compress = new qlz_state_compress;
|
||||
int destlen = qlz_compress(input,
|
||||
string_as_array(compressed),
|
||||
input_size,
|
||||
state_compress);
|
||||
delete state_compress;
|
||||
CHECK_NE(0, destlen);
|
||||
if (!compressed_is_preallocated) {
|
||||
compressed->resize(destlen);
|
||||
}
|
||||
break;
|
||||
}
|
||||
#endif // QLZ_VERSION_MAJOR
|
||||
|
||||
#ifdef FASTLZ_VERSION
|
||||
case FASTLZ: {
|
||||
// Use level 1 compression since we mostly care about speed.
|
||||
int destlen = fastlz_compress_level(
|
||||
1,
|
||||
input,
|
||||
input_size,
|
||||
string_as_array(compressed));
|
||||
if (!compressed_is_preallocated) {
|
||||
compressed->resize(destlen);
|
||||
}
|
||||
CHECK_NE(destlen, 0);
|
||||
break;
|
||||
}
|
||||
#endif // FASTLZ_VERSION
|
||||
|
||||
case SNAPPY: {
|
||||
size_t destlen;
|
||||
snappy::RawCompress(input, input_size,
|
||||
@@ -278,7 +208,6 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
default: {
|
||||
return false; // the asked-for library wasn't compiled in
|
||||
}
|
||||
@@ -321,56 +250,12 @@ static bool Uncompress(const string& compressed, CompressorType comp,
|
||||
}
|
||||
#endif // LZO_VERSION
|
||||
|
||||
#ifdef LZF_VERSION
|
||||
case LIBLZF: {
|
||||
output->resize(size);
|
||||
int destlen = lzf_decompress(compressed.data(),
|
||||
compressed.size(),
|
||||
string_as_array(output),
|
||||
output->size());
|
||||
if (destlen == 0) {
|
||||
// This error probably means we had decided not to compress,
|
||||
// and thus have stored input in output directly.
|
||||
output->assign(compressed.data(), compressed.size());
|
||||
destlen = compressed.size();
|
||||
}
|
||||
CHECK_EQ(destlen, size);
|
||||
break;
|
||||
}
|
||||
#endif // LZF_VERSION
|
||||
|
||||
#ifdef QLZ_VERSION_MAJOR
|
||||
case QUICKLZ: {
|
||||
output->resize(size);
|
||||
qlz_state_decompress *state_decompress = new qlz_state_decompress;
|
||||
int destlen = qlz_decompress(compressed.data(),
|
||||
string_as_array(output),
|
||||
state_decompress);
|
||||
delete state_decompress;
|
||||
CHECK_EQ(destlen, size);
|
||||
break;
|
||||
}
|
||||
#endif // QLZ_VERSION_MAJOR
|
||||
|
||||
#ifdef FASTLZ_VERSION
|
||||
case FASTLZ: {
|
||||
output->resize(size);
|
||||
int destlen = fastlz_decompress(compressed.data(),
|
||||
compressed.length(),
|
||||
string_as_array(output),
|
||||
size);
|
||||
CHECK_EQ(destlen, size);
|
||||
break;
|
||||
}
|
||||
#endif // FASTLZ_VERSION
|
||||
|
||||
case SNAPPY: {
|
||||
snappy::RawUncompress(compressed.data(), compressed.size(),
|
||||
string_as_array(output));
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
default: {
|
||||
return false; // the asked-for library wasn't compiled in
|
||||
}
|
||||
@@ -392,13 +277,13 @@ static void Measure(const char* data,
|
||||
{
|
||||
// Chop the input into blocks
|
||||
int num_blocks = (length + block_size - 1) / block_size;
|
||||
vector<const char*> input(num_blocks);
|
||||
vector<size_t> input_length(num_blocks);
|
||||
vector<string> compressed(num_blocks);
|
||||
vector<string> output(num_blocks);
|
||||
std::vector<const char*> input(num_blocks);
|
||||
std::vector<size_t> input_length(num_blocks);
|
||||
std::vector<string> compressed(num_blocks);
|
||||
std::vector<string> output(num_blocks);
|
||||
for (int b = 0; b < num_blocks; b++) {
|
||||
int input_start = b * block_size;
|
||||
int input_limit = min<int>((b+1)*block_size, length);
|
||||
int input_limit = std::min<int>((b+1)*block_size, length);
|
||||
input[b] = data+input_start;
|
||||
input_length[b] = input_limit-input_start;
|
||||
|
||||
@@ -448,13 +333,13 @@ static void Measure(const char* data,
|
||||
}
|
||||
|
||||
compressed_size = 0;
|
||||
for (int i = 0; i < compressed.size(); i++) {
|
||||
for (size_t i = 0; i < compressed.size(); i++) {
|
||||
compressed_size += compressed[i].size();
|
||||
}
|
||||
}
|
||||
|
||||
sort(ctime, ctime + kRuns);
|
||||
sort(utime, utime + kRuns);
|
||||
std::sort(ctime, ctime + kRuns);
|
||||
std::sort(utime, utime + kRuns);
|
||||
const int med = kRuns/2;
|
||||
|
||||
float comp_rate = (length / ctime[med]) * repeats / 1048576.0;
|
||||
@@ -469,12 +354,11 @@ static void Measure(const char* data,
|
||||
x.c_str(),
|
||||
block_size/(1<<20),
|
||||
static_cast<int>(length), static_cast<uint32>(compressed_size),
|
||||
(compressed_size * 100.0) / max<int>(1, length),
|
||||
(compressed_size * 100.0) / std::max<int>(1, length),
|
||||
comp_rate,
|
||||
urate.c_str());
|
||||
}
|
||||
|
||||
|
||||
static int VerifyString(const string& input) {
|
||||
string compressed;
|
||||
DataEndingAtUnreadablePage i(input);
|
||||
@@ -491,6 +375,23 @@ static int VerifyString(const string& input) {
|
||||
return uncompressed.size();
|
||||
}
|
||||
|
||||
static void VerifyStringSink(const string& input) {
|
||||
string compressed;
|
||||
DataEndingAtUnreadablePage i(input);
|
||||
const size_t written = snappy::Compress(i.data(), i.size(), &compressed);
|
||||
CHECK_EQ(written, compressed.size());
|
||||
CHECK_LE(compressed.size(),
|
||||
snappy::MaxCompressedLength(input.size()));
|
||||
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
|
||||
|
||||
string uncompressed;
|
||||
uncompressed.resize(input.size());
|
||||
snappy::UncheckedByteArraySink sink(string_as_array(&uncompressed));
|
||||
DataEndingAtUnreadablePage c(compressed);
|
||||
snappy::ByteArraySource source(c.data(), c.size());
|
||||
CHECK(snappy::Uncompress(&source, &sink));
|
||||
CHECK_EQ(uncompressed, input);
|
||||
}
|
||||
|
||||
static void VerifyIOVec(const string& input) {
|
||||
string compressed;
|
||||
@@ -505,13 +406,13 @@ static void VerifyIOVec(const string& input) {
|
||||
// ranging from 1 to 10.
|
||||
char* buf = new char[input.size()];
|
||||
ACMRandom rnd(input.size());
|
||||
int num = rnd.Next() % 10 + 1;
|
||||
size_t num = rnd.Next() % 10 + 1;
|
||||
if (input.size() < num) {
|
||||
num = input.size();
|
||||
}
|
||||
struct iovec* iov = new iovec[num];
|
||||
int used_so_far = 0;
|
||||
for (int i = 0; i < num; ++i) {
|
||||
for (size_t i = 0; i < num; ++i) {
|
||||
iov[i].iov_base = buf + used_so_far;
|
||||
if (i == num - 1) {
|
||||
iov[i].iov_len = input.size() - used_so_far;
|
||||
@@ -562,6 +463,28 @@ static void VerifyNonBlockedCompression(const string& input) {
|
||||
CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncomp_str));
|
||||
CHECK_EQ(uncomp_str, input);
|
||||
|
||||
// Uncompress using source/sink
|
||||
string uncomp_str2;
|
||||
uncomp_str2.resize(input.size());
|
||||
snappy::UncheckedByteArraySink sink(string_as_array(&uncomp_str2));
|
||||
snappy::ByteArraySource source(compressed.data(), compressed.size());
|
||||
CHECK(snappy::Uncompress(&source, &sink));
|
||||
CHECK_EQ(uncomp_str2, input);
|
||||
|
||||
// Uncompress into iovec
|
||||
{
|
||||
static const int kNumBlocks = 10;
|
||||
struct iovec vec[kNumBlocks];
|
||||
const int block_size = 1 + input.size() / kNumBlocks;
|
||||
string iovec_data(block_size * kNumBlocks, 'x');
|
||||
for (int i = 0; i < kNumBlocks; i++) {
|
||||
vec[i].iov_base = string_as_array(&iovec_data) + i * block_size;
|
||||
vec[i].iov_len = block_size;
|
||||
}
|
||||
CHECK(snappy::RawUncompressToIOVec(compressed.data(), compressed.size(),
|
||||
vec, kNumBlocks));
|
||||
CHECK_EQ(string(iovec_data.data(), input.size()), input);
|
||||
}
|
||||
}
|
||||
|
||||
// Expand the input so that it is at least K times as big as block size
|
||||
@@ -580,6 +503,8 @@ static int Verify(const string& input) {
|
||||
// Compress using string based routines
|
||||
const int result = VerifyString(input);
|
||||
|
||||
// Verify using sink based routines
|
||||
VerifyStringSink(input);
|
||||
|
||||
VerifyNonBlockedCompression(input);
|
||||
VerifyIOVec(input);
|
||||
@@ -589,12 +514,9 @@ static int Verify(const string& input) {
|
||||
VerifyIOVec(input);
|
||||
}
|
||||
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// This test checks to ensure that snappy doesn't coredump if it gets
|
||||
// corrupted data.
|
||||
|
||||
static bool IsValidCompressedBuffer(const string& c) {
|
||||
return snappy::IsValidCompressedBuffer(c.data(), c.size());
|
||||
@@ -603,11 +525,13 @@ static bool Uncompress(const string& c, string* u) {
|
||||
return snappy::Uncompress(c.data(), c.size(), u);
|
||||
}
|
||||
|
||||
TYPED_TEST(CorruptedTest, VerifyCorrupted) {
|
||||
// This test checks to ensure that snappy doesn't coredump if it gets
|
||||
// corrupted data.
|
||||
TEST(CorruptedTest, VerifyCorrupted) {
|
||||
string source = "making sure we don't crash with corrupted input";
|
||||
VLOG(1) << source;
|
||||
string dest;
|
||||
TypeParam uncmp;
|
||||
string uncmp;
|
||||
snappy::Compress(source.data(), source.size(), &dest);
|
||||
|
||||
// Mess around with the data. It's hard to simulate all possible
|
||||
@@ -616,19 +540,19 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
|
||||
dest[1]--;
|
||||
dest[3]++;
|
||||
// this really ought to fail.
|
||||
CHECK(!IsValidCompressedBuffer(TypeParam(dest)));
|
||||
CHECK(!Uncompress(TypeParam(dest), &uncmp));
|
||||
CHECK(!IsValidCompressedBuffer(dest));
|
||||
CHECK(!Uncompress(dest, &uncmp));
|
||||
|
||||
// This is testing for a security bug - a buffer that decompresses to 100k
|
||||
// but we lie in the snappy header and only reserve 0 bytes of memory :)
|
||||
source.resize(100000);
|
||||
for (int i = 0; i < source.length(); ++i) {
|
||||
for (size_t i = 0; i < source.length(); ++i) {
|
||||
source[i] = 'A';
|
||||
}
|
||||
snappy::Compress(source.data(), source.size(), &dest);
|
||||
dest[0] = dest[1] = dest[2] = dest[3] = 0;
|
||||
CHECK(!IsValidCompressedBuffer(TypeParam(dest)));
|
||||
CHECK(!Uncompress(TypeParam(dest), &uncmp));
|
||||
CHECK(!IsValidCompressedBuffer(dest));
|
||||
CHECK(!Uncompress(dest, &uncmp));
|
||||
|
||||
if (sizeof(void *) == 4) {
|
||||
// Another security check; check a crazy big length can't DoS us with an
|
||||
@@ -637,20 +561,20 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
|
||||
// where 3 GB might be an acceptable allocation size, Uncompress()
|
||||
// attempts to decompress, and sometimes causes the test to run out of
|
||||
// memory.
|
||||
dest[0] = dest[1] = dest[2] = dest[3] = 0xff;
|
||||
dest[0] = dest[1] = dest[2] = dest[3] = '\xff';
|
||||
// This decodes to a really large size, i.e., about 3 GB.
|
||||
dest[4] = 'k';
|
||||
CHECK(!IsValidCompressedBuffer(TypeParam(dest)));
|
||||
CHECK(!Uncompress(TypeParam(dest), &uncmp));
|
||||
CHECK(!IsValidCompressedBuffer(dest));
|
||||
CHECK(!Uncompress(dest, &uncmp));
|
||||
} else {
|
||||
LOG(WARNING) << "Crazy decompression lengths not checked on 64-bit build";
|
||||
}
|
||||
|
||||
// This decodes to about 2 MB; much smaller, but should still fail.
|
||||
dest[0] = dest[1] = dest[2] = 0xff;
|
||||
dest[0] = dest[1] = dest[2] = '\xff';
|
||||
dest[3] = 0x00;
|
||||
CHECK(!IsValidCompressedBuffer(TypeParam(dest)));
|
||||
CHECK(!Uncompress(TypeParam(dest), &uncmp));
|
||||
CHECK(!IsValidCompressedBuffer(dest));
|
||||
CHECK(!Uncompress(dest, &uncmp));
|
||||
|
||||
// try reading stuff in from a bad file.
|
||||
for (int i = 1; i <= 3; ++i) {
|
||||
@@ -665,8 +589,8 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
|
||||
snappy::ByteArraySource source(data.data(), data.size());
|
||||
CHECK(!snappy::GetUncompressedLength(&source, &ulen2) ||
|
||||
(ulen2 < (1<<20)));
|
||||
CHECK(!IsValidCompressedBuffer(TypeParam(data)));
|
||||
CHECK(!Uncompress(TypeParam(data), &uncmp));
|
||||
CHECK(!IsValidCompressedBuffer(data));
|
||||
CHECK(!Uncompress(data, &uncmp));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -764,7 +688,7 @@ TEST(Snappy, RandomData) {
|
||||
}
|
||||
|
||||
string x;
|
||||
int len = rnd.Uniform(4096);
|
||||
size_t len = rnd.Uniform(4096);
|
||||
if (i < 100) {
|
||||
len = 65536 + rnd.Uniform(65536);
|
||||
}
|
||||
@@ -929,7 +853,6 @@ TEST(Snappy, IOVecCopyOverflow) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static bool CheckUncompressedLength(const string& compressed,
|
||||
size_t* ulength) {
|
||||
const bool result1 = snappy::GetUncompressedLength(compressed.data(),
|
||||
@@ -956,11 +879,11 @@ TEST(SnappyCorruption, TruncatedVarint) {
|
||||
TEST(SnappyCorruption, UnterminatedVarint) {
|
||||
string compressed, uncompressed;
|
||||
size_t ulength;
|
||||
compressed.push_back(128);
|
||||
compressed.push_back(128);
|
||||
compressed.push_back(128);
|
||||
compressed.push_back(128);
|
||||
compressed.push_back(128);
|
||||
compressed.push_back('\x80');
|
||||
compressed.push_back('\x80');
|
||||
compressed.push_back('\x80');
|
||||
compressed.push_back('\x80');
|
||||
compressed.push_back('\x80');
|
||||
compressed.push_back(10);
|
||||
CHECK(!CheckUncompressedLength(compressed, &ulength));
|
||||
CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
|
||||
@@ -968,6 +891,20 @@ TEST(SnappyCorruption, UnterminatedVarint) {
|
||||
&uncompressed));
|
||||
}
|
||||
|
||||
TEST(SnappyCorruption, OverflowingVarint) {
|
||||
string compressed, uncompressed;
|
||||
size_t ulength;
|
||||
compressed.push_back('\xfb');
|
||||
compressed.push_back('\xff');
|
||||
compressed.push_back('\xff');
|
||||
compressed.push_back('\xff');
|
||||
compressed.push_back('\x7f');
|
||||
CHECK(!CheckUncompressedLength(compressed, &ulength));
|
||||
CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
|
||||
CHECK(!snappy::Uncompress(compressed.data(), compressed.size(),
|
||||
&uncompressed));
|
||||
}
|
||||
|
||||
TEST(Snappy, ReadPastEndOfBuffer) {
|
||||
// Check that we do not read past end of input
|
||||
|
||||
@@ -998,11 +935,13 @@ TEST(Snappy, ZeroOffsetCopyValidation) {
|
||||
EXPECT_FALSE(snappy::IsValidCompressedBuffer(compressed, 4));
|
||||
}
|
||||
|
||||
|
||||
namespace {
|
||||
|
||||
int TestFindMatchLength(const char* s1, const char *s2, unsigned length) {
|
||||
return snappy::internal::FindMatchLength(s1, s2, s2 + length);
|
||||
std::pair<size_t, bool> p =
|
||||
snappy::internal::FindMatchLength(s1, s2, s2 + length);
|
||||
CHECK_EQ(p.first < 8, p.second);
|
||||
return p.first;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
@@ -1112,8 +1051,7 @@ TEST(Snappy, FindMatchLengthRandom) {
|
||||
}
|
||||
DataEndingAtUnreadablePage u(s);
|
||||
DataEndingAtUnreadablePage v(t);
|
||||
int matched = snappy::internal::FindMatchLength(
|
||||
u.data(), v.data(), v.data() + t.size());
|
||||
int matched = TestFindMatchLength(u.data(), v.data(), t.size());
|
||||
if (matched == t.size()) {
|
||||
EXPECT_EQ(s, t);
|
||||
} else {
|
||||
@@ -1125,21 +1063,113 @@ TEST(Snappy, FindMatchLengthRandom) {
|
||||
}
|
||||
}
|
||||
|
||||
static uint16 MakeEntry(unsigned int extra,
|
||||
unsigned int len,
|
||||
unsigned int copy_offset) {
|
||||
// Check that all of the fields fit within the allocated space
|
||||
assert(extra == (extra & 0x7)); // At most 3 bits
|
||||
assert(copy_offset == (copy_offset & 0x7)); // At most 3 bits
|
||||
assert(len == (len & 0x7f)); // At most 7 bits
|
||||
return len | (copy_offset << 8) | (extra << 11);
|
||||
}
|
||||
|
||||
// Check that the decompression table is correct, and optionally print out
|
||||
// the computed one.
|
||||
TEST(Snappy, VerifyCharTable) {
|
||||
using snappy::internal::LITERAL;
|
||||
using snappy::internal::COPY_1_BYTE_OFFSET;
|
||||
using snappy::internal::COPY_2_BYTE_OFFSET;
|
||||
using snappy::internal::COPY_4_BYTE_OFFSET;
|
||||
using snappy::internal::char_table;
|
||||
|
||||
uint16 dst[256];
|
||||
|
||||
// Place invalid entries in all places to detect missing initialization
|
||||
int assigned = 0;
|
||||
for (int i = 0; i < 256; i++) {
|
||||
dst[i] = 0xffff;
|
||||
}
|
||||
|
||||
// Small LITERAL entries. We store (len-1) in the top 6 bits.
|
||||
for (unsigned int len = 1; len <= 60; len++) {
|
||||
dst[LITERAL | ((len-1) << 2)] = MakeEntry(0, len, 0);
|
||||
assigned++;
|
||||
}
|
||||
|
||||
// Large LITERAL entries. We use 60..63 in the high 6 bits to
|
||||
// encode the number of bytes of length info that follow the opcode.
|
||||
for (unsigned int extra_bytes = 1; extra_bytes <= 4; extra_bytes++) {
|
||||
// We set the length field in the lookup table to 1 because extra
|
||||
// bytes encode len-1.
|
||||
dst[LITERAL | ((extra_bytes+59) << 2)] = MakeEntry(extra_bytes, 1, 0);
|
||||
assigned++;
|
||||
}
|
||||
|
||||
// COPY_1_BYTE_OFFSET.
|
||||
//
|
||||
// The tag byte in the compressed data stores len-4 in 3 bits, and
|
||||
// offset/256 in 5 bits. offset%256 is stored in the next byte.
|
||||
//
|
||||
// This format is used for length in range [4..11] and offset in
|
||||
// range [0..2047]
|
||||
for (unsigned int len = 4; len < 12; len++) {
|
||||
for (unsigned int offset = 0; offset < 2048; offset += 256) {
|
||||
dst[COPY_1_BYTE_OFFSET | ((len-4)<<2) | ((offset>>8)<<5)] =
|
||||
MakeEntry(1, len, offset>>8);
|
||||
assigned++;
|
||||
}
|
||||
}
|
||||
|
||||
// COPY_2_BYTE_OFFSET.
|
||||
// Tag contains len-1 in top 6 bits, and offset in next two bytes.
|
||||
for (unsigned int len = 1; len <= 64; len++) {
|
||||
dst[COPY_2_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(2, len, 0);
|
||||
assigned++;
|
||||
}
|
||||
|
||||
// COPY_4_BYTE_OFFSET.
|
||||
// Tag contents len-1 in top 6 bits, and offset in next four bytes.
|
||||
for (unsigned int len = 1; len <= 64; len++) {
|
||||
dst[COPY_4_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(4, len, 0);
|
||||
assigned++;
|
||||
}
|
||||
|
||||
// Check that each entry was initialized exactly once.
|
||||
EXPECT_EQ(256, assigned) << "Assigned only " << assigned << " of 256";
|
||||
for (int i = 0; i < 256; i++) {
|
||||
EXPECT_NE(0xffff, dst[i]) << "Did not assign byte " << i;
|
||||
}
|
||||
|
||||
if (FLAGS_snappy_dump_decompression_table) {
|
||||
printf("static const uint16 char_table[256] = {\n ");
|
||||
for (int i = 0; i < 256; i++) {
|
||||
printf("0x%04x%s",
|
||||
dst[i],
|
||||
((i == 255) ? "\n" : (((i%8) == 7) ? ",\n " : ", ")));
|
||||
}
|
||||
printf("};\n");
|
||||
}
|
||||
|
||||
// Check that computed table matched recorded table.
|
||||
for (int i = 0; i < 256; i++) {
|
||||
EXPECT_EQ(dst[i], char_table[i]) << "Mismatch in byte " << i;
|
||||
}
|
||||
}
|
||||
|
||||
static void CompressFile(const char* fname) {
|
||||
string fullinput;
|
||||
file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
|
||||
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
|
||||
|
||||
string compressed;
|
||||
Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false);
|
||||
|
||||
file::SetContents(string(fname).append(".comp"), compressed, file::Defaults())
|
||||
.CheckSuccess();
|
||||
CHECK_OK(file::SetContents(string(fname).append(".comp"), compressed,
|
||||
file::Defaults()));
|
||||
}
|
||||
|
||||
static void UncompressFile(const char* fname) {
|
||||
string fullinput;
|
||||
file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
|
||||
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
|
||||
|
||||
size_t uncompLength;
|
||||
CHECK(CheckUncompressedLength(fullinput, &uncompLength));
|
||||
@@ -1148,28 +1178,25 @@ static void UncompressFile(const char* fname) {
|
||||
uncompressed.resize(uncompLength);
|
||||
CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed));
|
||||
|
||||
file::SetContents(string(fname).append(".uncomp"), uncompressed,
|
||||
file::Defaults()).CheckSuccess();
|
||||
CHECK_OK(file::SetContents(string(fname).append(".uncomp"), uncompressed,
|
||||
file::Defaults()));
|
||||
}
|
||||
|
||||
static void MeasureFile(const char* fname) {
|
||||
string fullinput;
|
||||
file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
|
||||
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
|
||||
printf("%-40s :\n", fname);
|
||||
|
||||
int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len;
|
||||
int end_len = fullinput.size();
|
||||
if (FLAGS_end_len >= 0) {
|
||||
end_len = min<int>(fullinput.size(), FLAGS_end_len);
|
||||
end_len = std::min<int>(fullinput.size(), FLAGS_end_len);
|
||||
}
|
||||
for (int len = start_len; len <= end_len; len++) {
|
||||
const char* const input = fullinput.data();
|
||||
int repeats = (FLAGS_bytes + len) / (len + 1);
|
||||
if (FLAGS_zlib) Measure(input, len, ZLIB, repeats, 1024<<10);
|
||||
if (FLAGS_lzo) Measure(input, len, LZO, repeats, 1024<<10);
|
||||
if (FLAGS_liblzf) Measure(input, len, LIBLZF, repeats, 1024<<10);
|
||||
if (FLAGS_quicklz) Measure(input, len, QUICKLZ, repeats, 1024<<10);
|
||||
if (FLAGS_fastlz) Measure(input, len, FASTLZ, repeats, 1024<<10);
|
||||
if (FLAGS_snappy) Measure(input, len, SNAPPY, repeats, 4096<<10);
|
||||
|
||||
// For block-size based measurements
|
||||
@@ -1298,6 +1325,37 @@ static void BM_UIOVec(int iters, int arg) {
|
||||
}
|
||||
BENCHMARK(BM_UIOVec)->DenseRange(0, 4);
|
||||
|
||||
static void BM_UFlatSink(int iters, int arg) {
|
||||
StopBenchmarkTiming();
|
||||
|
||||
// Pick file to process based on "arg"
|
||||
CHECK_GE(arg, 0);
|
||||
CHECK_LT(arg, ARRAYSIZE(files));
|
||||
string contents = ReadTestDataFile(files[arg].filename,
|
||||
files[arg].size_limit);
|
||||
|
||||
string zcontents;
|
||||
snappy::Compress(contents.data(), contents.size(), &zcontents);
|
||||
char* dst = new char[contents.size()];
|
||||
|
||||
SetBenchmarkBytesProcessed(static_cast<int64>(iters) *
|
||||
static_cast<int64>(contents.size()));
|
||||
SetBenchmarkLabel(files[arg].label);
|
||||
StartBenchmarkTiming();
|
||||
while (iters-- > 0) {
|
||||
snappy::ByteArraySource source(zcontents.data(), zcontents.size());
|
||||
snappy::UncheckedByteArraySink sink(dst);
|
||||
CHECK(snappy::Uncompress(&source, &sink));
|
||||
}
|
||||
StopBenchmarkTiming();
|
||||
|
||||
string s(dst, contents.size());
|
||||
CHECK_EQ(contents, s);
|
||||
|
||||
delete[] dst;
|
||||
}
|
||||
|
||||
BENCHMARK(BM_UFlatSink)->DenseRange(0, ARRAYSIZE(files) - 1);
|
||||
|
||||
static void BM_ZFlat(int iters, int arg) {
|
||||
StopBenchmarkTiming();
|
||||
@@ -1329,23 +1387,20 @@ static void BM_ZFlat(int iters, int arg) {
|
||||
}
|
||||
BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
|
||||
|
||||
|
||||
} // namespace snappy
|
||||
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
InitGoogle(argv[0], &argc, &argv, true);
|
||||
RunSpecifiedBenchmarks();
|
||||
|
||||
|
||||
if (argc >= 2) {
|
||||
for (int arg = 1; arg < argc; arg++) {
|
||||
if (FLAGS_write_compressed) {
|
||||
CompressFile(argv[arg]);
|
||||
snappy::CompressFile(argv[arg]);
|
||||
} else if (FLAGS_write_uncompressed) {
|
||||
UncompressFile(argv[arg]);
|
||||
snappy::UncompressFile(argv[arg]);
|
||||
} else {
|
||||
MeasureFile(argv[arg]);
|
||||
snappy::MeasureFile(argv[arg]);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
|
||||
Reference in New Issue
Block a user