ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Squashed 'src/snappy/snappy/' changes from 1ff9be9b8..b02bfa754

Browse Source 
b02bfa754 Tag open source release 1.1.7.
824e6718b Add a loop alignment directive to work around a performance regression.
55924d110 Add GNUInstallDirs to CMake configuration.
632cd0f12 Use 64-bit optimized code path for ARM64.
77c12adc1 Add unistd.h checks back to the CMake build.
c8049c582 Replace getpagesize() with sysconf(_SC_PAGESIZE).
18e2f220d Add guidelines for opensource contributions.
f0d3237c3 Use _BitScanForward and _BitScanReverse on MSVC.
71b8f8688 Add SNAPPY_ prefix to PREDICT_{TRUE,FALSE} macros.
be6dc3db8 Redo CMake configuration.
e4de6ce08 Small improvements to open source CI configuration.
c756f7f5d Support both static and shared library CMake builds.
038a3329b Inline DISALLOW_COPY_AND_ASSIGN.
a8b239c3d snappy: Remove autoconf build configuration.
27671c6ae Clean up CMake header and type checks.
548501c98 zippy: Re-release snappy 1.1.5 as 1.1.6.
513df5fb5 Tag open source release 1.1.5.
5bc9c82ae Set minimum CMake version to 3.1.
e9720a001 Update Travis CI config, add AppVeyor for Windows CI coverage.
f24f9d2d9 Explicitly copy internal::wordmask to the stack array to work around a compiler optimization with LLVM that converts const stack arrays to global arrays.  This is a temporary change and should be reverted when https://reviews.llvm.org/D30759 is fixed.
82deffcde Remove benchmarking support for fastlz.
18488d621 Use 64 bit little endian on ppc64le.
7b9532b87 Improve the SSE2 macro check on Windows.
7dadceea5 Check for the existence of sys/uio.h in autoconf build.
83179dd8b Remove quicklz and lzf support in benchmarks.
c8131680d Provide a CMakeLists.txt.
ed3b7b242 Clean up unused function warnings in snappy.
8b60aac4f Remove "using namespace std;" from zippy-stubs-internal.h.
7d7a8ec80 Add Travis CI configuration to snappy and fix the make build.
1cd3ab02e Rename README to README.md. It already in markdown, we might as well let github know so that it renders nicely.
597fa795d Delete UnalignedCopy64 from snappy-stubs since the version in snappy.cc is more robust and possibly faster (assuming the compiler knows how to best copy 8 bytes between locations in memory the fastest way possible - a rather safe bet).
039b3a7ac Add std:: prefix to STL non-type names.
3c706d223 Make UnalignedCopy64 not exhibit undefined behavior when src and dst overlap.
d3c6d20d0 Add compression size reporting hooks.
626e1b9fa Use #ifdef __SSE2__ for the emmintrin.h include, otherwise snappy.cc does not compile with -march=prescott.
2d99bd14d 1.1.4 release.
8bfb028b6 Improve zippy decompression speed.
818b58338 adds std:: to stl types (#061)
27c5d8652 Re-work fast path for handling copies in zippy decompression.
4a7409408 Speed up Zippy decompression in PIE mode by removing the penalty for global array access.
38a5ec5fc Re-work fast path that emits copies in zippy compression.
094c67de8 Speed up the EmitLiteral fast path, +1.62% for ZFlat benchmarks.
fce661fa8 Speed up zippy decompression by removing some zero-extensions.
e788e527d Avoid calling memset when resizing the buffer.
32d6d7d8a Merge pull request #6 from deviance/provide-pkg-config-data
971613510 Add #ifdef to guard against macro redefinition if this is included in another Google project that also defines this.
0000f997d Merge pull request #13 from huachaohuang/patch-1
d53de1879 Make heuristic match skipping more aggressive.
2b9152d9c Default to glibtoolize instead of libtoolize if it exists, and also make it customizable through the environment variable $LIBTOOLIZE.
0800b1e4c Work around an issue where some compilers interpret <:: as a trigraph. Also correct the namespace name.
e7d2818d1 Unbreak the open-source build for ARM due to missing ATTRIBUTE_PACKED declaration.
7525a1600 Fix an issue where the ByteSource path (used for parsing std::string) would incorrectly accept some invalid varints that the other path would not, causing potential CHECK-failures if the unit test were run with --write_uncompressed and a corrupted input file.
ef5598aa0 Make UNALIGNED_LOAD16/32 on ARMv7 go through an explicitly unaligned struct, to avoid the compiler coalescing multiple loads into a single load instruction (which only work for aligned accesses).
b8cd908a8 Allow to compile in nested packages.
96a2e340f Update URLs in the Snappy README to reflect the move to GitHub.
0852af760 Move the logic from ComputeTable into the unit test, which means it's run automatically together with the other tests, and also removes the stray function ComputeTable() (which was never referenced by anything else in the open-source version, causing compiler warnings for some) out of the core library.
d80342922 Fix signed-vs.-unsigned comparison warnings.
d2cb73b6a Provide pkg-config data
efb39e81b Release Snappy 1.1.3; getting the new Uncompress variant in a release is nice, and it's also good to finally get an official release out after the migration to GitHub.
eb66d8176 Initialized members of SnappyArrayWriter and SnappyDecompressionValidator. These members were almost surely initialized before use by other member functions, but Coverity was warning about this. Eliminating these warnings minimizes clutter in that report and the likelihood of overlooking a real bug.
b2312c4c2 Add support for Uncompress(source, sink). Various changes to allow Uncompress(source, sink) to get the same performance as the different variants of Uncompress to Cord/DataBuffer/String/FlatBuffer.
b2ad96006 Changes to eliminate compiler warnings on MSVC
e7a897e18 Fixed unit tests to compile under MSVC.
86eb8b152 Change a few branch annotations that profiling found to be wrong. Overall performance is neutral or slightly positive.
11ccdfb86 Sync with various Google-internal changes.
22acaf438 Change some internal path names.

git-subtree-dir: src/snappy/snappy
git-subtree-split: b02bfa754ebf27921d8da3bd2517eab445b84ff9
This commit is contained in:
Brad Chase
2017-12-18 13:41:54 -05:00
parent 8b0602a582
commit 2836c20649
25 changed files with 1594 additions and 2820 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
.appveyor.yml Normal file
View File

@@ -0,0 +1,41 @@
# Build matrix / environment variables are explained on:
# https://www.appveyor.com/docs/appveyor-yml/
# This file can be validated on: https://ci.appveyor.com/tools/validate-yaml
version: "{build}"
environment:
matrix:
# AppVeyor currently has no custom job name feature.
# http://help.appveyor.com/discussions/questions/1623-can-i-provide-a-friendly-name-for-jobs
- JOB: Visual Studio 2017
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
CMAKE_GENERATOR: Visual Studio 15 2017
- JOB: Visual Studio 2015
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
CMAKE_GENERATOR: Visual Studio 14 2015
platform:
- x86
- x64
configuration:
- RelWithDebInfo
- Debug
build:
verbosity: minimal
build_script:
- git submodule update --init --recursive
- mkdir out
- cd out
- if "%platform%"=="x64" set CMAKE_GENERATOR=%CMAKE_GENERATOR% Win64
- cmake --version
- cmake .. -G "%CMAKE_GENERATOR%"
-DCMAKE_CONFIGURATION_TYPES="%CONFIGURATION%"
- cmake --build . --config %CONFIGURATION%
- cd ..
test_script:
- out\%CONFIGURATION%\snappy_unittest

59
.travis.yml Normal file
View File

@@ -0,0 +1,59 @@
# Build matrix / environment variables are explained on:
# http://about.travis-ci.org/docs/user/build-configuration/
# This file can be validated on: http://lint.travis-ci.org/
sudo: false
dist: trusty
language: cpp
compiler:
- gcc
- clang
os:
- linux
- osx
env:
- BUILD_TYPE=Debug
- BUILD_TYPE=RelWithDebInfo
matrix:
allow_failures:
- compiler: clang
env: BUILD_TYPE=RelWithDebInfo
addons:
apt:
# List of whitelisted in travis packages for ubuntu-trusty can be found here:
# https://github.com/travis-ci/apt-package-whitelist/blob/master/ubuntu-trusty
# List of whitelisted in travis apt-sources:
# https://github.com/travis-ci/apt-source-whitelist/blob/master/ubuntu.json
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty-4.0
packages:
- cmake
- gcc-6
- g++-6
- clang-4.0
install:
# Travis doesn't have a nice way to install homebrew packages yet.
# https://github.com/travis-ci/travis-ci/issues/5377
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew update; fi
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install gcc@6; fi
# /usr/bin/gcc is stuck to old versions by on both Linux and OSX.
- if [ "$CXX" = "g++" ]; then export CXX="g++-6" CC="gcc-6"; fi
- echo ${CC}
- echo ${CXX}
- ${CXX} --version
- cmake --version
before_script:
- mkdir -p build && cd build
- cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE
- cmake --build .
- cd ..
script:
- build/snappy_unittest

174
CMakeLists.txt Normal file
View File

@@ -0,0 +1,174 @@
cmake_minimum_required(VERSION 3.1)
project(Snappy VERSION 1.1.7 LANGUAGES C CXX)
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make
# it prominent in the GUI.
option(BUILD_SHARED_LIBS "Build shared libraries(DLLs)." OFF)
option(SNAPPY_BUILD_TESTS "Build Snappy's own tests." ON)
include(TestBigEndian)
test_big_endian(SNAPPY_IS_BIG_ENDIAN)
include(CheckIncludeFile)
check_include_file("byteswap.h" HAVE_BYTESWAP_H)
check_include_file("stddef.h" HAVE_STDDEF_H)
check_include_file("stdint.h" HAVE_STDINT_H)
check_include_file("sys/endian.h" HAVE_SYS_ENDIAN_H)
check_include_file("sys/mman.h" HAVE_SYS_MMAN_H)
check_include_file("sys/resource.h" HAVE_SYS_RESOURCE_H)
check_include_file("sys/time.h" HAVE_SYS_TIME_H)
check_include_file("sys/uio.h" HAVE_SYS_UIO_H)
check_include_file("unistd.h" HAVE_UNISTD_H)
check_include_file("windows.h" HAVE_WINDOWS_H)
include(CheckLibraryExists)
check_library_exists(z zlibVersion "" HAVE_LIBZ)
check_library_exists(lzo2 lzo1x_1_15_compress "" HAVE_LIBLZO2)
include(CheckCXXSourceCompiles)
check_cxx_source_compiles(
"int main(void) { return __builtin_expect(0, 1); }" HAVE_BUILTIN_EXPECT)
check_cxx_source_compiles(
"int main(void) { return __builtin_ctzll(0); }" HAVE_BUILTIN_CTZ)
include(CheckSymbolExists)
check_symbol_exists("mmap" "sys/mman.h" HAVE_FUNC_MMAP)
check_symbol_exists("sysconf" "unistd.h" HAVE_FUNC_SYSCONF)
find_package(GTest QUIET)
if(GTEST_FOUND)
set(HAVE_GTEST 1)
endif(GTEST_FOUND)
find_package(Gflags QUIET)
if(GFLAGS_FOUND)
set(HAVE_GFLAGS 1)
endif(GFLAGS_FOUND)
configure_file(
"${PROJECT_SOURCE_DIR}/cmake/config.h.in"
"${PROJECT_BINARY_DIR}/config.h"
)
# We don't want to define HAVE_ macros in public headers. Instead, we use
# CMake's variable substitution with 0/1 variables, which will be seen by the
# preprocessor as constants.
set(HAVE_STDINT_H_01 ${HAVE_STDINT_H})
set(HAVE_STDDEF_H_01 ${HAVE_STDDEF_H})
set(HAVE_SYS_UIO_H_01 ${HAVE_SYS_UIO_H})
if(NOT HAVE_STDINT_H_01)
set(HAVE_STDINT_H_01 0)
endif(NOT HAVE_STDINT_H_01)
if(NOT HAVE_STDDEF_H_01)
set(HAVE_STDDEF_H_01 0)
endif(NOT HAVE_STDDEF_H_01)
if(NOT HAVE_SYS_UIO_H_01)
set(HAVE_SYS_UIO_H_01 0)
endif(NOT HAVE_SYS_UIO_H_01)
configure_file(
"${PROJECT_SOURCE_DIR}/snappy-stubs-public.h.in"
"${PROJECT_BINARY_DIR}/snappy-stubs-public.h")
add_library(snappy "")
target_sources(snappy
PRIVATE
"${PROJECT_SOURCE_DIR}/snappy-internal.h"
"${PROJECT_SOURCE_DIR}/snappy-stubs-internal.h"
"${PROJECT_SOURCE_DIR}/snappy-c.cc"
"${PROJECT_SOURCE_DIR}/snappy-sinksource.cc"
"${PROJECT_SOURCE_DIR}/snappy-stubs-internal.cc"
"${PROJECT_SOURCE_DIR}/snappy.cc"
"${PROJECT_BINARY_DIR}/config.h"
# Only CMake 3.3+ supports PUBLIC sources in targets exported by "install".
$<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-c.h>
$<INSTALL_INTERFACE:include/snappy-c.h>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-sinksource.h>
$<INSTALL_INTERFACE:include/snappy-sinksource.h>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy.h>
$<INSTALL_INTERFACE:include/snappy.h>
$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/snappy-stubs-public.h>
$<INSTALL_INTERFACE:include/snappy-stubs-public.h>
)
target_include_directories(snappy
PUBLIC
$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>
$<INSTALL_INTERFACE:include>
)
set_target_properties(snappy
PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
target_compile_definitions(snappy PRIVATE -DHAVE_CONFIG_H)
if(BUILD_SHARED_LIBS)
set_target_properties(snappy PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
endif(BUILD_SHARED_LIBS)
if(SNAPPY_BUILD_TESTS)
enable_testing()
add_executable(snappy_unittest "")
target_sources(snappy_unittest
PRIVATE
"${PROJECT_SOURCE_DIR}/snappy_unittest.cc"
"${PROJECT_SOURCE_DIR}/snappy-test.cc"
)
target_compile_definitions(snappy_unittest PRIVATE -DHAVE_CONFIG_H)
target_link_libraries(snappy_unittest snappy ${GFLAGS_LIBRARIES})
if(HAVE_LIBZ)
target_link_libraries(snappy_unittest z)
endif(HAVE_LIBZ)
if(HAVE_LIBLZO2)
target_link_libraries(snappy_unittest lzo2)
endif(HAVE_LIBLZO2)
target_include_directories(snappy_unittest
BEFORE PRIVATE
"${PROJECT_SOURCE_DIR}"
"${GTEST_INCLUDE_DIRS}"
"${GFLAGS_INCLUDE_DIRS}"
)
add_test(
NAME snappy_unittest
WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
COMMAND "${PROJECT_BINARY_DIR}/snappy_unittest")
endif(SNAPPY_BUILD_TESTS)
include(GNUInstallDirs)
install(TARGETS snappy
EXPORT SnappyTargets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
install(
FILES
"${PROJECT_SOURCE_DIR}/snappy-c.h"
"${PROJECT_SOURCE_DIR}/snappy-sinksource.h"
"${PROJECT_SOURCE_DIR}/snappy.h"
"${PROJECT_BINARY_DIR}/snappy-stubs-public.h"
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
)
include(CMakePackageConfigHelpers)
write_basic_package_version_file(
"${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake"
COMPATIBILITY SameMajorVersion
)
install(
EXPORT SnappyTargets
NAMESPACE Snappy::
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy"
)
install(
FILES
"${PROJECT_SOURCE_DIR}/cmake/SnappyConfig.cmake"
"${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy"
)

26
CONTRIBUTING.md Normal file
View File

@@ -0,0 +1,26 @@
# How to Contribute
We'd love to accept your patches and contributions to this project. There are
just a few small guidelines you need to follow.
## Contributor License Agreement
Contributions to this project must be accompanied by a Contributor License
Agreement. You (or your employer) retain the copyright to your contribution,
this simply gives us permission to use and redistribute your contributions as
part of the project. Head over to <https://cla.developers.google.com/> to see
your current agreements on file or to sign a new one.
You generally only need to submit a CLA once, so if you've already submitted one
(even if it was for a different project), you probably don't need to do it
again.
## Code reviews
All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult
[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
information on using pull requests.
Please make sure that all the automated checks (CLA, AppVeyor, Travis) pass for
your pull requests. Pull requests whose checks fail may be ignored.

2
COPYING
View File

@@ -29,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
===
Some of the benchmark data in util/zippy/testdata is licensed differently:
Some of the benchmark data in testdata/ is licensed differently:
- fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and
is licensed under the Creative Commons Attribution 3.0 license

1916
ChangeLog
View File

File diff suppressed because it is too large Load Diff

23
Makefile.am
View File

@@ -1,23 +0,0 @@
ACLOCAL_AMFLAGS = -I m4
# Library.
lib_LTLIBRARIES = libsnappy.la
libsnappy_la_SOURCES = snappy.cc snappy-sinksource.cc snappy-stubs-internal.cc snappy-c.cc
libsnappy_la_LDFLAGS = -version-info $(SNAPPY_LTVERSION)
include_HEADERS = snappy.h snappy-sinksource.h snappy-stubs-public.h snappy-c.h
noinst_HEADERS = snappy-internal.h snappy-stubs-internal.h snappy-test.h
# Unit tests and benchmarks.
snappy_unittest_CPPFLAGS = $(gflags_CFLAGS) $(GTEST_CPPFLAGS)
snappy_unittest_SOURCES = snappy_unittest.cc snappy-test.cc
snappy_unittest_LDFLAGS = $(GTEST_LDFLAGS)
snappy_unittest_LDADD = libsnappy.la $(UNITTEST_LIBS) $(gflags_LIBS) $(GTEST_LIBS)
TESTS = snappy_unittest
noinst_PROGRAMS = $(TESTS)
EXTRA_DIST = autogen.sh testdata/alice29.txt testdata/asyoulik.txt testdata/baddata1.snappy testdata/baddata2.snappy testdata/baddata3.snappy testdata/geo.protodata testdata/fireworks.jpeg testdata/html testdata/html_x_4 testdata/kppkn.gtb testdata/lcet10.txt testdata/paper-100k.pdf testdata/plrabn12.txt testdata/urls.10K
dist_doc_DATA = ChangeLog COPYING INSTALL NEWS README format_description.txt framing_format.txt
libtool: $(LIBTOOL_DEPS)
$(SHELL) ./config.status --recheck

52
NEWS
View File

@@ -1,3 +1,55 @@
Snappy v1.1.7, August 24th 2017:
* Improved CMake build support for 64-bit Linux distributions.
* MSVC builds now use MSVC-specific intrinsics that map to clzll.
* ARM64 (AArch64) builds use the code paths optimized for 64-bit processors.
Snappy v1.1.6, July 12th 2017:
This is a re-release of v1.1.5 with proper SONAME / SOVERSION values.
Snappy v1.1.5, June 28th 2017:
This release has broken SONAME / SOVERSION values. Users of snappy as a shared
library should avoid 1.1.5 and use 1.1.6 instead. SONAME / SOVERSION errors will
manifest as the dynamic library loader complaining that it cannot find snappy's
shared library file (libsnappy.so / libsnappy.dylib), or that the library it
found does not have the required version. 1.1.6 has the same code as 1.1.5, but
carries build configuration fixes for the issues above.
* Add CMake build support. The autoconf build support is now deprecated, and
will be removed in the next release.
* Add AppVeyor configuration, for Windows CI coverage.
* Small performance improvement on little-endian PowerPC.
* Small performance improvement on LLVM with position-independent executables.
* Fix a few issues with various build environments.
Snappy v1.1.4, January 25th 2017:
* Fix a 1% performance regression when snappy is used in PIE executables.
* Improve compression performance by 5%.
* Improve decompression performance by 20%.
Snappy v1.1.3, July 6th 2015:
This is the first release to be done from GitHub, which means that
some minor things like the ChangeLog format has changed (git log
format instead of svn log).
* Add support for Uncompress() from a Source to a Sink.
* Various minor changes to improve MSVC support; in particular,
the unit tests now compile and run under MSVC.
Snappy v1.1.2, February 28th 2014:
This is a maintenance release with no changes to the actual library

28
README → README.md
View File

@@ -34,7 +34,7 @@ Snappy is intended to be fast. On a single core of a Core i7 processor
in 64-bit mode, it compresses at about 250 MB/sec or more and decompresses at
about 500 MB/sec or more. (These numbers are for the slowest inputs in our
benchmark suite; others are much faster.) In our tests, Snappy usually
is faster than algorithms in the same class (e.g. LZO, LZF, FastLZ, QuickLZ,
is faster than algorithms in the same class (e.g. LZO, LZF, QuickLZ,
etc.) while achieving comparable compression ratios.
Typical compression ratios (based on the benchmark suite) are about 1.5-1.7x
@@ -62,12 +62,22 @@ Performance optimizations, whether for 64-bit x86 or other platforms,
are of course most welcome; see "Contact", below.
Building
========
CMake is supported and autotools will soon be deprecated.
You need CMake 3.4 or above to build:
mkdir build
cd build && cmake ../ && make
Usage
=====
Note that Snappy, both the implementation and the main interface,
is written in C++. However, several third-party bindings to other languages
are available; see the Google Code page at http://code.google.com/p/snappy/
are available; see the home page at http://google.github.io/snappy/
for more information. Also, if you want to use Snappy from C code, you can
use the included C bindings in snappy-c.h.
@@ -102,12 +112,12 @@ tests to verify you have not broken anything. Note that if you have the
Google Test library installed, unit test behavior (especially failures) will be
significantly more user-friendly. You can find Google Test at
http://code.google.com/p/googletest/
http://github.com/google/googletest
You probably also want the gflags library for handling of command-line flags;
you can find it at
http://code.google.com/p/google-gflags/
http://gflags.github.io/gflags/
In addition to the unit tests, snappy contains microbenchmarks used to
tune compression and decompression performance. These are automatically run
@@ -116,7 +126,7 @@ before the unit tests, but you can disable them using the flag
need to edit the source).
Finally, snappy can benchmark Snappy against a few other compression libraries
(zlib, LZO, LZF, FastLZ and QuickLZ), if they were detected at configure time.
(zlib, LZO, LZF, and QuickLZ), if they were detected at configure time.
To benchmark using a given file, give the compression algorithm you want to test
Snappy against (e.g. --zlib) and then a list of one or more file names on the
command line. The testdata/ directory contains the files used by the
@@ -129,7 +139,11 @@ test.)
Contact
=======
Snappy is distributed through Google Code. For the latest version, a bug tracker,
Snappy is distributed through GitHub. For the latest version, a bug tracker,
and other information, see
http://code.google.com/p/snappy/
http://google.github.io/snappy/
or the repository at
https://github.com/google/snappy

7
autogen.sh
View File

@@ -1,7 +0,0 @@
#! /bin/sh -e
rm -rf autom4te.cache
aclocal -I m4
autoheader
libtoolize --copy
automake --add-missing --copy
autoconf

1
cmake/SnappyConfig.cmake Normal file
View File

@@ -0,0 +1 @@
include("${CMAKE_CURRENT_LIST_DIR}/SnappyTargets.cmake")

62
cmake/config.h.in Normal file
View File

@@ -0,0 +1,62 @@
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
#define THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
/* Define to 1 if the compiler supports __builtin_ctz and friends. */
#cmakedefine HAVE_BUILTIN_CTZ 1
/* Define to 1 if the compiler supports __builtin_expect. */
#cmakedefine HAVE_BUILTIN_EXPECT 1
/* Define to 1 if you have the <byteswap.h> header file. */
#cmakedefine HAVE_BYTESWAP_H 1
/* Define to 1 if you have a definition for mmap() in <sys/mman.h>. */
#cmakedefine HAVE_FUNC_MMAP 1
/* Define to 1 if you have a definition for sysconf() in <unistd.h>. */
#cmakedefine HAVE_FUNC_SYSCONF 1
/* Define to 1 to use the gflags package for command-line parsing. */
#cmakedefine HAVE_GFLAGS 1
/* Define to 1 if you have Google Test. */
#cmakedefine HAVE_GTEST 1
/* Define to 1 if you have the `lzo2' library (-llzo2). */
#cmakedefine HAVE_LIBLZO2 1
/* Define to 1 if you have the `z' library (-lz). */
#cmakedefine HAVE_LIBZ 1
/* Define to 1 if you have the <stddef.h> header file. */
#cmakedefine HAVE_STDDEF_H 1
/* Define to 1 if you have the <stdint.h> header file. */
#cmakedefine HAVE_STDINT_H 1
/* Define to 1 if you have the <sys/endian.h> header file. */
#cmakedefine HAVE_SYS_ENDIAN_H 1
/* Define to 1 if you have the <sys/mman.h> header file. */
#cmakedefine HAVE_SYS_MMAN_H 1
/* Define to 1 if you have the <sys/resource.h> header file. */
#cmakedefine HAVE_SYS_RESOURCE_H 1
/* Define to 1 if you have the <sys/time.h> header file. */
#cmakedefine HAVE_SYS_TIME_H 1
/* Define to 1 if you have the <sys/uio.h> header file. */
#cmakedefine HAVE_SYS_UIO_H 1
/* Define to 1 if you have the <unistd.h> header file. */
#cmakedefine HAVE_UNISTD_H 1
/* Define to 1 if you have the <windows.h> header file. */
#cmakedefine HAVE_WINDOWS_H 1
/* Define to 1 if your processor stores words with the most significant byte
first (like Motorola and SPARC, unlike Intel and VAX). */
#cmakedefine SNAPPY_IS_BIG_ENDIAN 1
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_

133
configure.ac
View File

@@ -1,133 +0,0 @@
m4_define([snappy_major], [1])
m4_define([snappy_minor], [1])
m4_define([snappy_patchlevel], [2])
# Libtool shared library interface versions (current:revision:age)
# Update this value for every release! (A:B:C will map to foo.so.(A-C).C.B)
# http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
m4_define([snappy_ltversion], [3:1:2])
AC_INIT([snappy], [snappy_major.snappy_minor.snappy_patchlevel])
AC_CONFIG_MACRO_DIR([m4])
# These are flags passed to automake (though they look like gcc flags!)
AM_INIT_AUTOMAKE([-Wall])
LT_INIT
AC_SUBST([LIBTOOL_DEPS])
AC_PROG_CXX
AC_LANG([C++])
AC_C_BIGENDIAN
AC_TYPE_SIZE_T
AC_TYPE_SSIZE_T
AC_CHECK_HEADERS([stdint.h stddef.h sys/mman.h sys/resource.h windows.h byteswap.h sys/byteswap.h sys/endian.h sys/time.h])
# Don't use AC_FUNC_MMAP, as it checks for mappings of already-mapped memory,
# which we don't need (and does not exist on Windows).
AC_CHECK_FUNC([mmap])
GTEST_LIB_CHECK([], [true], [true # Ignore; we can live without it.])
AC_ARG_WITH([gflags],
[AS_HELP_STRING(
[--with-gflags],
[use Google Flags package to enhance the unit test @<:@default=check@:>@])],
[],
[with_gflags=check])
if test "x$with_gflags" != "xno"; then
PKG_CHECK_MODULES(
[gflags],
[libgflags],
[AC_DEFINE([HAVE_GFLAGS], [1], [Use the gflags package for command-line parsing.])],
[if test "x$with_gflags" != "xcheck"; then
AC_MSG_FAILURE([--with-gflags was given, but test for gflags failed])
fi])
fi
# See if we have __builtin_expect.
# TODO: Use AC_CACHE.
AC_MSG_CHECKING([if the compiler supports __builtin_expect])
AC_TRY_COMPILE(, [
return __builtin_expect(1, 1) ? 1 : 0
], [
snappy_have_builtin_expect=yes
AC_MSG_RESULT([yes])
], [
snappy_have_builtin_expect=no
AC_MSG_RESULT([no])
])
if test x$snappy_have_builtin_expect = xyes ; then
AC_DEFINE([HAVE_BUILTIN_EXPECT], [1], [Define to 1 if the compiler supports __builtin_expect.])
fi
# See if we have working count-trailing-zeros intrinsics.
# TODO: Use AC_CACHE.
AC_MSG_CHECKING([if the compiler supports __builtin_ctzll])
AC_TRY_COMPILE(, [
return (__builtin_ctzll(0x100000000LL) == 32) ? 1 : 0
], [
snappy_have_builtin_ctz=yes
AC_MSG_RESULT([yes])
], [
snappy_have_builtin_ctz=no
AC_MSG_RESULT([no])
])
if test x$snappy_have_builtin_ctz = xyes ; then
AC_DEFINE([HAVE_BUILTIN_CTZ], [1], [Define to 1 if the compiler supports __builtin_ctz and friends.])
fi
# Other compression libraries; the unit test can use these for comparison
# if they are available. If they are not found, just ignore.
UNITTEST_LIBS=""
AC_DEFUN([CHECK_EXT_COMPRESSION_LIB], [
AH_CHECK_LIB([$1])
AC_CHECK_LIB(
[$1],
[$2],
[
AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_LIB$1))
UNITTEST_LIBS="-l$1 $UNITTEST_LIBS"
],
[true]
)
])
CHECK_EXT_COMPRESSION_LIB([z], [zlibVersion])
CHECK_EXT_COMPRESSION_LIB([lzo2], [lzo1x_1_15_compress])
CHECK_EXT_COMPRESSION_LIB([lzf], [lzf_compress])
CHECK_EXT_COMPRESSION_LIB([fastlz], [fastlz_compress])
CHECK_EXT_COMPRESSION_LIB([quicklz], [qlz_compress])
AC_SUBST([UNITTEST_LIBS])
# These are used by snappy-stubs-public.h.in.
if test "$ac_cv_header_stdint_h" = "yes"; then
AC_SUBST([ac_cv_have_stdint_h], [1])
else
AC_SUBST([ac_cv_have_stdint_h], [0])
fi
if test "$ac_cv_header_stddef_h" = "yes"; then
AC_SUBST([ac_cv_have_stddef_h], [1])
else
AC_SUBST([ac_cv_have_stddef_h], [0])
fi
if test "$ac_cv_header_sys_uio_h" = "yes"; then
AC_SUBST([ac_cv_have_sys_uio_h], [1])
else
AC_SUBST([ac_cv_have_sys_uio_h], [0])
fi
# Export the version to snappy-stubs-public.h.
SNAPPY_MAJOR="snappy_major"
SNAPPY_MINOR="snappy_minor"
SNAPPY_PATCHLEVEL="snappy_patchlevel"
AC_SUBST([SNAPPY_MAJOR])
AC_SUBST([SNAPPY_MINOR])
AC_SUBST([SNAPPY_PATCHLEVEL])
AC_SUBST([SNAPPY_LTVERSION], snappy_ltversion)
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([Makefile snappy-stubs-public.h])
AC_OUTPUT

74
m4/gtest.m4
View File

@@ -1,74 +0,0 @@
dnl GTEST_LIB_CHECK([minimum version [,
dnl action if found [,action if not found]]])
dnl
dnl Check for the presence of the Google Test library, optionally at a minimum
dnl version, and indicate a viable version with the HAVE_GTEST flag. It defines
dnl standard variables for substitution including GTEST_CPPFLAGS,
dnl GTEST_CXXFLAGS, GTEST_LDFLAGS, and GTEST_LIBS. It also defines
dnl GTEST_VERSION as the version of Google Test found. Finally, it provides
dnl optional custom action slots in the event GTEST is found or not.
AC_DEFUN([GTEST_LIB_CHECK],
[
dnl Provide a flag to enable or disable Google Test usage.
AC_ARG_ENABLE([gtest],
[AS_HELP_STRING([--enable-gtest],
[Enable tests using the Google C++ Testing Framework.
(Default is enabled.)])],
[],
[enable_gtest=])
AC_ARG_VAR([GTEST_CONFIG],
[The exact path of Google Test's 'gtest-config' script.])
AC_ARG_VAR([GTEST_CPPFLAGS],
[C-like preprocessor flags for Google Test.])
AC_ARG_VAR([GTEST_CXXFLAGS],
[C++ compile flags for Google Test.])
AC_ARG_VAR([GTEST_LDFLAGS],
[Linker path and option flags for Google Test.])
AC_ARG_VAR([GTEST_LIBS],
[Library linking flags for Google Test.])
AC_ARG_VAR([GTEST_VERSION],
[The version of Google Test available.])
HAVE_GTEST="no"
AS_IF([test "x${enable_gtest}" != "xno"],
[AC_MSG_CHECKING([for 'gtest-config'])
AS_IF([test "x${enable_gtest}" = "xyes"],
[AS_IF([test -x "${enable_gtest}/scripts/gtest-config"],
[GTEST_CONFIG="${enable_gtest}/scripts/gtest-config"],
[GTEST_CONFIG="${enable_gtest}/bin/gtest-config"])
AS_IF([test -x "${GTEST_CONFIG}"], [],
[AC_MSG_RESULT([no])
AC_MSG_ERROR([dnl
Unable to locate either a built or installed Google Test.
The specific location '${enable_gtest}' was provided for a built or installed
Google Test, but no 'gtest-config' script could be found at this location.])
])],
[AC_PATH_PROG([GTEST_CONFIG], [gtest-config])])
AS_IF([test -x "${GTEST_CONFIG}"],
[AC_MSG_RESULT([${GTEST_CONFIG}])
m4_ifval([$1],
[_gtest_min_version="--min-version=$1"
AC_MSG_CHECKING([for Google Test at least version >= $1])],
[_gtest_min_version="--min-version=0"
AC_MSG_CHECKING([for Google Test])])
AS_IF([${GTEST_CONFIG} ${_gtest_min_version}],
[AC_MSG_RESULT([yes])
HAVE_GTEST='yes'],
[AC_MSG_RESULT([no])])],
[AC_MSG_RESULT([no])])
AS_IF([test "x${HAVE_GTEST}" = "xyes"],
[GTEST_CPPFLAGS=`${GTEST_CONFIG} --cppflags`
GTEST_CXXFLAGS=`${GTEST_CONFIG} --cxxflags`
GTEST_LDFLAGS=`${GTEST_CONFIG} --ldflags`
GTEST_LIBS=`${GTEST_CONFIG} --libs`
GTEST_VERSION=`${GTEST_CONFIG} --version`
AC_DEFINE([HAVE_GTEST],[1],[Defined when Google Test is available.])],
[AS_IF([test "x${enable_gtest}" = "xyes"],
[AC_MSG_ERROR([dnl
Google Test was enabled, but no viable version could be found.])
])])])
AC_SUBST([HAVE_GTEST])
AM_CONDITIONAL([HAVE_GTEST],[test "x$HAVE_GTEST" = "xyes"])
AS_IF([test "x$HAVE_GTEST" = "xyes"],
[m4_ifval([$2], [$2])],
[m4_ifval([$3], [$3])])
])

6
snappy-c.h
View File

@@ -30,8 +30,8 @@
* Plain C interface (a wrapper around the C++ implementation).
*/
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_
#ifdef __cplusplus
extern "C" {
@@ -135,4 +135,4 @@ snappy_status snappy_validate_compressed_buffer(const char* compressed,
} // extern "C"
#endif
#endif /* UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_ */
#endif /* THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_ */

128
snappy-internal.h
View File

@@ -28,8 +28,8 @@
//
// Internals shared between the Snappy implementation and its unittest.
#ifndef UTIL_SNAPPY_SNAPPY_INTERNAL_H_
#define UTIL_SNAPPY_SNAPPY_INTERNAL_H_
#ifndef THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_
#define THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_
#include "snappy-stubs-internal.h"
@@ -50,7 +50,9 @@ class WorkingMemory {
uint16 small_table_[1<<10]; // 2KB
uint16* large_table_; // Allocated only when needed
DISALLOW_COPY_AND_ASSIGN(WorkingMemory);
// No copying
WorkingMemory(const WorkingMemory&);
void operator=(const WorkingMemory&);
};
// Flat array compression that does not emit the "uncompressed length"
@@ -70,57 +72,72 @@ char* CompressFragment(const char* input,
uint16* table,
const int table_size);
// Return the largest n such that
// Find the largest n such that
//
// s1[0,n-1] == s2[0,n-1]
// and n <= (s2_limit - s2).
//
// Return make_pair(n, n < 8).
// Does not read *s2_limit or beyond.
// Does not read *(s1 + (s2_limit - s2)) or beyond.
// Requires that s2_limit >= s2.
//
// Separate implementation for x86_64, for speed. Uses the fact that
// x86_64 is little endian.
#if defined(ARCH_K8)
static inline int FindMatchLength(const char* s1,
const char* s2,
const char* s2_limit) {
// Separate implementation for 64-bit, little-endian cpus.
#if !defined(SNAPPY_IS_BIG_ENDIAN) && \
(defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM))
static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
const char* s2,
const char* s2_limit) {
assert(s2_limit >= s2);
int matched = 0;
size_t matched = 0;
// This block isn't necessary for correctness; we could just start looping
// immediately. As an optimization though, it is useful. It creates some not
// uncommon code paths that determine, without extra effort, whether the match
// length is less than 8. In short, we are hoping to avoid a conditional
// branch, and perhaps get better code layout from the C++ compiler.
if (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
uint64 a1 = UNALIGNED_LOAD64(s1);
uint64 a2 = UNALIGNED_LOAD64(s2);
if (a1 != a2) {
return std::pair<size_t, bool>(Bits::FindLSBSetNonZero64(a1 ^ a2) >> 3,
true);
} else {
matched = 8;
s2 += 8;
}
}
// Find out how long the match is. We loop over the data 64 bits at a
// time until we find a 64-bit block that doesn't match; then we find
// the first non-matching bit and use that to calculate the total
// length of the match.
while (PREDICT_TRUE(s2 <= s2_limit - 8)) {
if (PREDICT_FALSE(UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched))) {
while (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
if (UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched)) {
s2 += 8;
matched += 8;
} else {
// On current (mid-2008) Opteron models there is a 3% more
// efficient code sequence to find the first non-matching byte.
// However, what follows is ~10% better on Intel Core 2 and newer,
// and we expect AMD's bsf instruction to improve.
uint64 x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched);
int matching_bits = Bits::FindLSBSetNonZero64(x);
matched += matching_bits >> 3;
return matched;
assert(matched >= 8);
return std::pair<size_t, bool>(matched, false);
}
}
while (PREDICT_TRUE(s2 < s2_limit)) {
if (PREDICT_TRUE(s1[matched] == *s2)) {
while (SNAPPY_PREDICT_TRUE(s2 < s2_limit)) {
if (s1[matched] == *s2) {
++s2;
++matched;
} else {
return matched;
return std::pair<size_t, bool>(matched, matched < 8);
}
}
return matched;
return std::pair<size_t, bool>(matched, matched < 8);
}
#else
static inline int FindMatchLength(const char* s1,
const char* s2,
const char* s2_limit) {
static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
const char* s2,
const char* s2_limit) {
// Implementation based on the x86-64 version, above.
assert(s2_limit >= s2);
int matched = 0;
@@ -140,11 +157,68 @@ static inline int FindMatchLength(const char* s1,
++matched;
}
}
return matched;
return std::pair<size_t, bool>(matched, matched < 8);
}
#endif
// Lookup tables for decompression code. Give --snappy_dump_decompression_table
// to the unit test to recompute char_table.
enum {
LITERAL = 0,
COPY_1_BYTE_OFFSET = 1, // 3 bit length + 3 bits of offset in opcode
COPY_2_BYTE_OFFSET = 2,
COPY_4_BYTE_OFFSET = 3
};
static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset.
// Data stored per entry in lookup table:
// Range Bits-used Description
// ------------------------------------
// 1..64 0..7 Literal/copy length encoded in opcode byte
// 0..7 8..10 Copy offset encoded in opcode byte / 256
// 0..4 11..13 Extra bytes after opcode
//
// We use eight bits for the length even though 7 would have sufficed
// because of efficiency reasons:
// (1) Extracting a byte is faster than a bit-field
// (2) It properly aligns copy offset so we do not need a <<8
static const uint16 char_table[256] = {
0x0001, 0x0804, 0x1001, 0x2001, 0x0002, 0x0805, 0x1002, 0x2002,
0x0003, 0x0806, 0x1003, 0x2003, 0x0004, 0x0807, 0x1004, 0x2004,
0x0005, 0x0808, 0x1005, 0x2005, 0x0006, 0x0809, 0x1006, 0x2006,
0x0007, 0x080a, 0x1007, 0x2007, 0x0008, 0x080b, 0x1008, 0x2008,
0x0009, 0x0904, 0x1009, 0x2009, 0x000a, 0x0905, 0x100a, 0x200a,
0x000b, 0x0906, 0x100b, 0x200b, 0x000c, 0x0907, 0x100c, 0x200c,
0x000d, 0x0908, 0x100d, 0x200d, 0x000e, 0x0909, 0x100e, 0x200e,
0x000f, 0x090a, 0x100f, 0x200f, 0x0010, 0x090b, 0x1010, 0x2010,
0x0011, 0x0a04, 0x1011, 0x2011, 0x0012, 0x0a05, 0x1012, 0x2012,
0x0013, 0x0a06, 0x1013, 0x2013, 0x0014, 0x0a07, 0x1014, 0x2014,
0x0015, 0x0a08, 0x1015, 0x2015, 0x0016, 0x0a09, 0x1016, 0x2016,
0x0017, 0x0a0a, 0x1017, 0x2017, 0x0018, 0x0a0b, 0x1018, 0x2018,
0x0019, 0x0b04, 0x1019, 0x2019, 0x001a, 0x0b05, 0x101a, 0x201a,
0x001b, 0x0b06, 0x101b, 0x201b, 0x001c, 0x0b07, 0x101c, 0x201c,
0x001d, 0x0b08, 0x101d, 0x201d, 0x001e, 0x0b09, 0x101e, 0x201e,
0x001f, 0x0b0a, 0x101f, 0x201f, 0x0020, 0x0b0b, 0x1020, 0x2020,
0x0021, 0x0c04, 0x1021, 0x2021, 0x0022, 0x0c05, 0x1022, 0x2022,
0x0023, 0x0c06, 0x1023, 0x2023, 0x0024, 0x0c07, 0x1024, 0x2024,
0x0025, 0x0c08, 0x1025, 0x2025, 0x0026, 0x0c09, 0x1026, 0x2026,
0x0027, 0x0c0a, 0x1027, 0x2027, 0x0028, 0x0c0b, 0x1028, 0x2028,
0x0029, 0x0d04, 0x1029, 0x2029, 0x002a, 0x0d05, 0x102a, 0x202a,
0x002b, 0x0d06, 0x102b, 0x202b, 0x002c, 0x0d07, 0x102c, 0x202c,
0x002d, 0x0d08, 0x102d, 0x202d, 0x002e, 0x0d09, 0x102e, 0x202e,
0x002f, 0x0d0a, 0x102f, 0x202f, 0x0030, 0x0d0b, 0x1030, 0x2030,
0x0031, 0x0e04, 0x1031, 0x2031, 0x0032, 0x0e05, 0x1032, 0x2032,
0x0033, 0x0e06, 0x1033, 0x2033, 0x0034, 0x0e07, 0x1034, 0x2034,
0x0035, 0x0e08, 0x1035, 0x2035, 0x0036, 0x0e09, 0x1036, 0x2036,
0x0037, 0x0e0a, 0x1037, 0x2037, 0x0038, 0x0e0b, 0x1038, 0x2038,
0x0039, 0x0f04, 0x1039, 0x2039, 0x003a, 0x0f05, 0x103a, 0x203a,
0x003b, 0x0f06, 0x103b, 0x203b, 0x003c, 0x0f07, 0x103c, 0x203c,
0x0801, 0x0f08, 0x103d, 0x203d, 0x1001, 0x0f09, 0x103e, 0x203e,
0x1801, 0x0f0a, 0x103f, 0x203f, 0x2001, 0x0f0b, 0x1040, 0x2040
};
} // end namespace internal
} // end namespace snappy
#endif // UTIL_SNAPPY_SNAPPY_INTERNAL_H_
#endif // THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_

33
snappy-sinksource.cc
View File

@@ -40,6 +40,21 @@ char* Sink::GetAppendBuffer(size_t length, char* scratch) {
return scratch;
}
char* Sink::GetAppendBufferVariable(
size_t min_size, size_t desired_size_hint, char* scratch,
size_t scratch_size, size_t* allocated_size) {
*allocated_size = scratch_size;
return scratch;
}
void Sink::AppendAndTakeOwnership(
char* bytes, size_t n,
void (*deleter)(void*, const char*, size_t),
void *deleter_arg) {
Append(bytes, n);
(*deleter)(deleter_arg, bytes, n);
}
ByteArraySource::~ByteArraySource() { }
size_t ByteArraySource::Available() const { return left_; }
@@ -68,4 +83,22 @@ char* UncheckedByteArraySink::GetAppendBuffer(size_t len, char* scratch) {
return dest_;
}
void UncheckedByteArraySink::AppendAndTakeOwnership(
char* data, size_t n,
void (*deleter)(void*, const char*, size_t),
void *deleter_arg) {
if (data != dest_) {
memcpy(dest_, data, n);
(*deleter)(deleter_arg, data, n);
}
dest_ += n;
}
char* UncheckedByteArraySink::GetAppendBufferVariable(
size_t min_size, size_t desired_size_hint, char* scratch,
size_t scratch_size, size_t* allocated_size) {
*allocated_size = desired_size_hint;
return dest_;
}
} // namespace snappy

57
snappy-sinksource.h
View File

@@ -26,12 +26,11 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_
#define UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_
#ifndef THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
#define THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
#include <stddef.h>
namespace snappy {
// A Sink is an interface that consumes a sequence of bytes.
@@ -60,6 +59,47 @@ class Sink {
// The default implementation always returns the scratch buffer.
virtual char* GetAppendBuffer(size_t length, char* scratch);
// For higher performance, Sink implementations can provide custom
// AppendAndTakeOwnership() and GetAppendBufferVariable() methods.
// These methods can reduce the number of copies done during
// compression/decompression.
// Append "bytes[0,n-1] to the sink. Takes ownership of "bytes"
// and calls the deleter function as (*deleter)(deleter_arg, bytes, n)
// to free the buffer. deleter function must be non NULL.
//
// The default implementation just calls Append and frees "bytes".
// Other implementations may avoid a copy while appending the buffer.
virtual void AppendAndTakeOwnership(
char* bytes, size_t n, void (*deleter)(void*, const char*, size_t),
void *deleter_arg);
// Returns a writable buffer for appending and writes the buffer's capacity to
// *allocated_size. Guarantees *allocated_size >= min_size.
// May return a pointer to the caller-owned scratch buffer which must have
// scratch_size >= min_size.
//
// The returned buffer is only valid until the next operation
// on this ByteSink.
//
// After writing at most *allocated_size bytes, call Append() with the
// pointer returned from this function and the number of bytes written.
// Many Append() implementations will avoid copying bytes if this function
// returned an internal buffer.
//
// If the sink implementation allocates or reallocates an internal buffer,
// it should use the desired_size_hint if appropriate. If a caller cannot
// provide a reasonable guess at the desired capacity, it should set
// desired_size_hint = 0.
//
// If a non-scratch buffer is returned, the caller may only pass
// a prefix to it to Append(). That is, it is not correct to pass an
// interior pointer to Append().
//
// The default implementation always returns the scratch buffer.
virtual char* GetAppendBufferVariable(
size_t min_size, size_t desired_size_hint, char* scratch,
size_t scratch_size, size_t* allocated_size);
private:
// No copying
@@ -122,6 +162,12 @@ class UncheckedByteArraySink : public Sink {
virtual ~UncheckedByteArraySink();
virtual void Append(const char* data, size_t n);
virtual char* GetAppendBuffer(size_t len, char* scratch);
virtual char* GetAppendBufferVariable(
size_t min_size, size_t desired_size_hint, char* scratch,
size_t scratch_size, size_t* allocated_size);
virtual void AppendAndTakeOwnership(
char* bytes, size_t n, void (*deleter)(void*, const char*, size_t),
void *deleter_arg);
// Return the current output pointer so that a caller can see how
// many bytes were produced.
@@ -131,7 +177,6 @@ class UncheckedByteArraySink : public Sink {
char* dest_;
};
} // namespace snappy
}
#endif // UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_
#endif // THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_

144
snappy-stubs-internal.h
View File

@@ -28,8 +28,8 @@
//
// Various stubs for the open-source version of Snappy.
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
#ifdef HAVE_CONFIG_H
#include "config.h"
@@ -45,6 +45,14 @@
#include <sys/mman.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#if defined(_MSC_VER)
#include <intrin.h>
#endif // defined(_MSC_VER)
#include "snappy-stubs-public.h"
#if defined(__x86_64__)
@@ -52,6 +60,14 @@
// Enable 64-bit optimized versions of some routines.
#define ARCH_K8 1
#elif defined(__ppc64__)
#define ARCH_PPC 1
#elif defined(__aarch64__)
#define ARCH_ARM 1
#endif
// Needed by OS X, among others.
@@ -59,10 +75,6 @@
#define MAP_ANONYMOUS MAP_ANON
#endif
// Pull in std::min, std::ostream, and the likes. This is safe because this
// header file is never used from any public header files.
using namespace std;
// The size of an array, if known at compile-time.
// Will give unexpected results if used on a pointer.
// We undefine it first, since some compilers already have a definition.
@@ -73,11 +85,11 @@ using namespace std;
// Static prediction hints.
#ifdef HAVE_BUILTIN_EXPECT
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
#define SNAPPY_PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define SNAPPY_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
#else
#define PREDICT_FALSE(x) x
#define PREDICT_TRUE(x) x
#define SNAPPY_PREDICT_FALSE(x) x
#define SNAPPY_PREDICT_TRUE(x) x
#endif
// This is only used for recomputing the tag byte table used during
@@ -96,9 +108,10 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
// Potentially unaligned loads and stores.
// x86 and PowerPC can simply do these loads and stores native.
// x86, PowerPC, and ARM64 can simply do these loads and stores native.
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__)
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || \
defined(__aarch64__)
#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p))
#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p))
@@ -116,6 +129,15 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
// sub-architectures.
//
// This is a mess, but there's not much we can do about it.
//
// To further complicate matters, only LDR instructions (single reads) are
// allowed to be unaligned, not LDRD (two reads) or LDM (many reads). Unless we
// explicitly tell the compiler that these accesses can be unaligned, it can and
// will combine accesses. On armcc, the way to signal this is done by accessing
// through the type (uint32 __packed *), but GCC has no such attribute
// (it ignores __attribute__((packed)) on individual variables). However,
// we can tell it that a _struct_ is unaligned, which has the same effect,
// so we do that.
#elif defined(__arm__) && \
!defined(__ARM_ARCH_4__) && \
@@ -131,11 +153,39 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
!defined(__ARM_ARCH_6ZK__) && \
!defined(__ARM_ARCH_6T2__)
#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p))
#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p))
#if __GNUC__
#define ATTRIBUTE_PACKED __attribute__((__packed__))
#else
#define ATTRIBUTE_PACKED
#endif
#define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast<uint16 *>(_p) = (_val))
#define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast<uint32 *>(_p) = (_val))
namespace base {
namespace internal {
struct Unaligned16Struct {
uint16 value;
uint8 dummy; // To make the size non-power-of-two.
} ATTRIBUTE_PACKED;
struct Unaligned32Struct {
uint32 value;
uint8 dummy; // To make the size non-power-of-two.
} ATTRIBUTE_PACKED;
} // namespace internal
} // namespace base
#define UNALIGNED_LOAD16(_p) \
((reinterpret_cast<const ::snappy::base::internal::Unaligned16Struct *>(_p))->value)
#define UNALIGNED_LOAD32(_p) \
((reinterpret_cast<const ::snappy::base::internal::Unaligned32Struct *>(_p))->value)
#define UNALIGNED_STORE16(_p, _val) \
((reinterpret_cast< ::snappy::base::internal::Unaligned16Struct *>(_p))->value = \
(_val))
#define UNALIGNED_STORE32(_p, _val) \
((reinterpret_cast< ::snappy::base::internal::Unaligned32Struct *>(_p))->value = \
(_val))
// TODO(user): NEON supports unaligned 64-bit loads and stores.
// See if that would be more efficient on platforms supporting it,
@@ -188,22 +238,8 @@ inline void UNALIGNED_STORE64(void *p, uint64 v) {
#endif
// This can be more efficient than UNALIGNED_LOAD64 + UNALIGNED_STORE64
// on some platforms, in particular ARM.
inline void UnalignedCopy64(const void *src, void *dst) {
if (sizeof(void *) == 8) {
UNALIGNED_STORE64(dst, UNALIGNED_LOAD64(src));
} else {
const char *src_char = reinterpret_cast<const char *>(src);
char *dst_char = reinterpret_cast<char *>(dst);
UNALIGNED_STORE32(dst_char, UNALIGNED_LOAD32(src_char));
UNALIGNED_STORE32(dst_char + 4, UNALIGNED_LOAD32(src_char + 4));
}
}
// The following guarantees declaration of the byte swap functions.
#ifdef WORDS_BIGENDIAN
#if defined(SNAPPY_IS_BIG_ENDIAN)
#ifdef HAVE_SYS_BYTEORDER_H
#include <sys/byteorder.h>
@@ -260,7 +296,7 @@ inline uint64 bswap_64(uint64 x) {
#endif
#endif // WORDS_BIGENDIAN
#endif // defined(SNAPPY_IS_BIG_ENDIAN)
// Convert to little-endian storage, opposite of network format.
// Convert x from host to little endian: x = LittleEndian.FromHost(x);
@@ -274,7 +310,7 @@ inline uint64 bswap_64(uint64 x) {
class LittleEndian {
public:
// Conversion functions.
#ifdef WORDS_BIGENDIAN
#if defined(SNAPPY_IS_BIG_ENDIAN)
static uint16 FromHost16(uint16 x) { return bswap_16(x); }
static uint16 ToHost16(uint16 x) { return bswap_16(x); }
@@ -284,7 +320,7 @@ class LittleEndian {
static bool IsLittleEndian() { return false; }
#else // !defined(WORDS_BIGENDIAN)
#else // !defined(SNAPPY_IS_BIG_ENDIAN)
static uint16 FromHost16(uint16 x) { return x; }
static uint16 ToHost16(uint16 x) { return x; }
@@ -294,7 +330,7 @@ class LittleEndian {
static bool IsLittleEndian() { return true; }
#endif // !defined(WORDS_BIGENDIAN)
#endif // !defined(SNAPPY_IS_BIG_ENDIAN)
// Functions to do unaligned loads and stores in little-endian order.
static uint16 Load16(const void *p) {
@@ -324,10 +360,15 @@ class Bits {
// undefined value if n == 0. FindLSBSetNonZero() is similar to ffs() except
// that it's 0-indexed.
static int FindLSBSetNonZero(uint32 n);
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
static int FindLSBSetNonZero64(uint64 n);
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
private:
DISALLOW_COPY_AND_ASSIGN(Bits);
// No copying
Bits(const Bits&);
void operator=(const Bits&);
};
#ifdef HAVE_BUILTIN_CTZ
@@ -340,9 +381,36 @@ inline int Bits::FindLSBSetNonZero(uint32 n) {
return __builtin_ctz(n);
}
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
inline int Bits::FindLSBSetNonZero64(uint64 n) {
return __builtin_ctzll(n);
}
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
#elif defined(_MSC_VER)
inline int Bits::Log2Floor(uint32 n) {
unsigned long where;
if (_BitScanReverse(&where, n)) {
return where;
} else {
return -1;
}
}
inline int Bits::FindLSBSetNonZero(uint32 n) {
unsigned long where;
if (_BitScanForward(&where, n)) return static_cast<int>(where);
return 32;
}
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
inline int Bits::FindLSBSetNonZero64(uint64 n) {
unsigned long where;
if (_BitScanForward64(&where, n)) return static_cast<int>(where);
return 64;
}
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
#else // Portable versions.
@@ -376,6 +444,7 @@ inline int Bits::FindLSBSetNonZero(uint32 n) {
return rc;
}
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
// FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero().
inline int Bits::FindLSBSetNonZero64(uint64 n) {
const uint32 bottombits = static_cast<uint32>(n);
@@ -386,6 +455,7 @@ inline int Bits::FindLSBSetNonZero64(uint64 n) {
return FindLSBSetNonZero(bottombits);
}
}
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
#endif // End portable versions.
@@ -488,4 +558,4 @@ inline char* string_as_array(string* str) {
} // namespace snappy
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_

36
snappy-stubs-public.h.in
View File

@@ -33,24 +33,24 @@
// which is a public header. Instead, snappy-stubs-public.h is generated by
// from snappy-stubs-public.h.in at configure time.
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
#if @ac_cv_have_stdint_h@
#if ${HAVE_STDINT_H_01} // HAVE_STDINT_H
#include <stdint.h>
#endif
#endif // HAVE_STDDEF_H
#if @ac_cv_have_stddef_h@
#if ${HAVE_STDDEF_H_01} // HAVE_STDDEF_H
#include <stddef.h>
#endif
#endif // HAVE_STDDEF_H
#if @ac_cv_have_sys_uio_h@
#if ${HAVE_SYS_UIO_H_01} // HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#endif // HAVE_SYS_UIO_H
#define SNAPPY_MAJOR @SNAPPY_MAJOR@
#define SNAPPY_MINOR @SNAPPY_MINOR@
#define SNAPPY_PATCHLEVEL @SNAPPY_PATCHLEVEL@
#define SNAPPY_MAJOR ${SNAPPY_MAJOR}
#define SNAPPY_MINOR ${SNAPPY_MINOR}
#define SNAPPY_PATCHLEVEL ${SNAPPY_PATCHLEVEL}
#define SNAPPY_VERSION \
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
@@ -58,7 +58,7 @@
namespace snappy {
#if @ac_cv_have_stdint_h@
#if ${HAVE_STDINT_H_01} // HAVE_STDINT_H
typedef int8_t int8;
typedef uint8_t uint8;
typedef int16_t int16;
@@ -76,23 +76,19 @@ typedef int int32;
typedef unsigned int uint32;
typedef long long int64;
typedef unsigned long long uint64;
#endif
#endif // HAVE_STDINT_H
typedef std::string string;
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
TypeName(const TypeName&); \
void operator=(const TypeName&)
#if !@ac_cv_have_sys_uio_h@
#if !${HAVE_SYS_UIO_H_01} // !HAVE_SYS_UIO_H
// Windows does not have an iovec type, yet the concept is universally useful.
// It is simple to define it ourselves, so we put it inside our own namespace.
struct iovec {
void* iov_base;
size_t iov_len;
};
#endif
#endif // !HAVE_SYS_UIO_H
} // namespace snappy
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_

24
snappy-test.cc
View File

@@ -28,13 +28,19 @@
//
// Various stubs for the unit tests for the open-source version of Snappy.
#include "snappy-test.h"
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#ifdef HAVE_WINDOWS_H
#define WIN32_LEAN_AND_MEAN
// Needed to be able to use std::max without workarounds in the source code.
// https://support.microsoft.com/en-us/help/143208/prb-using-stl-in-windows-program-can-cause-min-max-conflicts
#define NOMINMAX
#include <windows.h>
#endif
#include "snappy-test.h"
#include <algorithm>
DEFINE_bool(run_microbenchmarks, true,
@@ -198,7 +204,7 @@ void Benchmark::Run() {
if (benchmark_real_time_us > 0) {
num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us;
}
num_iterations = max(num_iterations, kCalibrateIterations);
num_iterations = std::max(num_iterations, kCalibrateIterations);
BenchmarkRun benchmark_runs[kNumRuns];
for (int run = 0; run < kNumRuns; ++run) {
@@ -214,10 +220,10 @@ void Benchmark::Run() {
string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num);
string human_readable_speed;
nth_element(benchmark_runs,
benchmark_runs + kMedianPos,
benchmark_runs + kNumRuns,
BenchmarkCompareCPUTime());
std::nth_element(benchmark_runs,
benchmark_runs + kMedianPos,
benchmark_runs + kNumRuns,
BenchmarkCompareCPUTime());
int64 real_time_us = benchmark_runs[kMedianPos].real_time_us;
int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us;
if (cpu_time_us <= 0) {
@@ -520,8 +526,8 @@ int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen,
LOG(WARNING)
<< "UncompressChunkOrAll: Received some extra data, bytes total: "
<< uncomp_stream_.avail_in << " bytes: "
<< string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
min(int(uncomp_stream_.avail_in), 20));
<< std::string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
std::min(int(uncomp_stream_.avail_in), 20));
UncompressErrorInit();
return Z_DATA_ERROR; // what's the extra data for?
} else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {

77
snappy-test.h
View File

@@ -28,8 +28,8 @@
//
// Various stubs for the unit tests for the open-source version of Snappy.
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
#include <iostream>
#include <string>
@@ -52,7 +52,6 @@
#endif
#ifdef HAVE_WINDOWS_H
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#endif
@@ -111,35 +110,18 @@
#include "lzo/lzo1x.h"
#endif
#ifdef HAVE_LIBLZF
extern "C" {
#include "lzf.h"
}
#endif
#ifdef HAVE_LIBFASTLZ
#include "fastlz.h"
#endif
#ifdef HAVE_LIBQUICKLZ
#include "quicklz.h"
#endif
namespace {
namespace File {
void Init() { }
} // namespace File
namespace file {
int Defaults() { }
int Defaults() { return 0; }
class DummyStatus {
public:
void CheckSuccess() { }
};
DummyStatus GetContents(const string& filename, string* data, int unused) {
DummyStatus GetContents(
const std::string& filename, std::string* data, int unused) {
FILE* fp = fopen(filename.c_str(), "rb");
if (fp == NULL) {
perror(filename.c_str());
@@ -154,15 +136,16 @@ namespace file {
perror("fread");
exit(1);
}
data->append(string(buf, ret));
data->append(std::string(buf, ret));
}
fclose(fp);
return DummyStatus();
}
DummyStatus SetContents(const string& filename,
const string& str,
int unused) {
inline DummyStatus SetContents(
const std::string& filename, const std::string& str, int unused) {
FILE* fp = fopen(filename.c_str(), "wb");
if (fp == NULL) {
perror(filename.c_str());
@@ -176,6 +159,8 @@ namespace file {
}
fclose(fp);
return DummyStatus();
}
} // namespace file
@@ -193,6 +178,7 @@ void Test_Snappy_RandomData();
void Test_Snappy_FourByteOffset();
void Test_SnappyCorruption_TruncatedVarint();
void Test_SnappyCorruption_UnterminatedVarint();
void Test_SnappyCorruption_OverflowingVarint();
void Test_Snappy_ReadPastEndOfBuffer();
void Test_Snappy_FindMatchLength();
void Test_Snappy_FindMatchLengthRandom();
@@ -463,7 +449,7 @@ class ZLib {
DECLARE_bool(run_microbenchmarks);
static void RunSpecifiedBenchmarks() {
static inline void RunSpecifiedBenchmarks() {
if (!FLAGS_run_microbenchmarks) {
return;
}
@@ -497,6 +483,7 @@ static inline int RUN_ALL_TESTS() {
snappy::Test_Snappy_FourByteOffset();
snappy::Test_SnappyCorruption_TruncatedVarint();
snappy::Test_SnappyCorruption_UnterminatedVarint();
snappy::Test_SnappyCorruption_OverflowingVarint();
snappy::Test_Snappy_ReadPastEndOfBuffer();
snappy::Test_Snappy_FindMatchLength();
snappy::Test_Snappy_FindMatchLengthRandom();
@@ -510,10 +497,6 @@ static inline int RUN_ALL_TESTS() {
// For main().
namespace snappy {
static void CompressFile(const char* fname);
static void UncompressFile(const char* fname);
static void MeasureFile(const char* fname);
// Logging.
#define LOG(level) LogMessage()
@@ -524,15 +507,15 @@ class LogMessage {
public:
LogMessage() { }
~LogMessage() {
cerr << endl;
std::cerr << std::endl;
}
LogMessage& operator<<(const std::string& msg) {
cerr << msg;
std::cerr << msg;
return *this;
}
LogMessage& operator<<(int x) {
cerr << x;
std::cerr << x;
return *this;
}
};
@@ -541,18 +524,29 @@ class LogMessage {
// and ones that are always active.
#define CRASH_UNLESS(condition) \
PREDICT_TRUE(condition) ? (void)0 : \
SNAPPY_PREDICT_TRUE(condition) ? (void)0 : \
snappy::LogMessageVoidify() & snappy::LogMessageCrash()
#ifdef _MSC_VER
// ~LogMessageCrash calls abort() and therefore never exits. This is by design
// so temporarily disable warning C4722.
#pragma warning(push)
#pragma warning(disable:4722)
#endif
class LogMessageCrash : public LogMessage {
public:
LogMessageCrash() { }
~LogMessageCrash() {
cerr << endl;
std::cerr << std::endl;
abort();
}
};
#ifdef _MSC_VER
#pragma warning(pop)
#endif
// This class is used to explicitly ignore values in the conditional
// logging macros. This avoids compiler warnings like "value computed
// is not used" and "statement has no effect".
@@ -572,11 +566,8 @@ class LogMessageVoidify {
#define CHECK_NE(a, b) CRASH_UNLESS((a) != (b))
#define CHECK_LT(a, b) CRASH_UNLESS((a) < (b))
#define CHECK_GT(a, b) CRASH_UNLESS((a) > (b))
#define CHECK_OK(cond) (cond).CheckSuccess()
} // namespace
} // namespace snappy
using snappy::CompressFile;
using snappy::UncompressFile;
using snappy::MeasureFile;
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_

855
snappy.cc
View File

File diff suppressed because it is too large Load Diff

27
snappy.h
View File

@@ -36,8 +36,8 @@
// using BMDiff and then compressing the output of BMDiff with
// Snappy.
#ifndef UTIL_SNAPPY_SNAPPY_H__
#define UTIL_SNAPPY_SNAPPY_H__
#ifndef THIRD_PARTY_SNAPPY_SNAPPY_H__
#define THIRD_PARTY_SNAPPY_SNAPPY_H__
#include <stddef.h>
#include <string>
@@ -84,6 +84,18 @@ namespace snappy {
bool Uncompress(const char* compressed, size_t compressed_length,
string* uncompressed);
// Decompresses "compressed" to "*uncompressed".
//
// returns false if the message is corrupted and could not be decompressed
bool Uncompress(Source* compressed, Sink* uncompressed);
// This routine uncompresses as much of the "compressed" as possible
// into sink. It returns the number of valid bytes added to sink
// (extra invalid bytes may have been added due to errors; the caller
// should ignore those). The emitted data typically has length
// GetUncompressedLength(), but may be shorter if an error is
// encountered.
size_t UncompressAsMuchAsPossible(Source* compressed, Sink* uncompressed);
// ------------------------------------------------------------------------
// Lower-level character array based routines. May be useful for
@@ -164,6 +176,14 @@ namespace snappy {
bool IsValidCompressedBuffer(const char* compressed,
size_t compressed_length);
// Returns true iff the contents of "compressed" can be uncompressed
// successfully. Does not return the uncompressed data. Takes
// time proportional to *compressed length, but is usually at least
// a factor of four faster than actual decompression.
// On success, consumes all of *compressed. On failure, consumes an
// unspecified prefix of *compressed.
bool IsValidCompressed(Source* compressed);
// The size of a compression block. Note that many parts of the compression
// code assumes that kBlockSize <= 65536; in particular, the hash table
// can only store 16-bit offsets, and EmitCopy() also assumes the offset
@@ -180,5 +200,4 @@ namespace snappy {
static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
} // end namespace snappy
#endif // UTIL_SNAPPY_SNAPPY_H__
#endif // THIRD_PARTY_SNAPPY_SNAPPY_H__

425
snappy_unittest.cc
View File

@@ -32,6 +32,7 @@
#include <algorithm>
#include <string>
#include <utility>
#include <vector>
#include "snappy.h"
@@ -50,25 +51,19 @@ DEFINE_bool(zlib, false,
"Run zlib compression (http://www.zlib.net)");
DEFINE_bool(lzo, false,
"Run LZO compression (http://www.oberhumer.com/opensource/lzo/)");
DEFINE_bool(quicklz, false,
"Run quickLZ compression (http://www.quicklz.com/)");
DEFINE_bool(liblzf, false,
"Run libLZF compression "
"(http://www.goof.com/pcg/marc/liblzf.html)");
DEFINE_bool(fastlz, false,
"Run FastLZ compression (http://www.fastlz.org/");
DEFINE_bool(snappy, true, "Run snappy compression");
DEFINE_bool(write_compressed, false,
"Write compressed versions of each file to <file>.comp");
DEFINE_bool(write_uncompressed, false,
"Write uncompressed versions of each file to <file>.uncomp");
DEFINE_bool(snappy_dump_decompression_table, false,
"If true, we print the decompression table during tests.");
namespace snappy {
#ifdef HAVE_FUNC_MMAP
#if defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF)
// To test against code that reads beyond its input, this class copies a
// string to a newly allocated group of pages, the last of which
@@ -79,7 +74,7 @@ namespace snappy {
class DataEndingAtUnreadablePage {
public:
explicit DataEndingAtUnreadablePage(const string& s) {
const size_t page_size = getpagesize();
const size_t page_size = sysconf(_SC_PAGESIZE);
const size_t size = s.size();
// Round up space for string to a multiple of page_size.
size_t space_for_string = (size + page_size - 1) & ~(page_size - 1);
@@ -97,8 +92,9 @@ class DataEndingAtUnreadablePage {
}
~DataEndingAtUnreadablePage() {
const size_t page_size = sysconf(_SC_PAGESIZE);
// Undo the mprotect.
CHECK_EQ(0, mprotect(protected_page_, getpagesize(), PROT_READ|PROT_WRITE));
CHECK_EQ(0, mprotect(protected_page_, page_size, PROT_READ|PROT_WRITE));
CHECK_EQ(0, munmap(mem_, alloc_size_));
}
@@ -113,7 +109,7 @@ class DataEndingAtUnreadablePage {
size_t size_;
};
#else // HAVE_FUNC_MMAP
#else // defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF)
// Fallback for systems without mmap.
typedef string DataEndingAtUnreadablePage;
@@ -121,11 +117,11 @@ typedef string DataEndingAtUnreadablePage;
#endif
enum CompressorType {
ZLIB, LZO, LIBLZF, QUICKLZ, FASTLZ, SNAPPY
ZLIB, LZO, SNAPPY
};
const char* names[] = {
"ZLIB", "LZO", "LIBLZF", "QUICKLZ", "FASTLZ", "SNAPPY"
"ZLIB", "LZO", "SNAPPY"
};
static size_t MinimumRequiredOutputSpace(size_t input_size,
@@ -141,26 +137,12 @@ static size_t MinimumRequiredOutputSpace(size_t input_size,
return input_size + input_size/64 + 16 + 3;
#endif // LZO_VERSION
#ifdef LZF_VERSION
case LIBLZF:
return input_size;
#endif // LZF_VERSION
#ifdef QLZ_VERSION_MAJOR
case QUICKLZ:
return input_size + 36000; // 36000 is used for scratch.
#endif // QLZ_VERSION_MAJOR
#ifdef FASTLZ_VERSION
case FASTLZ:
return max(static_cast<int>(ceil(input_size * 1.05)), 66);
#endif // FASTLZ_VERSION
case SNAPPY:
return snappy::MaxCompressedLength(input_size);
default:
LOG(FATAL) << "Unknown compression type number " << comp;
return 0;
}
}
@@ -214,58 +196,6 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
}
#endif // LZO_VERSION
#ifdef LZF_VERSION
case LIBLZF: {
int destlen = lzf_compress(input,
input_size,
string_as_array(compressed),
input_size);
if (destlen == 0) {
// lzf *can* cause lots of blowup when compressing, so they
// recommend to limit outsize to insize, and just not compress
// if it's bigger. Ideally, we'd just swap input and output.
compressed->assign(input, input_size);
destlen = input_size;
}
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
break;
}
#endif // LZF_VERSION
#ifdef QLZ_VERSION_MAJOR
case QUICKLZ: {
qlz_state_compress *state_compress = new qlz_state_compress;
int destlen = qlz_compress(input,
string_as_array(compressed),
input_size,
state_compress);
delete state_compress;
CHECK_NE(0, destlen);
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
break;
}
#endif // QLZ_VERSION_MAJOR
#ifdef FASTLZ_VERSION
case FASTLZ: {
// Use level 1 compression since we mostly care about speed.
int destlen = fastlz_compress_level(
1,
input,
input_size,
string_as_array(compressed));
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
CHECK_NE(destlen, 0);
break;
}
#endif // FASTLZ_VERSION
case SNAPPY: {
size_t destlen;
snappy::RawCompress(input, input_size,
@@ -278,7 +208,6 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
break;
}
default: {
return false; // the asked-for library wasn't compiled in
}
@@ -321,56 +250,12 @@ static bool Uncompress(const string& compressed, CompressorType comp,
}
#endif // LZO_VERSION
#ifdef LZF_VERSION
case LIBLZF: {
output->resize(size);
int destlen = lzf_decompress(compressed.data(),
compressed.size(),
string_as_array(output),
output->size());
if (destlen == 0) {
// This error probably means we had decided not to compress,
// and thus have stored input in output directly.
output->assign(compressed.data(), compressed.size());
destlen = compressed.size();
}
CHECK_EQ(destlen, size);
break;
}
#endif // LZF_VERSION
#ifdef QLZ_VERSION_MAJOR
case QUICKLZ: {
output->resize(size);
qlz_state_decompress *state_decompress = new qlz_state_decompress;
int destlen = qlz_decompress(compressed.data(),
string_as_array(output),
state_decompress);
delete state_decompress;
CHECK_EQ(destlen, size);
break;
}
#endif // QLZ_VERSION_MAJOR
#ifdef FASTLZ_VERSION
case FASTLZ: {
output->resize(size);
int destlen = fastlz_decompress(compressed.data(),
compressed.length(),
string_as_array(output),
size);
CHECK_EQ(destlen, size);
break;
}
#endif // FASTLZ_VERSION
case SNAPPY: {
snappy::RawUncompress(compressed.data(), compressed.size(),
string_as_array(output));
break;
}
default: {
return false; // the asked-for library wasn't compiled in
}
@@ -392,13 +277,13 @@ static void Measure(const char* data,
{
// Chop the input into blocks
int num_blocks = (length + block_size - 1) / block_size;
vector<const char*> input(num_blocks);
vector<size_t> input_length(num_blocks);
vector<string> compressed(num_blocks);
vector<string> output(num_blocks);
std::vector<const char*> input(num_blocks);
std::vector<size_t> input_length(num_blocks);
std::vector<string> compressed(num_blocks);
std::vector<string> output(num_blocks);
for (int b = 0; b < num_blocks; b++) {
int input_start = b * block_size;
int input_limit = min<int>((b+1)*block_size, length);
int input_limit = std::min<int>((b+1)*block_size, length);
input[b] = data+input_start;
input_length[b] = input_limit-input_start;
@@ -448,13 +333,13 @@ static void Measure(const char* data,
}
compressed_size = 0;
for (int i = 0; i < compressed.size(); i++) {
for (size_t i = 0; i < compressed.size(); i++) {
compressed_size += compressed[i].size();
}
}
sort(ctime, ctime + kRuns);
sort(utime, utime + kRuns);
std::sort(ctime, ctime + kRuns);
std::sort(utime, utime + kRuns);
const int med = kRuns/2;
float comp_rate = (length / ctime[med]) * repeats / 1048576.0;
@@ -469,12 +354,11 @@ static void Measure(const char* data,
x.c_str(),
block_size/(1<<20),
static_cast<int>(length), static_cast<uint32>(compressed_size),
(compressed_size * 100.0) / max<int>(1, length),
(compressed_size * 100.0) / std::max<int>(1, length),
comp_rate,
urate.c_str());
}
static int VerifyString(const string& input) {
string compressed;
DataEndingAtUnreadablePage i(input);
@@ -491,6 +375,23 @@ static int VerifyString(const string& input) {
return uncompressed.size();
}
static void VerifyStringSink(const string& input) {
string compressed;
DataEndingAtUnreadablePage i(input);
const size_t written = snappy::Compress(i.data(), i.size(), &compressed);
CHECK_EQ(written, compressed.size());
CHECK_LE(compressed.size(),
snappy::MaxCompressedLength(input.size()));
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
string uncompressed;
uncompressed.resize(input.size());
snappy::UncheckedByteArraySink sink(string_as_array(&uncompressed));
DataEndingAtUnreadablePage c(compressed);
snappy::ByteArraySource source(c.data(), c.size());
CHECK(snappy::Uncompress(&source, &sink));
CHECK_EQ(uncompressed, input);
}
static void VerifyIOVec(const string& input) {
string compressed;
@@ -505,13 +406,13 @@ static void VerifyIOVec(const string& input) {
// ranging from 1 to 10.
char* buf = new char[input.size()];
ACMRandom rnd(input.size());
int num = rnd.Next() % 10 + 1;
size_t num = rnd.Next() % 10 + 1;
if (input.size() < num) {
num = input.size();
}
struct iovec* iov = new iovec[num];
int used_so_far = 0;
for (int i = 0; i < num; ++i) {
for (size_t i = 0; i < num; ++i) {
iov[i].iov_base = buf + used_so_far;
if (i == num - 1) {
iov[i].iov_len = input.size() - used_so_far;
@@ -562,6 +463,28 @@ static void VerifyNonBlockedCompression(const string& input) {
CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncomp_str));
CHECK_EQ(uncomp_str, input);
// Uncompress using source/sink
string uncomp_str2;
uncomp_str2.resize(input.size());
snappy::UncheckedByteArraySink sink(string_as_array(&uncomp_str2));
snappy::ByteArraySource source(compressed.data(), compressed.size());
CHECK(snappy::Uncompress(&source, &sink));
CHECK_EQ(uncomp_str2, input);
// Uncompress into iovec
{
static const int kNumBlocks = 10;
struct iovec vec[kNumBlocks];
const int block_size = 1 + input.size() / kNumBlocks;
string iovec_data(block_size * kNumBlocks, 'x');
for (int i = 0; i < kNumBlocks; i++) {
vec[i].iov_base = string_as_array(&iovec_data) + i * block_size;
vec[i].iov_len = block_size;
}
CHECK(snappy::RawUncompressToIOVec(compressed.data(), compressed.size(),
vec, kNumBlocks));
CHECK_EQ(string(iovec_data.data(), input.size()), input);
}
}
// Expand the input so that it is at least K times as big as block size
@@ -580,6 +503,8 @@ static int Verify(const string& input) {
// Compress using string based routines
const int result = VerifyString(input);
// Verify using sink based routines
VerifyStringSink(input);
VerifyNonBlockedCompression(input);
VerifyIOVec(input);
@@ -589,12 +514,9 @@ static int Verify(const string& input) {
VerifyIOVec(input);
}
return result;
}
// This test checks to ensure that snappy doesn't coredump if it gets
// corrupted data.
static bool IsValidCompressedBuffer(const string& c) {
return snappy::IsValidCompressedBuffer(c.data(), c.size());
@@ -603,11 +525,13 @@ static bool Uncompress(const string& c, string* u) {
return snappy::Uncompress(c.data(), c.size(), u);
}
TYPED_TEST(CorruptedTest, VerifyCorrupted) {
// This test checks to ensure that snappy doesn't coredump if it gets
// corrupted data.
TEST(CorruptedTest, VerifyCorrupted) {
string source = "making sure we don't crash with corrupted input";
VLOG(1) << source;
string dest;
TypeParam uncmp;
string uncmp;
snappy::Compress(source.data(), source.size(), &dest);
// Mess around with the data. It's hard to simulate all possible
@@ -616,19 +540,19 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
dest[1]--;
dest[3]++;
// this really ought to fail.
CHECK(!IsValidCompressedBuffer(TypeParam(dest)));
CHECK(!Uncompress(TypeParam(dest), &uncmp));
CHECK(!IsValidCompressedBuffer(dest));
CHECK(!Uncompress(dest, &uncmp));
// This is testing for a security bug - a buffer that decompresses to 100k
// but we lie in the snappy header and only reserve 0 bytes of memory :)
source.resize(100000);
for (int i = 0; i < source.length(); ++i) {
for (size_t i = 0; i < source.length(); ++i) {
source[i] = 'A';
}
snappy::Compress(source.data(), source.size(), &dest);
dest[0] = dest[1] = dest[2] = dest[3] = 0;
CHECK(!IsValidCompressedBuffer(TypeParam(dest)));
CHECK(!Uncompress(TypeParam(dest), &uncmp));
CHECK(!IsValidCompressedBuffer(dest));
CHECK(!Uncompress(dest, &uncmp));
if (sizeof(void *) == 4) {
// Another security check; check a crazy big length can't DoS us with an
@@ -637,20 +561,20 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
// where 3 GB might be an acceptable allocation size, Uncompress()
// attempts to decompress, and sometimes causes the test to run out of
// memory.
dest[0] = dest[1] = dest[2] = dest[3] = 0xff;
dest[0] = dest[1] = dest[2] = dest[3] = '\xff';
// This decodes to a really large size, i.e., about 3 GB.
dest[4] = 'k';
CHECK(!IsValidCompressedBuffer(TypeParam(dest)));
CHECK(!Uncompress(TypeParam(dest), &uncmp));
CHECK(!IsValidCompressedBuffer(dest));
CHECK(!Uncompress(dest, &uncmp));
} else {
LOG(WARNING) << "Crazy decompression lengths not checked on 64-bit build";
}
// This decodes to about 2 MB; much smaller, but should still fail.
dest[0] = dest[1] = dest[2] = 0xff;
dest[0] = dest[1] = dest[2] = '\xff';
dest[3] = 0x00;
CHECK(!IsValidCompressedBuffer(TypeParam(dest)));
CHECK(!Uncompress(TypeParam(dest), &uncmp));
CHECK(!IsValidCompressedBuffer(dest));
CHECK(!Uncompress(dest, &uncmp));
// try reading stuff in from a bad file.
for (int i = 1; i <= 3; ++i) {
@@ -665,8 +589,8 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
snappy::ByteArraySource source(data.data(), data.size());
CHECK(!snappy::GetUncompressedLength(&source, &ulen2) ||
(ulen2 < (1<<20)));
CHECK(!IsValidCompressedBuffer(TypeParam(data)));
CHECK(!Uncompress(TypeParam(data), &uncmp));
CHECK(!IsValidCompressedBuffer(data));
CHECK(!Uncompress(data, &uncmp));
}
}
@@ -764,7 +688,7 @@ TEST(Snappy, RandomData) {
}
string x;
int len = rnd.Uniform(4096);
size_t len = rnd.Uniform(4096);
if (i < 100) {
len = 65536 + rnd.Uniform(65536);
}
@@ -929,7 +853,6 @@ TEST(Snappy, IOVecCopyOverflow) {
}
}
static bool CheckUncompressedLength(const string& compressed,
size_t* ulength) {
const bool result1 = snappy::GetUncompressedLength(compressed.data(),
@@ -956,11 +879,11 @@ TEST(SnappyCorruption, TruncatedVarint) {
TEST(SnappyCorruption, UnterminatedVarint) {
string compressed, uncompressed;
size_t ulength;
compressed.push_back(128);
compressed.push_back(128);
compressed.push_back(128);
compressed.push_back(128);
compressed.push_back(128);
compressed.push_back('\x80');
compressed.push_back('\x80');
compressed.push_back('\x80');
compressed.push_back('\x80');
compressed.push_back('\x80');
compressed.push_back(10);
CHECK(!CheckUncompressedLength(compressed, &ulength));
CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
@@ -968,6 +891,20 @@ TEST(SnappyCorruption, UnterminatedVarint) {
&uncompressed));
}
TEST(SnappyCorruption, OverflowingVarint) {
string compressed, uncompressed;
size_t ulength;
compressed.push_back('\xfb');
compressed.push_back('\xff');
compressed.push_back('\xff');
compressed.push_back('\xff');
compressed.push_back('\x7f');
CHECK(!CheckUncompressedLength(compressed, &ulength));
CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
CHECK(!snappy::Uncompress(compressed.data(), compressed.size(),
&uncompressed));
}
TEST(Snappy, ReadPastEndOfBuffer) {
// Check that we do not read past end of input
@@ -998,11 +935,13 @@ TEST(Snappy, ZeroOffsetCopyValidation) {
EXPECT_FALSE(snappy::IsValidCompressedBuffer(compressed, 4));
}
namespace {
int TestFindMatchLength(const char* s1, const char *s2, unsigned length) {
return snappy::internal::FindMatchLength(s1, s2, s2 + length);
std::pair<size_t, bool> p =
snappy::internal::FindMatchLength(s1, s2, s2 + length);
CHECK_EQ(p.first < 8, p.second);
return p.first;
}
} // namespace
@@ -1112,8 +1051,7 @@ TEST(Snappy, FindMatchLengthRandom) {
}
DataEndingAtUnreadablePage u(s);
DataEndingAtUnreadablePage v(t);
int matched = snappy::internal::FindMatchLength(
u.data(), v.data(), v.data() + t.size());
int matched = TestFindMatchLength(u.data(), v.data(), t.size());
if (matched == t.size()) {
EXPECT_EQ(s, t);
} else {
@@ -1125,21 +1063,113 @@ TEST(Snappy, FindMatchLengthRandom) {
}
}
static uint16 MakeEntry(unsigned int extra,
unsigned int len,
unsigned int copy_offset) {
// Check that all of the fields fit within the allocated space
assert(extra == (extra & 0x7)); // At most 3 bits
assert(copy_offset == (copy_offset & 0x7)); // At most 3 bits
assert(len == (len & 0x7f)); // At most 7 bits
return len | (copy_offset << 8) | (extra << 11);
}
// Check that the decompression table is correct, and optionally print out
// the computed one.
TEST(Snappy, VerifyCharTable) {
using snappy::internal::LITERAL;
using snappy::internal::COPY_1_BYTE_OFFSET;
using snappy::internal::COPY_2_BYTE_OFFSET;
using snappy::internal::COPY_4_BYTE_OFFSET;
using snappy::internal::char_table;
uint16 dst[256];
// Place invalid entries in all places to detect missing initialization
int assigned = 0;
for (int i = 0; i < 256; i++) {
dst[i] = 0xffff;
}
// Small LITERAL entries. We store (len-1) in the top 6 bits.
for (unsigned int len = 1; len <= 60; len++) {
dst[LITERAL | ((len-1) << 2)] = MakeEntry(0, len, 0);
assigned++;
}
// Large LITERAL entries. We use 60..63 in the high 6 bits to
// encode the number of bytes of length info that follow the opcode.
for (unsigned int extra_bytes = 1; extra_bytes <= 4; extra_bytes++) {
// We set the length field in the lookup table to 1 because extra
// bytes encode len-1.
dst[LITERAL | ((extra_bytes+59) << 2)] = MakeEntry(extra_bytes, 1, 0);
assigned++;
}
// COPY_1_BYTE_OFFSET.
//
// The tag byte in the compressed data stores len-4 in 3 bits, and
// offset/256 in 5 bits. offset%256 is stored in the next byte.
//
// This format is used for length in range [4..11] and offset in
// range [0..2047]
for (unsigned int len = 4; len < 12; len++) {
for (unsigned int offset = 0; offset < 2048; offset += 256) {
dst[COPY_1_BYTE_OFFSET | ((len-4)<<2) | ((offset>>8)<<5)] =
MakeEntry(1, len, offset>>8);
assigned++;
}
}
// COPY_2_BYTE_OFFSET.
// Tag contains len-1 in top 6 bits, and offset in next two bytes.
for (unsigned int len = 1; len <= 64; len++) {
dst[COPY_2_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(2, len, 0);
assigned++;
}
// COPY_4_BYTE_OFFSET.
// Tag contents len-1 in top 6 bits, and offset in next four bytes.
for (unsigned int len = 1; len <= 64; len++) {
dst[COPY_4_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(4, len, 0);
assigned++;
}
// Check that each entry was initialized exactly once.
EXPECT_EQ(256, assigned) << "Assigned only " << assigned << " of 256";
for (int i = 0; i < 256; i++) {
EXPECT_NE(0xffff, dst[i]) << "Did not assign byte " << i;
}
if (FLAGS_snappy_dump_decompression_table) {
printf("static const uint16 char_table[256] = {\n ");
for (int i = 0; i < 256; i++) {
printf("0x%04x%s",
dst[i],
((i == 255) ? "\n" : (((i%8) == 7) ? ",\n " : ", ")));
}
printf("};\n");
}
// Check that computed table matched recorded table.
for (int i = 0; i < 256; i++) {
EXPECT_EQ(dst[i], char_table[i]) << "Mismatch in byte " << i;
}
}
static void CompressFile(const char* fname) {
string fullinput;
file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
string compressed;
Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false);
file::SetContents(string(fname).append(".comp"), compressed, file::Defaults())
.CheckSuccess();
CHECK_OK(file::SetContents(string(fname).append(".comp"), compressed,
file::Defaults()));
}
static void UncompressFile(const char* fname) {
string fullinput;
file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
size_t uncompLength;
CHECK(CheckUncompressedLength(fullinput, &uncompLength));
@@ -1148,28 +1178,25 @@ static void UncompressFile(const char* fname) {
uncompressed.resize(uncompLength);
CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed));
file::SetContents(string(fname).append(".uncomp"), uncompressed,
file::Defaults()).CheckSuccess();
CHECK_OK(file::SetContents(string(fname).append(".uncomp"), uncompressed,
file::Defaults()));
}
static void MeasureFile(const char* fname) {
string fullinput;
file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
printf("%-40s :\n", fname);
int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len;
int end_len = fullinput.size();
if (FLAGS_end_len >= 0) {
end_len = min<int>(fullinput.size(), FLAGS_end_len);
end_len = std::min<int>(fullinput.size(), FLAGS_end_len);
}
for (int len = start_len; len <= end_len; len++) {
const char* const input = fullinput.data();
int repeats = (FLAGS_bytes + len) / (len + 1);
if (FLAGS_zlib) Measure(input, len, ZLIB, repeats, 1024<<10);
if (FLAGS_lzo) Measure(input, len, LZO, repeats, 1024<<10);
if (FLAGS_liblzf) Measure(input, len, LIBLZF, repeats, 1024<<10);
if (FLAGS_quicklz) Measure(input, len, QUICKLZ, repeats, 1024<<10);
if (FLAGS_fastlz) Measure(input, len, FASTLZ, repeats, 1024<<10);
if (FLAGS_snappy) Measure(input, len, SNAPPY, repeats, 4096<<10);
// For block-size based measurements
@@ -1298,6 +1325,37 @@ static void BM_UIOVec(int iters, int arg) {
}
BENCHMARK(BM_UIOVec)->DenseRange(0, 4);
static void BM_UFlatSink(int iters, int arg) {
StopBenchmarkTiming();
// Pick file to process based on "arg"
CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files));
string contents = ReadTestDataFile(files[arg].filename,
files[arg].size_limit);
string zcontents;
snappy::Compress(contents.data(), contents.size(), &zcontents);
char* dst = new char[contents.size()];
SetBenchmarkBytesProcessed(static_cast<int64>(iters) *
static_cast<int64>(contents.size()));
SetBenchmarkLabel(files[arg].label);
StartBenchmarkTiming();
while (iters-- > 0) {
snappy::ByteArraySource source(zcontents.data(), zcontents.size());
snappy::UncheckedByteArraySink sink(dst);
CHECK(snappy::Uncompress(&source, &sink));
}
StopBenchmarkTiming();
string s(dst, contents.size());
CHECK_EQ(contents, s);
delete[] dst;
}
BENCHMARK(BM_UFlatSink)->DenseRange(0, ARRAYSIZE(files) - 1);
static void BM_ZFlat(int iters, int arg) {
StopBenchmarkTiming();
@@ -1329,23 +1387,20 @@ static void BM_ZFlat(int iters, int arg) {
}
BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
} // namespace snappy
int main(int argc, char** argv) {
InitGoogle(argv[0], &argc, &argv, true);
RunSpecifiedBenchmarks();
if (argc >= 2) {
for (int arg = 1; arg < argc; arg++) {
if (FLAGS_write_compressed) {
CompressFile(argv[arg]);
snappy::CompressFile(argv[arg]);
} else if (FLAGS_write_uncompressed) {
UncompressFile(argv[arg]);
snappy::UncompressFile(argv[arg]);
} else {
MeasureFile(argv[arg]);
snappy::MeasureFile(argv[arg]);
}
}
return 0;