ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-11-20 11:05:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update snappy to 1.1.7

Browse Source
This commit is contained in:
Brad Chase
2017-12-18 13:41:54 -05:00
parent 80050c110f 2836c20649
commit 516a1b85d1
25 changed files with 1594 additions and 2820 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/snappy/snappy/.appveyor.yml Normal file
View File

@@ -0,0 +1,41 @@
# Build matrix / environment variables are explained on:
# https://www.appveyor.com/docs/appveyor-yml/
# This file can be validated on: https://ci.appveyor.com/tools/validate-yaml
version: "{build}"
environment:
matrix:
# AppVeyor currently has no custom job name feature.
# http://help.appveyor.com/discussions/questions/1623-can-i-provide-a-friendly-name-for-jobs
- JOB: Visual Studio 2017
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
CMAKE_GENERATOR: Visual Studio 15 2017
- JOB: Visual Studio 2015
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
CMAKE_GENERATOR: Visual Studio 14 2015
platform:
- x86
- x64
configuration:
- RelWithDebInfo
- Debug
build:
verbosity: minimal
build_script:
- git submodule update --init --recursive
- mkdir out
- cd out
- if "%platform%"=="x64" set CMAKE_GENERATOR=%CMAKE_GENERATOR% Win64
- cmake --version
- cmake .. -G "%CMAKE_GENERATOR%"
-DCMAKE_CONFIGURATION_TYPES="%CONFIGURATION%"
- cmake --build . --config %CONFIGURATION%
- cd ..
test_script:
- out\%CONFIGURATION%\snappy_unittest

59
src/snappy/snappy/.travis.yml Normal file
View File

@@ -0,0 +1,59 @@
# Build matrix / environment variables are explained on:
# http://about.travis-ci.org/docs/user/build-configuration/
# This file can be validated on: http://lint.travis-ci.org/
sudo: false
dist: trusty
language: cpp
compiler:
- gcc
- clang
os:
- linux
- osx
env:
- BUILD_TYPE=Debug
- BUILD_TYPE=RelWithDebInfo
matrix:
allow_failures:
- compiler: clang
env: BUILD_TYPE=RelWithDebInfo
addons:
apt:
# List of whitelisted in travis packages for ubuntu-trusty can be found here:
# https://github.com/travis-ci/apt-package-whitelist/blob/master/ubuntu-trusty
# List of whitelisted in travis apt-sources:
# https://github.com/travis-ci/apt-source-whitelist/blob/master/ubuntu.json
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty-4.0
packages:
- cmake
- gcc-6
- g++-6
- clang-4.0
install:
# Travis doesn't have a nice way to install homebrew packages yet.
# https://github.com/travis-ci/travis-ci/issues/5377
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew update; fi
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install gcc@6; fi
# /usr/bin/gcc is stuck to old versions by on both Linux and OSX.
- if [ "$CXX" = "g++" ]; then export CXX="g++-6" CC="gcc-6"; fi
- echo ${CC}
- echo ${CXX}
- ${CXX} --version
- cmake --version
before_script:
- mkdir -p build && cd build
- cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE
- cmake --build .
- cd ..
script:
- build/snappy_unittest

174
src/snappy/snappy/CMakeLists.txt Normal file
View File

@@ -0,0 +1,174 @@
cmake_minimum_required(VERSION 3.1)
project(Snappy VERSION 1.1.7 LANGUAGES C CXX)
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make
# it prominent in the GUI.
option(BUILD_SHARED_LIBS "Build shared libraries(DLLs)." OFF)
option(SNAPPY_BUILD_TESTS "Build Snappy's own tests." ON)
include(TestBigEndian)
test_big_endian(SNAPPY_IS_BIG_ENDIAN)
include(CheckIncludeFile)
check_include_file("byteswap.h" HAVE_BYTESWAP_H)
check_include_file("stddef.h" HAVE_STDDEF_H)
check_include_file("stdint.h" HAVE_STDINT_H)
check_include_file("sys/endian.h" HAVE_SYS_ENDIAN_H)
check_include_file("sys/mman.h" HAVE_SYS_MMAN_H)
check_include_file("sys/resource.h" HAVE_SYS_RESOURCE_H)
check_include_file("sys/time.h" HAVE_SYS_TIME_H)
check_include_file("sys/uio.h" HAVE_SYS_UIO_H)
check_include_file("unistd.h" HAVE_UNISTD_H)
check_include_file("windows.h" HAVE_WINDOWS_H)
include(CheckLibraryExists)
check_library_exists(z zlibVersion "" HAVE_LIBZ)
check_library_exists(lzo2 lzo1x_1_15_compress "" HAVE_LIBLZO2)
include(CheckCXXSourceCompiles)
check_cxx_source_compiles(
"int main(void) { return __builtin_expect(0, 1); }" HAVE_BUILTIN_EXPECT)
check_cxx_source_compiles(
"int main(void) { return __builtin_ctzll(0); }" HAVE_BUILTIN_CTZ)
include(CheckSymbolExists)
check_symbol_exists("mmap" "sys/mman.h" HAVE_FUNC_MMAP)
check_symbol_exists("sysconf" "unistd.h" HAVE_FUNC_SYSCONF)
find_package(GTest QUIET)
if(GTEST_FOUND)
set(HAVE_GTEST 1)
endif(GTEST_FOUND)
find_package(Gflags QUIET)
if(GFLAGS_FOUND)
set(HAVE_GFLAGS 1)
endif(GFLAGS_FOUND)
configure_file(
"${PROJECT_SOURCE_DIR}/cmake/config.h.in"
"${PROJECT_BINARY_DIR}/config.h"
)
# We don't want to define HAVE_ macros in public headers. Instead, we use
# CMake's variable substitution with 0/1 variables, which will be seen by the
# preprocessor as constants.
set(HAVE_STDINT_H_01 ${HAVE_STDINT_H})
set(HAVE_STDDEF_H_01 ${HAVE_STDDEF_H})
set(HAVE_SYS_UIO_H_01 ${HAVE_SYS_UIO_H})
if(NOT HAVE_STDINT_H_01)
set(HAVE_STDINT_H_01 0)
endif(NOT HAVE_STDINT_H_01)
if(NOT HAVE_STDDEF_H_01)
set(HAVE_STDDEF_H_01 0)
endif(NOT HAVE_STDDEF_H_01)
if(NOT HAVE_SYS_UIO_H_01)
set(HAVE_SYS_UIO_H_01 0)
endif(NOT HAVE_SYS_UIO_H_01)
configure_file(
"${PROJECT_SOURCE_DIR}/snappy-stubs-public.h.in"
"${PROJECT_BINARY_DIR}/snappy-stubs-public.h")
add_library(snappy "")
target_sources(snappy
PRIVATE
"${PROJECT_SOURCE_DIR}/snappy-internal.h"
"${PROJECT_SOURCE_DIR}/snappy-stubs-internal.h"
"${PROJECT_SOURCE_DIR}/snappy-c.cc"
"${PROJECT_SOURCE_DIR}/snappy-sinksource.cc"
"${PROJECT_SOURCE_DIR}/snappy-stubs-internal.cc"
"${PROJECT_SOURCE_DIR}/snappy.cc"
"${PROJECT_BINARY_DIR}/config.h"
# Only CMake 3.3+ supports PUBLIC sources in targets exported by "install".
$<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-c.h>
$<INSTALL_INTERFACE:include/snappy-c.h>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-sinksource.h>
$<INSTALL_INTERFACE:include/snappy-sinksource.h>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy.h>
$<INSTALL_INTERFACE:include/snappy.h>
$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/snappy-stubs-public.h>
$<INSTALL_INTERFACE:include/snappy-stubs-public.h>
)
target_include_directories(snappy
PUBLIC
$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>
$<INSTALL_INTERFACE:include>
)
set_target_properties(snappy
PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
target_compile_definitions(snappy PRIVATE -DHAVE_CONFIG_H)
if(BUILD_SHARED_LIBS)
set_target_properties(snappy PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
endif(BUILD_SHARED_LIBS)
if(SNAPPY_BUILD_TESTS)
enable_testing()
add_executable(snappy_unittest "")
target_sources(snappy_unittest
PRIVATE
"${PROJECT_SOURCE_DIR}/snappy_unittest.cc"
"${PROJECT_SOURCE_DIR}/snappy-test.cc"
)
target_compile_definitions(snappy_unittest PRIVATE -DHAVE_CONFIG_H)
target_link_libraries(snappy_unittest snappy ${GFLAGS_LIBRARIES})
if(HAVE_LIBZ)
target_link_libraries(snappy_unittest z)
endif(HAVE_LIBZ)
if(HAVE_LIBLZO2)
target_link_libraries(snappy_unittest lzo2)
endif(HAVE_LIBLZO2)
target_include_directories(snappy_unittest
BEFORE PRIVATE
"${PROJECT_SOURCE_DIR}"
"${GTEST_INCLUDE_DIRS}"
"${GFLAGS_INCLUDE_DIRS}"
)
add_test(
NAME snappy_unittest
WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
COMMAND "${PROJECT_BINARY_DIR}/snappy_unittest")
endif(SNAPPY_BUILD_TESTS)
include(GNUInstallDirs)
install(TARGETS snappy
EXPORT SnappyTargets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
install(
FILES
"${PROJECT_SOURCE_DIR}/snappy-c.h"
"${PROJECT_SOURCE_DIR}/snappy-sinksource.h"
"${PROJECT_SOURCE_DIR}/snappy.h"
"${PROJECT_BINARY_DIR}/snappy-stubs-public.h"
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
)
include(CMakePackageConfigHelpers)
write_basic_package_version_file(
"${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake"
COMPATIBILITY SameMajorVersion
)
install(
EXPORT SnappyTargets
NAMESPACE Snappy::
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy"
)
install(
FILES
"${PROJECT_SOURCE_DIR}/cmake/SnappyConfig.cmake"
"${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy"
)

26
src/snappy/snappy/CONTRIBUTING.md Normal file
View File

@@ -0,0 +1,26 @@
# How to Contribute
We'd love to accept your patches and contributions to this project. There are
just a few small guidelines you need to follow.
## Contributor License Agreement
Contributions to this project must be accompanied by a Contributor License
Agreement. You (or your employer) retain the copyright to your contribution,
this simply gives us permission to use and redistribute your contributions as
part of the project. Head over to <https://cla.developers.google.com/> to see
your current agreements on file or to sign a new one.
You generally only need to submit a CLA once, so if you've already submitted one
(even if it was for a different project), you probably don't need to do it
again.
## Code reviews
All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult
[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
information on using pull requests.
Please make sure that all the automated checks (CLA, AppVeyor, Travis) pass for
your pull requests. Pull requests whose checks fail may be ignored.

2
src/snappy/snappy/COPYING
View File

@@ -29,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
=== ===
Some of the benchmark data in util/zippy/testdata is licensed differently: Some of the benchmark data in testdata/ is licensed differently:
- fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and
is licensed under the Creative Commons Attribution 3.0 license is licensed under the Creative Commons Attribution 3.0 license

1916
src/snappy/snappy/ChangeLog
View File

File diff suppressed because it is too large Load Diff

23
src/snappy/snappy/Makefile.am
View File

@@ -1,23 +0,0 @@
ACLOCAL_AMFLAGS = -I m4
# Library.
lib_LTLIBRARIES = libsnappy.la
libsnappy_la_SOURCES = snappy.cc snappy-sinksource.cc snappy-stubs-internal.cc snappy-c.cc
libsnappy_la_LDFLAGS = -version-info $(SNAPPY_LTVERSION)
include_HEADERS = snappy.h snappy-sinksource.h snappy-stubs-public.h snappy-c.h
noinst_HEADERS = snappy-internal.h snappy-stubs-internal.h snappy-test.h
# Unit tests and benchmarks.
snappy_unittest_CPPFLAGS = $(gflags_CFLAGS) $(GTEST_CPPFLAGS)
snappy_unittest_SOURCES = snappy_unittest.cc snappy-test.cc
snappy_unittest_LDFLAGS = $(GTEST_LDFLAGS)
snappy_unittest_LDADD = libsnappy.la $(UNITTEST_LIBS) $(gflags_LIBS) $(GTEST_LIBS)
TESTS = snappy_unittest
noinst_PROGRAMS = $(TESTS)
EXTRA_DIST = autogen.sh testdata/alice29.txt testdata/asyoulik.txt testdata/baddata1.snappy testdata/baddata2.snappy testdata/baddata3.snappy testdata/geo.protodata testdata/fireworks.jpeg testdata/html testdata/html_x_4 testdata/kppkn.gtb testdata/lcet10.txt testdata/paper-100k.pdf testdata/plrabn12.txt testdata/urls.10K
dist_doc_DATA = ChangeLog COPYING INSTALL NEWS README format_description.txt framing_format.txt
libtool: $(LIBTOOL_DEPS)
$(SHELL) ./config.status --recheck

52
src/snappy/snappy/NEWS
View File

@@ -1,3 +1,55 @@
Snappy v1.1.7, August 24th 2017:
* Improved CMake build support for 64-bit Linux distributions.
* MSVC builds now use MSVC-specific intrinsics that map to clzll.
* ARM64 (AArch64) builds use the code paths optimized for 64-bit processors.
Snappy v1.1.6, July 12th 2017:
This is a re-release of v1.1.5 with proper SONAME / SOVERSION values.
Snappy v1.1.5, June 28th 2017:
This release has broken SONAME / SOVERSION values. Users of snappy as a shared
library should avoid 1.1.5 and use 1.1.6 instead. SONAME / SOVERSION errors will
manifest as the dynamic library loader complaining that it cannot find snappy's
shared library file (libsnappy.so / libsnappy.dylib), or that the library it
found does not have the required version. 1.1.6 has the same code as 1.1.5, but
carries build configuration fixes for the issues above.
* Add CMake build support. The autoconf build support is now deprecated, and
will be removed in the next release.
* Add AppVeyor configuration, for Windows CI coverage.
* Small performance improvement on little-endian PowerPC.
* Small performance improvement on LLVM with position-independent executables.
* Fix a few issues with various build environments.
Snappy v1.1.4, January 25th 2017:
* Fix a 1% performance regression when snappy is used in PIE executables.
* Improve compression performance by 5%.
* Improve decompression performance by 20%.
Snappy v1.1.3, July 6th 2015:
This is the first release to be done from GitHub, which means that
some minor things like the ChangeLog format has changed (git log
format instead of svn log).
* Add support for Uncompress() from a Source to a Sink.
* Various minor changes to improve MSVC support; in particular,
the unit tests now compile and run under MSVC.
Snappy v1.1.2, February 28th 2014: Snappy v1.1.2, February 28th 2014:
This is a maintenance release with no changes to the actual library This is a maintenance release with no changes to the actual library

32
src/snappy/snappy/README → src/snappy/snappy/README.md
View File

@@ -29,12 +29,12 @@ and the like.
Performance Performance
=========== ===========
Snappy is intended to be fast. On a single core of a Core i7 processor Snappy is intended to be fast. On a single core of a Core i7 processor
in 64-bit mode, it compresses at about 250 MB/sec or more and decompresses at in 64-bit mode, it compresses at about 250 MB/sec or more and decompresses at
about 500 MB/sec or more. (These numbers are for the slowest inputs in our about 500 MB/sec or more. (These numbers are for the slowest inputs in our
benchmark suite; others are much faster.) In our tests, Snappy usually benchmark suite; others are much faster.) In our tests, Snappy usually
is faster than algorithms in the same class (e.g. LZO, LZF, FastLZ, QuickLZ, is faster than algorithms in the same class (e.g. LZO, LZF, QuickLZ,
etc.) while achieving comparable compression ratios. etc.) while achieving comparable compression ratios.
Typical compression ratios (based on the benchmark suite) are about 1.5-1.7x Typical compression ratios (based on the benchmark suite) are about 1.5-1.7x
@@ -52,7 +52,7 @@ In particular:
- Snappy uses 64-bit operations in several places to process more data at - Snappy uses 64-bit operations in several places to process more data at
once than would otherwise be possible. once than would otherwise be possible.
- Snappy assumes unaligned 32- and 64-bit loads and stores are cheap. - Snappy assumes unaligned 32- and 64-bit loads and stores are cheap.
On some platforms, these must be emulated with single-byte loads On some platforms, these must be emulated with single-byte loads
and stores, which is much slower. and stores, which is much slower.
- Snappy assumes little-endian throughout, and needs to byte-swap data in - Snappy assumes little-endian throughout, and needs to byte-swap data in
several places if running on a big-endian platform. several places if running on a big-endian platform.
@@ -62,12 +62,22 @@ Performance optimizations, whether for 64-bit x86 or other platforms,
are of course most welcome; see "Contact", below. are of course most welcome; see "Contact", below.
Building
========
CMake is supported and autotools will soon be deprecated.
You need CMake 3.4 or above to build:
mkdir build
cd build && cmake ../ && make
Usage Usage
===== =====
Note that Snappy, both the implementation and the main interface, Note that Snappy, both the implementation and the main interface,
is written in C++. However, several third-party bindings to other languages is written in C++. However, several third-party bindings to other languages
are available; see the Google Code page at http://code.google.com/p/snappy/ are available; see the home page at http://google.github.io/snappy/
for more information. Also, if you want to use Snappy from C code, you can for more information. Also, if you want to use Snappy from C code, you can
use the included C bindings in snappy-c.h. use the included C bindings in snappy-c.h.
@@ -102,12 +112,12 @@ tests to verify you have not broken anything. Note that if you have the
Google Test library installed, unit test behavior (especially failures) will be Google Test library installed, unit test behavior (especially failures) will be
significantly more user-friendly. You can find Google Test at significantly more user-friendly. You can find Google Test at
http://code.google.com/p/googletest/ http://github.com/google/googletest
You probably also want the gflags library for handling of command-line flags; You probably also want the gflags library for handling of command-line flags;
you can find it at you can find it at
http://code.google.com/p/google-gflags/ http://gflags.github.io/gflags/
In addition to the unit tests, snappy contains microbenchmarks used to In addition to the unit tests, snappy contains microbenchmarks used to
tune compression and decompression performance. These are automatically run tune compression and decompression performance. These are automatically run
@@ -116,7 +126,7 @@ before the unit tests, but you can disable them using the flag
need to edit the source). need to edit the source).
Finally, snappy can benchmark Snappy against a few other compression libraries Finally, snappy can benchmark Snappy against a few other compression libraries
(zlib, LZO, LZF, FastLZ and QuickLZ), if they were detected at configure time. (zlib, LZO, LZF, and QuickLZ), if they were detected at configure time.
To benchmark using a given file, give the compression algorithm you want to test To benchmark using a given file, give the compression algorithm you want to test
Snappy against (e.g. --zlib) and then a list of one or more file names on the Snappy against (e.g. --zlib) and then a list of one or more file names on the
command line. The testdata/ directory contains the files used by the command line. The testdata/ directory contains the files used by the
@@ -129,7 +139,11 @@ test.)
Contact Contact
======= =======
Snappy is distributed through Google Code. For the latest version, a bug tracker, Snappy is distributed through GitHub. For the latest version, a bug tracker,
and other information, see and other information, see
http://code.google.com/p/snappy/ http://google.github.io/snappy/
or the repository at
https://github.com/google/snappy

7
src/snappy/snappy/autogen.sh
View File

@@ -1,7 +0,0 @@
#! /bin/sh -e
rm -rf autom4te.cache
aclocal -I m4
autoheader
libtoolize --copy
automake --add-missing --copy
autoconf

1
src/snappy/snappy/cmake/SnappyConfig.cmake Normal file
View File

@@ -0,0 +1 @@
include("${CMAKE_CURRENT_LIST_DIR}/SnappyTargets.cmake")

62
src/snappy/snappy/cmake/config.h.in Normal file
View File

@@ -0,0 +1,62 @@
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
#define THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
/* Define to 1 if the compiler supports __builtin_ctz and friends. */
#cmakedefine HAVE_BUILTIN_CTZ 1
/* Define to 1 if the compiler supports __builtin_expect. */
#cmakedefine HAVE_BUILTIN_EXPECT 1
/* Define to 1 if you have the <byteswap.h> header file. */
#cmakedefine HAVE_BYTESWAP_H 1
/* Define to 1 if you have a definition for mmap() in <sys/mman.h>. */
#cmakedefine HAVE_FUNC_MMAP 1
/* Define to 1 if you have a definition for sysconf() in <unistd.h>. */
#cmakedefine HAVE_FUNC_SYSCONF 1
/* Define to 1 to use the gflags package for command-line parsing. */
#cmakedefine HAVE_GFLAGS 1
/* Define to 1 if you have Google Test. */
#cmakedefine HAVE_GTEST 1
/* Define to 1 if you have the `lzo2' library (-llzo2). */
#cmakedefine HAVE_LIBLZO2 1
/* Define to 1 if you have the `z' library (-lz). */
#cmakedefine HAVE_LIBZ 1
/* Define to 1 if you have the <stddef.h> header file. */
#cmakedefine HAVE_STDDEF_H 1
/* Define to 1 if you have the <stdint.h> header file. */
#cmakedefine HAVE_STDINT_H 1
/* Define to 1 if you have the <sys/endian.h> header file. */
#cmakedefine HAVE_SYS_ENDIAN_H 1
/* Define to 1 if you have the <sys/mman.h> header file. */
#cmakedefine HAVE_SYS_MMAN_H 1
/* Define to 1 if you have the <sys/resource.h> header file. */
#cmakedefine HAVE_SYS_RESOURCE_H 1
/* Define to 1 if you have the <sys/time.h> header file. */
#cmakedefine HAVE_SYS_TIME_H 1
/* Define to 1 if you have the <sys/uio.h> header file. */
#cmakedefine HAVE_SYS_UIO_H 1
/* Define to 1 if you have the <unistd.h> header file. */
#cmakedefine HAVE_UNISTD_H 1
/* Define to 1 if you have the <windows.h> header file. */
#cmakedefine HAVE_WINDOWS_H 1
/* Define to 1 if your processor stores words with the most significant byte
first (like Motorola and SPARC, unlike Intel and VAX). */
#cmakedefine SNAPPY_IS_BIG_ENDIAN 1
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_

133
src/snappy/snappy/configure.ac
View File

@@ -1,133 +0,0 @@
m4_define([snappy_major], [1])
m4_define([snappy_minor], [1])
m4_define([snappy_patchlevel], [2])
# Libtool shared library interface versions (current:revision:age)
# Update this value for every release! (A:B:C will map to foo.so.(A-C).C.B)
# http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
m4_define([snappy_ltversion], [3:1:2])
AC_INIT([snappy], [snappy_major.snappy_minor.snappy_patchlevel])
AC_CONFIG_MACRO_DIR([m4])
# These are flags passed to automake (though they look like gcc flags!)
AM_INIT_AUTOMAKE([-Wall])
LT_INIT
AC_SUBST([LIBTOOL_DEPS])
AC_PROG_CXX
AC_LANG([C++])
AC_C_BIGENDIAN
AC_TYPE_SIZE_T
AC_TYPE_SSIZE_T
AC_CHECK_HEADERS([stdint.h stddef.h sys/mman.h sys/resource.h windows.h byteswap.h sys/byteswap.h sys/endian.h sys/time.h])
# Don't use AC_FUNC_MMAP, as it checks for mappings of already-mapped memory,
# which we don't need (and does not exist on Windows).
AC_CHECK_FUNC([mmap])
GTEST_LIB_CHECK([], [true], [true # Ignore; we can live without it.])
AC_ARG_WITH([gflags],
[AS_HELP_STRING(
[--with-gflags],
[use Google Flags package to enhance the unit test @<:@default=check@:>@])],
[],
[with_gflags=check])
if test "x$with_gflags" != "xno"; then
PKG_CHECK_MODULES(
[gflags],
[libgflags],
[AC_DEFINE([HAVE_GFLAGS], [1], [Use the gflags package for command-line parsing.])],
[if test "x$with_gflags" != "xcheck"; then
AC_MSG_FAILURE([--with-gflags was given, but test for gflags failed])
fi])
fi
# See if we have __builtin_expect.
# TODO: Use AC_CACHE.
AC_MSG_CHECKING([if the compiler supports __builtin_expect])
AC_TRY_COMPILE(, [
return __builtin_expect(1, 1) ? 1 : 0
], [
snappy_have_builtin_expect=yes
AC_MSG_RESULT([yes])
], [
snappy_have_builtin_expect=no
AC_MSG_RESULT([no])
])
if test x$snappy_have_builtin_expect = xyes ; then
AC_DEFINE([HAVE_BUILTIN_EXPECT], [1], [Define to 1 if the compiler supports __builtin_expect.])
fi
# See if we have working count-trailing-zeros intrinsics.
# TODO: Use AC_CACHE.
AC_MSG_CHECKING([if the compiler supports __builtin_ctzll])
AC_TRY_COMPILE(, [
return (__builtin_ctzll(0x100000000LL) == 32) ? 1 : 0
], [
snappy_have_builtin_ctz=yes
AC_MSG_RESULT([yes])
], [
snappy_have_builtin_ctz=no
AC_MSG_RESULT([no])
])
if test x$snappy_have_builtin_ctz = xyes ; then
AC_DEFINE([HAVE_BUILTIN_CTZ], [1], [Define to 1 if the compiler supports __builtin_ctz and friends.])
fi
# Other compression libraries; the unit test can use these for comparison
# if they are available. If they are not found, just ignore.
UNITTEST_LIBS=""
AC_DEFUN([CHECK_EXT_COMPRESSION_LIB], [
AH_CHECK_LIB([$1])
AC_CHECK_LIB(
[$1],
[$2],
[
AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_LIB$1))
UNITTEST_LIBS="-l$1 $UNITTEST_LIBS"
],
[true]
)
])
CHECK_EXT_COMPRESSION_LIB([z], [zlibVersion])
CHECK_EXT_COMPRESSION_LIB([lzo2], [lzo1x_1_15_compress])
CHECK_EXT_COMPRESSION_LIB([lzf], [lzf_compress])
CHECK_EXT_COMPRESSION_LIB([fastlz], [fastlz_compress])
CHECK_EXT_COMPRESSION_LIB([quicklz], [qlz_compress])
AC_SUBST([UNITTEST_LIBS])
# These are used by snappy-stubs-public.h.in.
if test "$ac_cv_header_stdint_h" = "yes"; then
AC_SUBST([ac_cv_have_stdint_h], [1])
else
AC_SUBST([ac_cv_have_stdint_h], [0])
fi
if test "$ac_cv_header_stddef_h" = "yes"; then
AC_SUBST([ac_cv_have_stddef_h], [1])
else
AC_SUBST([ac_cv_have_stddef_h], [0])
fi
if test "$ac_cv_header_sys_uio_h" = "yes"; then
AC_SUBST([ac_cv_have_sys_uio_h], [1])
else
AC_SUBST([ac_cv_have_sys_uio_h], [0])
fi
# Export the version to snappy-stubs-public.h.
SNAPPY_MAJOR="snappy_major"
SNAPPY_MINOR="snappy_minor"
SNAPPY_PATCHLEVEL="snappy_patchlevel"
AC_SUBST([SNAPPY_MAJOR])
AC_SUBST([SNAPPY_MINOR])
AC_SUBST([SNAPPY_PATCHLEVEL])
AC_SUBST([SNAPPY_LTVERSION], snappy_ltversion)
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([Makefile snappy-stubs-public.h])
AC_OUTPUT

74
src/snappy/snappy/m4/gtest.m4
View File

@@ -1,74 +0,0 @@
dnl GTEST_LIB_CHECK([minimum version [,
dnl action if found [,action if not found]]])
dnl
dnl Check for the presence of the Google Test library, optionally at a minimum
dnl version, and indicate a viable version with the HAVE_GTEST flag. It defines
dnl standard variables for substitution including GTEST_CPPFLAGS,
dnl GTEST_CXXFLAGS, GTEST_LDFLAGS, and GTEST_LIBS. It also defines
dnl GTEST_VERSION as the version of Google Test found. Finally, it provides
dnl optional custom action slots in the event GTEST is found or not.
AC_DEFUN([GTEST_LIB_CHECK],
[
dnl Provide a flag to enable or disable Google Test usage.
AC_ARG_ENABLE([gtest],
[AS_HELP_STRING([--enable-gtest],
[Enable tests using the Google C++ Testing Framework.
(Default is enabled.)])],
[],
[enable_gtest=])
AC_ARG_VAR([GTEST_CONFIG],
[The exact path of Google Test's 'gtest-config' script.])
AC_ARG_VAR([GTEST_CPPFLAGS],
[C-like preprocessor flags for Google Test.])
AC_ARG_VAR([GTEST_CXXFLAGS],
[C++ compile flags for Google Test.])
AC_ARG_VAR([GTEST_LDFLAGS],
[Linker path and option flags for Google Test.])
AC_ARG_VAR([GTEST_LIBS],
[Library linking flags for Google Test.])
AC_ARG_VAR([GTEST_VERSION],
[The version of Google Test available.])
HAVE_GTEST="no"
AS_IF([test "x${enable_gtest}" != "xno"],
[AC_MSG_CHECKING([for 'gtest-config'])
AS_IF([test "x${enable_gtest}" = "xyes"],
[AS_IF([test -x "${enable_gtest}/scripts/gtest-config"],
[GTEST_CONFIG="${enable_gtest}/scripts/gtest-config"],
[GTEST_CONFIG="${enable_gtest}/bin/gtest-config"])
AS_IF([test -x "${GTEST_CONFIG}"], [],
[AC_MSG_RESULT([no])
AC_MSG_ERROR([dnl
Unable to locate either a built or installed Google Test.
The specific location '${enable_gtest}' was provided for a built or installed
Google Test, but no 'gtest-config' script could be found at this location.])
])],
[AC_PATH_PROG([GTEST_CONFIG], [gtest-config])])
AS_IF([test -x "${GTEST_CONFIG}"],
[AC_MSG_RESULT([${GTEST_CONFIG}])
m4_ifval([$1],
[_gtest_min_version="--min-version=$1"
AC_MSG_CHECKING([for Google Test at least version >= $1])],
[_gtest_min_version="--min-version=0"
AC_MSG_CHECKING([for Google Test])])
AS_IF([${GTEST_CONFIG} ${_gtest_min_version}],
[AC_MSG_RESULT([yes])
HAVE_GTEST='yes'],
[AC_MSG_RESULT([no])])],
[AC_MSG_RESULT([no])])
AS_IF([test "x${HAVE_GTEST}" = "xyes"],
[GTEST_CPPFLAGS=`${GTEST_CONFIG} --cppflags`
GTEST_CXXFLAGS=`${GTEST_CONFIG} --cxxflags`
GTEST_LDFLAGS=`${GTEST_CONFIG} --ldflags`
GTEST_LIBS=`${GTEST_CONFIG} --libs`
GTEST_VERSION=`${GTEST_CONFIG} --version`
AC_DEFINE([HAVE_GTEST],[1],[Defined when Google Test is available.])],
[AS_IF([test "x${enable_gtest}" = "xyes"],
[AC_MSG_ERROR([dnl
Google Test was enabled, but no viable version could be found.])
])])])
AC_SUBST([HAVE_GTEST])
AM_CONDITIONAL([HAVE_GTEST],[test "x$HAVE_GTEST" = "xyes"])
AS_IF([test "x$HAVE_GTEST" = "xyes"],
[m4_ifval([$2], [$2])],
[m4_ifval([$3], [$3])])
])

6
src/snappy/snappy/snappy-c.h
View File

@@ -30,8 +30,8 @@
* Plain C interface (a wrapper around the C++ implementation). * Plain C interface (a wrapper around the C++ implementation).
*/ */
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_ #ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_ #define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@@ -135,4 +135,4 @@ snappy_status snappy_validate_compressed_buffer(const char* compressed,
} // extern "C" } // extern "C"
#endif #endif
#endif /* UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_ */ #endif /* THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_ */

128
src/snappy/snappy/snappy-internal.h
View File

@@ -28,8 +28,8 @@
// //
// Internals shared between the Snappy implementation and its unittest. // Internals shared between the Snappy implementation and its unittest.
#ifndef UTIL_SNAPPY_SNAPPY_INTERNAL_H_ #ifndef THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_
#define UTIL_SNAPPY_SNAPPY_INTERNAL_H_ #define THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_
#include "snappy-stubs-internal.h" #include "snappy-stubs-internal.h"
@@ -50,7 +50,9 @@ class WorkingMemory {
uint16 small_table_[1<<10]; // 2KB uint16 small_table_[1<<10]; // 2KB
uint16* large_table_; // Allocated only when needed uint16* large_table_; // Allocated only when needed
DISALLOW_COPY_AND_ASSIGN(WorkingMemory); // No copying
WorkingMemory(const WorkingMemory&);
void operator=(const WorkingMemory&);
}; };
// Flat array compression that does not emit the "uncompressed length" // Flat array compression that does not emit the "uncompressed length"
@@ -70,57 +72,72 @@ char* CompressFragment(const char* input,
uint16* table, uint16* table,
const int table_size); const int table_size);
// Return the largest n such that // Find the largest n such that
// //
// s1[0,n-1] == s2[0,n-1] // s1[0,n-1] == s2[0,n-1]
// and n <= (s2_limit - s2). // and n <= (s2_limit - s2).
// //
// Return make_pair(n, n < 8).
// Does not read *s2_limit or beyond. // Does not read *s2_limit or beyond.
// Does not read *(s1 + (s2_limit - s2)) or beyond. // Does not read *(s1 + (s2_limit - s2)) or beyond.
// Requires that s2_limit >= s2. // Requires that s2_limit >= s2.
// //
// Separate implementation for x86_64, for speed. Uses the fact that // Separate implementation for 64-bit, little-endian cpus.
// x86_64 is little endian. #if !defined(SNAPPY_IS_BIG_ENDIAN) && \
#if defined(ARCH_K8) (defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM))
static inline int FindMatchLength(const char* s1, static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
const char* s2, const char* s2,
const char* s2_limit) { const char* s2_limit) {
assert(s2_limit >= s2); assert(s2_limit >= s2);
int matched = 0; size_t matched = 0;
// This block isn't necessary for correctness; we could just start looping
// immediately. As an optimization though, it is useful. It creates some not
// uncommon code paths that determine, without extra effort, whether the match
// length is less than 8. In short, we are hoping to avoid a conditional
// branch, and perhaps get better code layout from the C++ compiler.
if (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
uint64 a1 = UNALIGNED_LOAD64(s1);
uint64 a2 = UNALIGNED_LOAD64(s2);
if (a1 != a2) {
return std::pair<size_t, bool>(Bits::FindLSBSetNonZero64(a1 ^ a2) >> 3,
true);
} else {
matched = 8;
s2 += 8;
}
}
// Find out how long the match is. We loop over the data 64 bits at a // Find out how long the match is. We loop over the data 64 bits at a
// time until we find a 64-bit block that doesn't match; then we find // time until we find a 64-bit block that doesn't match; then we find
// the first non-matching bit and use that to calculate the total // the first non-matching bit and use that to calculate the total
// length of the match. // length of the match.
while (PREDICT_TRUE(s2 <= s2_limit - 8)) { while (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
if (PREDICT_FALSE(UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched))) { if (UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched)) {
s2 += 8; s2 += 8;
matched += 8; matched += 8;
} else { } else {
// On current (mid-2008) Opteron models there is a 3% more
// efficient code sequence to find the first non-matching byte.
// However, what follows is ~10% better on Intel Core 2 and newer,
// and we expect AMD's bsf instruction to improve.
uint64 x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched); uint64 x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched);
int matching_bits = Bits::FindLSBSetNonZero64(x); int matching_bits = Bits::FindLSBSetNonZero64(x);
matched += matching_bits >> 3; matched += matching_bits >> 3;
return matched; assert(matched >= 8);
return std::pair<size_t, bool>(matched, false);
} }
} }
while (PREDICT_TRUE(s2 < s2_limit)) { while (SNAPPY_PREDICT_TRUE(s2 < s2_limit)) {
if (PREDICT_TRUE(s1[matched] == *s2)) { if (s1[matched] == *s2) {
++s2; ++s2;
++matched; ++matched;
} else { } else {
return matched; return std::pair<size_t, bool>(matched, matched < 8);
} }
} }
return matched; return std::pair<size_t, bool>(matched, matched < 8);
} }
#else #else
static inline int FindMatchLength(const char* s1, static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
const char* s2, const char* s2,
const char* s2_limit) { const char* s2_limit) {
// Implementation based on the x86-64 version, above. // Implementation based on the x86-64 version, above.
assert(s2_limit >= s2); assert(s2_limit >= s2);
int matched = 0; int matched = 0;
@@ -140,11 +157,68 @@ static inline int FindMatchLength(const char* s1,
++matched; ++matched;
} }
} }
return matched; return std::pair<size_t, bool>(matched, matched < 8);
} }
#endif #endif
// Lookup tables for decompression code. Give --snappy_dump_decompression_table
// to the unit test to recompute char_table.
enum {
LITERAL = 0,
COPY_1_BYTE_OFFSET = 1, // 3 bit length + 3 bits of offset in opcode
COPY_2_BYTE_OFFSET = 2,
COPY_4_BYTE_OFFSET = 3
};
static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset.
// Data stored per entry in lookup table:
// Range Bits-used Description
// ------------------------------------
// 1..64 0..7 Literal/copy length encoded in opcode byte
// 0..7 8..10 Copy offset encoded in opcode byte / 256
// 0..4 11..13 Extra bytes after opcode
//
// We use eight bits for the length even though 7 would have sufficed
// because of efficiency reasons:
// (1) Extracting a byte is faster than a bit-field
// (2) It properly aligns copy offset so we do not need a <<8
static const uint16 char_table[256] = {
0x0001, 0x0804, 0x1001, 0x2001, 0x0002, 0x0805, 0x1002, 0x2002,
0x0003, 0x0806, 0x1003, 0x2003, 0x0004, 0x0807, 0x1004, 0x2004,
0x0005, 0x0808, 0x1005, 0x2005, 0x0006, 0x0809, 0x1006, 0x2006,
0x0007, 0x080a, 0x1007, 0x2007, 0x0008, 0x080b, 0x1008, 0x2008,
0x0009, 0x0904, 0x1009, 0x2009, 0x000a, 0x0905, 0x100a, 0x200a,
0x000b, 0x0906, 0x100b, 0x200b, 0x000c, 0x0907, 0x100c, 0x200c,
0x000d, 0x0908, 0x100d, 0x200d, 0x000e, 0x0909, 0x100e, 0x200e,
0x000f, 0x090a, 0x100f, 0x200f, 0x0010, 0x090b, 0x1010, 0x2010,
0x0011, 0x0a04, 0x1011, 0x2011, 0x0012, 0x0a05, 0x1012, 0x2012,
0x0013, 0x0a06, 0x1013, 0x2013, 0x0014, 0x0a07, 0x1014, 0x2014,
0x0015, 0x0a08, 0x1015, 0x2015, 0x0016, 0x0a09, 0x1016, 0x2016,
0x0017, 0x0a0a, 0x1017, 0x2017, 0x0018, 0x0a0b, 0x1018, 0x2018,
0x0019, 0x0b04, 0x1019, 0x2019, 0x001a, 0x0b05, 0x101a, 0x201a,
0x001b, 0x0b06, 0x101b, 0x201b, 0x001c, 0x0b07, 0x101c, 0x201c,
0x001d, 0x0b08, 0x101d, 0x201d, 0x001e, 0x0b09, 0x101e, 0x201e,
0x001f, 0x0b0a, 0x101f, 0x201f, 0x0020, 0x0b0b, 0x1020, 0x2020,
0x0021, 0x0c04, 0x1021, 0x2021, 0x0022, 0x0c05, 0x1022, 0x2022,
0x0023, 0x0c06, 0x1023, 0x2023, 0x0024, 0x0c07, 0x1024, 0x2024,
0x0025, 0x0c08, 0x1025, 0x2025, 0x0026, 0x0c09, 0x1026, 0x2026,
0x0027, 0x0c0a, 0x1027, 0x2027, 0x0028, 0x0c0b, 0x1028, 0x2028,
0x0029, 0x0d04, 0x1029, 0x2029, 0x002a, 0x0d05, 0x102a, 0x202a,
0x002b, 0x0d06, 0x102b, 0x202b, 0x002c, 0x0d07, 0x102c, 0x202c,
0x002d, 0x0d08, 0x102d, 0x202d, 0x002e, 0x0d09, 0x102e, 0x202e,
0x002f, 0x0d0a, 0x102f, 0x202f, 0x0030, 0x0d0b, 0x1030, 0x2030,
0x0031, 0x0e04, 0x1031, 0x2031, 0x0032, 0x0e05, 0x1032, 0x2032,
0x0033, 0x0e06, 0x1033, 0x2033, 0x0034, 0x0e07, 0x1034, 0x2034,
0x0035, 0x0e08, 0x1035, 0x2035, 0x0036, 0x0e09, 0x1036, 0x2036,
0x0037, 0x0e0a, 0x1037, 0x2037, 0x0038, 0x0e0b, 0x1038, 0x2038,
0x0039, 0x0f04, 0x1039, 0x2039, 0x003a, 0x0f05, 0x103a, 0x203a,
0x003b, 0x0f06, 0x103b, 0x203b, 0x003c, 0x0f07, 0x103c, 0x203c,
0x0801, 0x0f08, 0x103d, 0x203d, 0x1001, 0x0f09, 0x103e, 0x203e,
0x1801, 0x0f0a, 0x103f, 0x203f, 0x2001, 0x0f0b, 0x1040, 0x2040
};
} // end namespace internal } // end namespace internal
} // end namespace snappy } // end namespace snappy
#endif // UTIL_SNAPPY_SNAPPY_INTERNAL_H_ #endif // THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_

33
src/snappy/snappy/snappy-sinksource.cc
View File

@@ -40,6 +40,21 @@ char* Sink::GetAppendBuffer(size_t length, char* scratch) {
return scratch; return scratch;
} }
char* Sink::GetAppendBufferVariable(
size_t min_size, size_t desired_size_hint, char* scratch,
size_t scratch_size, size_t* allocated_size) {
*allocated_size = scratch_size;
return scratch;
}
void Sink::AppendAndTakeOwnership(
char* bytes, size_t n,
void (*deleter)(void*, const char*, size_t),
void *deleter_arg) {
Append(bytes, n);
(*deleter)(deleter_arg, bytes, n);
}
ByteArraySource::~ByteArraySource() { } ByteArraySource::~ByteArraySource() { }
size_t ByteArraySource::Available() const { return left_; } size_t ByteArraySource::Available() const { return left_; }
@@ -68,4 +83,22 @@ char* UncheckedByteArraySink::GetAppendBuffer(size_t len, char* scratch) {
return dest_; return dest_;
} }
void UncheckedByteArraySink::AppendAndTakeOwnership(
char* data, size_t n,
void (*deleter)(void*, const char*, size_t),
void *deleter_arg) {
if (data != dest_) {
memcpy(dest_, data, n);
(*deleter)(deleter_arg, data, n);
}
dest_ += n;
} }
char* UncheckedByteArraySink::GetAppendBufferVariable(
size_t min_size, size_t desired_size_hint, char* scratch,
size_t scratch_size, size_t* allocated_size) {
*allocated_size = desired_size_hint;
return dest_;
}
} // namespace snappy

57
src/snappy/snappy/snappy-sinksource.h
View File

@@ -26,12 +26,11 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_ #ifndef THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
#define UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_ #define THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
#include <stddef.h> #include <stddef.h>
namespace snappy { namespace snappy {
// A Sink is an interface that consumes a sequence of bytes. // A Sink is an interface that consumes a sequence of bytes.
@@ -60,6 +59,47 @@ class Sink {
// The default implementation always returns the scratch buffer. // The default implementation always returns the scratch buffer.
virtual char* GetAppendBuffer(size_t length, char* scratch); virtual char* GetAppendBuffer(size_t length, char* scratch);
// For higher performance, Sink implementations can provide custom
// AppendAndTakeOwnership() and GetAppendBufferVariable() methods.
// These methods can reduce the number of copies done during
// compression/decompression.
// Append "bytes[0,n-1] to the sink. Takes ownership of "bytes"
// and calls the deleter function as (*deleter)(deleter_arg, bytes, n)
// to free the buffer. deleter function must be non NULL.
//
// The default implementation just calls Append and frees "bytes".
// Other implementations may avoid a copy while appending the buffer.
virtual void AppendAndTakeOwnership(
char* bytes, size_t n, void (*deleter)(void*, const char*, size_t),
void *deleter_arg);
// Returns a writable buffer for appending and writes the buffer's capacity to
// *allocated_size. Guarantees *allocated_size >= min_size.
// May return a pointer to the caller-owned scratch buffer which must have
// scratch_size >= min_size.
//
// The returned buffer is only valid until the next operation
// on this ByteSink.
//
// After writing at most *allocated_size bytes, call Append() with the
// pointer returned from this function and the number of bytes written.
// Many Append() implementations will avoid copying bytes if this function
// returned an internal buffer.
//
// If the sink implementation allocates or reallocates an internal buffer,
// it should use the desired_size_hint if appropriate. If a caller cannot
// provide a reasonable guess at the desired capacity, it should set
// desired_size_hint = 0.
//
// If a non-scratch buffer is returned, the caller may only pass
// a prefix to it to Append(). That is, it is not correct to pass an
// interior pointer to Append().
//
// The default implementation always returns the scratch buffer.
virtual char* GetAppendBufferVariable(
size_t min_size, size_t desired_size_hint, char* scratch,
size_t scratch_size, size_t* allocated_size);
private: private:
// No copying // No copying
@@ -122,6 +162,12 @@ class UncheckedByteArraySink : public Sink {
virtual ~UncheckedByteArraySink(); virtual ~UncheckedByteArraySink();
virtual void Append(const char* data, size_t n); virtual void Append(const char* data, size_t n);
virtual char* GetAppendBuffer(size_t len, char* scratch); virtual char* GetAppendBuffer(size_t len, char* scratch);
virtual char* GetAppendBufferVariable(
size_t min_size, size_t desired_size_hint, char* scratch,
size_t scratch_size, size_t* allocated_size);
virtual void AppendAndTakeOwnership(
char* bytes, size_t n, void (*deleter)(void*, const char*, size_t),
void *deleter_arg);
// Return the current output pointer so that a caller can see how // Return the current output pointer so that a caller can see how
// many bytes were produced. // many bytes were produced.
@@ -131,7 +177,6 @@ class UncheckedByteArraySink : public Sink {
char* dest_; char* dest_;
}; };
} // namespace snappy
} #endif // THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
#endif // UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_

144
src/snappy/snappy/snappy-stubs-internal.h
View File

@@ -28,8 +28,8 @@
// //
// Various stubs for the open-source version of Snappy. // Various stubs for the open-source version of Snappy.
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ #ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ #define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_
#ifdef HAVE_CONFIG_H #ifdef HAVE_CONFIG_H
#include "config.h" #include "config.h"
@@ -45,6 +45,14 @@
#include <sys/mman.h> #include <sys/mman.h>
#endif #endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#if defined(_MSC_VER)
#include <intrin.h>
#endif // defined(_MSC_VER)
#include "snappy-stubs-public.h" #include "snappy-stubs-public.h"
#if defined(__x86_64__) #if defined(__x86_64__)
@@ -52,6 +60,14 @@
// Enable 64-bit optimized versions of some routines. // Enable 64-bit optimized versions of some routines.
#define ARCH_K8 1 #define ARCH_K8 1
#elif defined(__ppc64__)
#define ARCH_PPC 1
#elif defined(__aarch64__)
#define ARCH_ARM 1
#endif #endif
// Needed by OS X, among others. // Needed by OS X, among others.
@@ -59,10 +75,6 @@
#define MAP_ANONYMOUS MAP_ANON #define MAP_ANONYMOUS MAP_ANON
#endif #endif
// Pull in std::min, std::ostream, and the likes. This is safe because this
// header file is never used from any public header files.
using namespace std;
// The size of an array, if known at compile-time. // The size of an array, if known at compile-time.
// Will give unexpected results if used on a pointer. // Will give unexpected results if used on a pointer.
// We undefine it first, since some compilers already have a definition. // We undefine it first, since some compilers already have a definition.
@@ -73,11 +85,11 @@ using namespace std;
// Static prediction hints. // Static prediction hints.
#ifdef HAVE_BUILTIN_EXPECT #ifdef HAVE_BUILTIN_EXPECT
#define PREDICT_FALSE(x) (__builtin_expect(x, 0)) #define SNAPPY_PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) #define SNAPPY_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
#else #else
#define PREDICT_FALSE(x) x #define SNAPPY_PREDICT_FALSE(x) x
#define PREDICT_TRUE(x) x #define SNAPPY_PREDICT_TRUE(x) x
#endif #endif
// This is only used for recomputing the tag byte table used during // This is only used for recomputing the tag byte table used during
@@ -96,9 +108,10 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
// Potentially unaligned loads and stores. // Potentially unaligned loads and stores.
// x86 and PowerPC can simply do these loads and stores native. // x86, PowerPC, and ARM64 can simply do these loads and stores native.
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) #if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || \
defined(__aarch64__)
#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p)) #define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p))
#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p)) #define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p))
@@ -116,6 +129,15 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
// sub-architectures. // sub-architectures.
// //
// This is a mess, but there's not much we can do about it. // This is a mess, but there's not much we can do about it.
//
// To further complicate matters, only LDR instructions (single reads) are
// allowed to be unaligned, not LDRD (two reads) or LDM (many reads). Unless we
// explicitly tell the compiler that these accesses can be unaligned, it can and
// will combine accesses. On armcc, the way to signal this is done by accessing
// through the type (uint32 __packed *), but GCC has no such attribute
// (it ignores __attribute__((packed)) on individual variables). However,
// we can tell it that a _struct_ is unaligned, which has the same effect,
// so we do that.
#elif defined(__arm__) && \ #elif defined(__arm__) && \
!defined(__ARM_ARCH_4__) && \ !defined(__ARM_ARCH_4__) && \
@@ -131,11 +153,39 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
!defined(__ARM_ARCH_6ZK__) && \ !defined(__ARM_ARCH_6ZK__) && \
!defined(__ARM_ARCH_6T2__) !defined(__ARM_ARCH_6T2__)
#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p)) #if __GNUC__
#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p)) #define ATTRIBUTE_PACKED __attribute__((__packed__))
#else
#define ATTRIBUTE_PACKED
#endif
#define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast<uint16 *>(_p) = (_val)) namespace base {
#define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast<uint32 *>(_p) = (_val)) namespace internal {
struct Unaligned16Struct {
uint16 value;
uint8 dummy; // To make the size non-power-of-two.
} ATTRIBUTE_PACKED;
struct Unaligned32Struct {
uint32 value;
uint8 dummy; // To make the size non-power-of-two.
} ATTRIBUTE_PACKED;
} // namespace internal
} // namespace base
#define UNALIGNED_LOAD16(_p) \
((reinterpret_cast<const ::snappy::base::internal::Unaligned16Struct *>(_p))->value)
#define UNALIGNED_LOAD32(_p) \
((reinterpret_cast<const ::snappy::base::internal::Unaligned32Struct *>(_p))->value)
#define UNALIGNED_STORE16(_p, _val) \
((reinterpret_cast< ::snappy::base::internal::Unaligned16Struct *>(_p))->value = \
(_val))
#define UNALIGNED_STORE32(_p, _val) \
((reinterpret_cast< ::snappy::base::internal::Unaligned32Struct *>(_p))->value = \
(_val))
// TODO(user): NEON supports unaligned 64-bit loads and stores. // TODO(user): NEON supports unaligned 64-bit loads and stores.
// See if that would be more efficient on platforms supporting it, // See if that would be more efficient on platforms supporting it,
@@ -188,22 +238,8 @@ inline void UNALIGNED_STORE64(void *p, uint64 v) {
#endif #endif
// This can be more efficient than UNALIGNED_LOAD64 + UNALIGNED_STORE64
// on some platforms, in particular ARM.
inline void UnalignedCopy64(const void *src, void *dst) {
if (sizeof(void *) == 8) {
UNALIGNED_STORE64(dst, UNALIGNED_LOAD64(src));
} else {
const char *src_char = reinterpret_cast<const char *>(src);
char *dst_char = reinterpret_cast<char *>(dst);
UNALIGNED_STORE32(dst_char, UNALIGNED_LOAD32(src_char));
UNALIGNED_STORE32(dst_char + 4, UNALIGNED_LOAD32(src_char + 4));
}
}
// The following guarantees declaration of the byte swap functions. // The following guarantees declaration of the byte swap functions.
#ifdef WORDS_BIGENDIAN #if defined(SNAPPY_IS_BIG_ENDIAN)
#ifdef HAVE_SYS_BYTEORDER_H #ifdef HAVE_SYS_BYTEORDER_H
#include <sys/byteorder.h> #include <sys/byteorder.h>
@@ -260,7 +296,7 @@ inline uint64 bswap_64(uint64 x) {
#endif #endif
#endif // WORDS_BIGENDIAN #endif // defined(SNAPPY_IS_BIG_ENDIAN)
// Convert to little-endian storage, opposite of network format. // Convert to little-endian storage, opposite of network format.
// Convert x from host to little endian: x = LittleEndian.FromHost(x); // Convert x from host to little endian: x = LittleEndian.FromHost(x);
@@ -274,7 +310,7 @@ inline uint64 bswap_64(uint64 x) {
class LittleEndian { class LittleEndian {
public: public:
// Conversion functions. // Conversion functions.
#ifdef WORDS_BIGENDIAN #if defined(SNAPPY_IS_BIG_ENDIAN)
static uint16 FromHost16(uint16 x) { return bswap_16(x); } static uint16 FromHost16(uint16 x) { return bswap_16(x); }
static uint16 ToHost16(uint16 x) { return bswap_16(x); } static uint16 ToHost16(uint16 x) { return bswap_16(x); }
@@ -284,7 +320,7 @@ class LittleEndian {
static bool IsLittleEndian() { return false; } static bool IsLittleEndian() { return false; }
#else // !defined(WORDS_BIGENDIAN) #else // !defined(SNAPPY_IS_BIG_ENDIAN)
static uint16 FromHost16(uint16 x) { return x; } static uint16 FromHost16(uint16 x) { return x; }
static uint16 ToHost16(uint16 x) { return x; } static uint16 ToHost16(uint16 x) { return x; }
@@ -294,7 +330,7 @@ class LittleEndian {
static bool IsLittleEndian() { return true; } static bool IsLittleEndian() { return true; }
#endif // !defined(WORDS_BIGENDIAN) #endif // !defined(SNAPPY_IS_BIG_ENDIAN)
// Functions to do unaligned loads and stores in little-endian order. // Functions to do unaligned loads and stores in little-endian order.
static uint16 Load16(const void *p) { static uint16 Load16(const void *p) {
@@ -324,10 +360,15 @@ class Bits {
// undefined value if n == 0. FindLSBSetNonZero() is similar to ffs() except // undefined value if n == 0. FindLSBSetNonZero() is similar to ffs() except
// that it's 0-indexed. // that it's 0-indexed.
static int FindLSBSetNonZero(uint32 n); static int FindLSBSetNonZero(uint32 n);
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
static int FindLSBSetNonZero64(uint64 n); static int FindLSBSetNonZero64(uint64 n);
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
private: private:
DISALLOW_COPY_AND_ASSIGN(Bits); // No copying
Bits(const Bits&);
void operator=(const Bits&);
}; };
#ifdef HAVE_BUILTIN_CTZ #ifdef HAVE_BUILTIN_CTZ
@@ -340,9 +381,36 @@ inline int Bits::FindLSBSetNonZero(uint32 n) {
return __builtin_ctz(n); return __builtin_ctz(n);
} }
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
inline int Bits::FindLSBSetNonZero64(uint64 n) { inline int Bits::FindLSBSetNonZero64(uint64 n) {
return __builtin_ctzll(n); return __builtin_ctzll(n);
} }
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
#elif defined(_MSC_VER)
inline int Bits::Log2Floor(uint32 n) {
unsigned long where;
if (_BitScanReverse(&where, n)) {
return where;
} else {
return -1;
}
}
inline int Bits::FindLSBSetNonZero(uint32 n) {
unsigned long where;
if (_BitScanForward(&where, n)) return static_cast<int>(where);
return 32;
}
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
inline int Bits::FindLSBSetNonZero64(uint64 n) {
unsigned long where;
if (_BitScanForward64(&where, n)) return static_cast<int>(where);
return 64;
}
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
#else // Portable versions. #else // Portable versions.
@@ -376,6 +444,7 @@ inline int Bits::FindLSBSetNonZero(uint32 n) {
return rc; return rc;
} }
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
// FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero(). // FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero().
inline int Bits::FindLSBSetNonZero64(uint64 n) { inline int Bits::FindLSBSetNonZero64(uint64 n) {
const uint32 bottombits = static_cast<uint32>(n); const uint32 bottombits = static_cast<uint32>(n);
@@ -386,6 +455,7 @@ inline int Bits::FindLSBSetNonZero64(uint64 n) {
return FindLSBSetNonZero(bottombits); return FindLSBSetNonZero(bottombits);
} }
} }
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
#endif // End portable versions. #endif // End portable versions.
@@ -488,4 +558,4 @@ inline char* string_as_array(string* str) {
} // namespace snappy } // namespace snappy
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ #endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_

36
src/snappy/snappy/snappy-stubs-public.h.in
View File

@@ -33,24 +33,24 @@
// which is a public header. Instead, snappy-stubs-public.h is generated by // which is a public header. Instead, snappy-stubs-public.h is generated by
// from snappy-stubs-public.h.in at configure time. // from snappy-stubs-public.h.in at configure time.
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ #ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ #define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
#if @ac_cv_have_stdint_h@ #if ${HAVE_STDINT_H_01} // HAVE_STDINT_H
#include <stdint.h> #include <stdint.h>
#endif #endif // HAVE_STDDEF_H
#if @ac_cv_have_stddef_h@ #if ${HAVE_STDDEF_H_01} // HAVE_STDDEF_H
#include <stddef.h> #include <stddef.h>
#endif #endif // HAVE_STDDEF_H
#if @ac_cv_have_sys_uio_h@ #if ${HAVE_SYS_UIO_H_01} // HAVE_SYS_UIO_H
#include <sys/uio.h> #include <sys/uio.h>
#endif #endif // HAVE_SYS_UIO_H
#define SNAPPY_MAJOR @SNAPPY_MAJOR@ #define SNAPPY_MAJOR ${SNAPPY_MAJOR}
#define SNAPPY_MINOR @SNAPPY_MINOR@ #define SNAPPY_MINOR ${SNAPPY_MINOR}
#define SNAPPY_PATCHLEVEL @SNAPPY_PATCHLEVEL@ #define SNAPPY_PATCHLEVEL ${SNAPPY_PATCHLEVEL}
#define SNAPPY_VERSION \ #define SNAPPY_VERSION \
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL) ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
@@ -58,7 +58,7 @@
namespace snappy { namespace snappy {
#if @ac_cv_have_stdint_h@ #if ${HAVE_STDINT_H_01} // HAVE_STDINT_H
typedef int8_t int8; typedef int8_t int8;
typedef uint8_t uint8; typedef uint8_t uint8;
typedef int16_t int16; typedef int16_t int16;
@@ -76,23 +76,19 @@ typedef int int32;
typedef unsigned int uint32; typedef unsigned int uint32;
typedef long long int64; typedef long long int64;
typedef unsigned long long uint64; typedef unsigned long long uint64;
#endif #endif // HAVE_STDINT_H
typedef std::string string; typedef std::string string;
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ #if !${HAVE_SYS_UIO_H_01} // !HAVE_SYS_UIO_H
TypeName(const TypeName&); \
void operator=(const TypeName&)
#if !@ac_cv_have_sys_uio_h@
// Windows does not have an iovec type, yet the concept is universally useful. // Windows does not have an iovec type, yet the concept is universally useful.
// It is simple to define it ourselves, so we put it inside our own namespace. // It is simple to define it ourselves, so we put it inside our own namespace.
struct iovec { struct iovec {
void* iov_base; void* iov_base;
size_t iov_len; size_t iov_len;
}; };
#endif #endif // !HAVE_SYS_UIO_H
} // namespace snappy } // namespace snappy
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ #endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_

24
src/snappy/snappy/snappy-test.cc
View File

@@ -28,13 +28,19 @@
// //
// Various stubs for the unit tests for the open-source version of Snappy. // Various stubs for the unit tests for the open-source version of Snappy.
#include "snappy-test.h" #ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#ifdef HAVE_WINDOWS_H #ifdef HAVE_WINDOWS_H
#define WIN32_LEAN_AND_MEAN // Needed to be able to use std::max without workarounds in the source code.
// https://support.microsoft.com/en-us/help/143208/prb-using-stl-in-windows-program-can-cause-min-max-conflicts
#define NOMINMAX
#include <windows.h> #include <windows.h>
#endif #endif
#include "snappy-test.h"
#include <algorithm> #include <algorithm>
DEFINE_bool(run_microbenchmarks, true, DEFINE_bool(run_microbenchmarks, true,
@@ -198,7 +204,7 @@ void Benchmark::Run() {
if (benchmark_real_time_us > 0) { if (benchmark_real_time_us > 0) {
num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us; num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us;
} }
num_iterations = max(num_iterations, kCalibrateIterations); num_iterations = std::max(num_iterations, kCalibrateIterations);
BenchmarkRun benchmark_runs[kNumRuns]; BenchmarkRun benchmark_runs[kNumRuns];
for (int run = 0; run < kNumRuns; ++run) { for (int run = 0; run < kNumRuns; ++run) {
@@ -214,10 +220,10 @@ void Benchmark::Run() {
string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num); string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num);
string human_readable_speed; string human_readable_speed;
nth_element(benchmark_runs, std::nth_element(benchmark_runs,
benchmark_runs + kMedianPos, benchmark_runs + kMedianPos,
benchmark_runs + kNumRuns, benchmark_runs + kNumRuns,
BenchmarkCompareCPUTime()); BenchmarkCompareCPUTime());
int64 real_time_us = benchmark_runs[kMedianPos].real_time_us; int64 real_time_us = benchmark_runs[kMedianPos].real_time_us;
int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us; int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us;
if (cpu_time_us <= 0) { if (cpu_time_us <= 0) {
@@ -520,8 +526,8 @@ int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen,
LOG(WARNING) LOG(WARNING)
<< "UncompressChunkOrAll: Received some extra data, bytes total: " << "UncompressChunkOrAll: Received some extra data, bytes total: "
<< uncomp_stream_.avail_in << " bytes: " << uncomp_stream_.avail_in << " bytes: "
<< string(reinterpret_cast<const char *>(uncomp_stream_.next_in), << std::string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
min(int(uncomp_stream_.avail_in), 20)); std::min(int(uncomp_stream_.avail_in), 20));
UncompressErrorInit(); UncompressErrorInit();
return Z_DATA_ERROR; // what's the extra data for? return Z_DATA_ERROR; // what's the extra data for?
} else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) { } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {

77
src/snappy/snappy/snappy-test.h
View File

@@ -28,8 +28,8 @@
// //
// Various stubs for the unit tests for the open-source version of Snappy. // Various stubs for the unit tests for the open-source version of Snappy.
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_ #ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_ #define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
#include <iostream> #include <iostream>
#include <string> #include <string>
@@ -52,7 +52,6 @@
#endif #endif
#ifdef HAVE_WINDOWS_H #ifdef HAVE_WINDOWS_H
#define WIN32_LEAN_AND_MEAN
#include <windows.h> #include <windows.h>
#endif #endif
@@ -111,35 +110,18 @@
#include "lzo/lzo1x.h" #include "lzo/lzo1x.h"
#endif #endif
#ifdef HAVE_LIBLZF
extern "C" {
#include "lzf.h"
}
#endif
#ifdef HAVE_LIBFASTLZ
#include "fastlz.h"
#endif
#ifdef HAVE_LIBQUICKLZ
#include "quicklz.h"
#endif
namespace { namespace {
namespace File {
void Init() { }
} // namespace File
namespace file { namespace file {
int Defaults() { } int Defaults() { return 0; }
class DummyStatus { class DummyStatus {
public: public:
void CheckSuccess() { } void CheckSuccess() { }
}; };
DummyStatus GetContents(const string& filename, string* data, int unused) { DummyStatus GetContents(
const std::string& filename, std::string* data, int unused) {
FILE* fp = fopen(filename.c_str(), "rb"); FILE* fp = fopen(filename.c_str(), "rb");
if (fp == NULL) { if (fp == NULL) {
perror(filename.c_str()); perror(filename.c_str());
@@ -154,15 +136,16 @@ namespace file {
perror("fread"); perror("fread");
exit(1); exit(1);
} }
data->append(string(buf, ret)); data->append(std::string(buf, ret));
} }
fclose(fp); fclose(fp);
return DummyStatus();
} }
DummyStatus SetContents(const string& filename, inline DummyStatus SetContents(
const string& str, const std::string& filename, const std::string& str, int unused) {
int unused) {
FILE* fp = fopen(filename.c_str(), "wb"); FILE* fp = fopen(filename.c_str(), "wb");
if (fp == NULL) { if (fp == NULL) {
perror(filename.c_str()); perror(filename.c_str());
@@ -176,6 +159,8 @@ namespace file {
} }
fclose(fp); fclose(fp);
return DummyStatus();
} }
} // namespace file } // namespace file
@@ -193,6 +178,7 @@ void Test_Snappy_RandomData();
void Test_Snappy_FourByteOffset(); void Test_Snappy_FourByteOffset();
void Test_SnappyCorruption_TruncatedVarint(); void Test_SnappyCorruption_TruncatedVarint();
void Test_SnappyCorruption_UnterminatedVarint(); void Test_SnappyCorruption_UnterminatedVarint();
void Test_SnappyCorruption_OverflowingVarint();
void Test_Snappy_ReadPastEndOfBuffer(); void Test_Snappy_ReadPastEndOfBuffer();
void Test_Snappy_FindMatchLength(); void Test_Snappy_FindMatchLength();
void Test_Snappy_FindMatchLengthRandom(); void Test_Snappy_FindMatchLengthRandom();
@@ -463,7 +449,7 @@ class ZLib {
DECLARE_bool(run_microbenchmarks); DECLARE_bool(run_microbenchmarks);
static void RunSpecifiedBenchmarks() { static inline void RunSpecifiedBenchmarks() {
if (!FLAGS_run_microbenchmarks) { if (!FLAGS_run_microbenchmarks) {
return; return;
} }
@@ -497,6 +483,7 @@ static inline int RUN_ALL_TESTS() {
snappy::Test_Snappy_FourByteOffset(); snappy::Test_Snappy_FourByteOffset();
snappy::Test_SnappyCorruption_TruncatedVarint(); snappy::Test_SnappyCorruption_TruncatedVarint();
snappy::Test_SnappyCorruption_UnterminatedVarint(); snappy::Test_SnappyCorruption_UnterminatedVarint();
snappy::Test_SnappyCorruption_OverflowingVarint();
snappy::Test_Snappy_ReadPastEndOfBuffer(); snappy::Test_Snappy_ReadPastEndOfBuffer();
snappy::Test_Snappy_FindMatchLength(); snappy::Test_Snappy_FindMatchLength();
snappy::Test_Snappy_FindMatchLengthRandom(); snappy::Test_Snappy_FindMatchLengthRandom();
@@ -510,10 +497,6 @@ static inline int RUN_ALL_TESTS() {
// For main(). // For main().
namespace snappy { namespace snappy {
static void CompressFile(const char* fname);
static void UncompressFile(const char* fname);
static void MeasureFile(const char* fname);
// Logging. // Logging.
#define LOG(level) LogMessage() #define LOG(level) LogMessage()
@@ -524,15 +507,15 @@ class LogMessage {
public: public:
LogMessage() { } LogMessage() { }
~LogMessage() { ~LogMessage() {
cerr << endl; std::cerr << std::endl;
} }
LogMessage& operator<<(const std::string& msg) { LogMessage& operator<<(const std::string& msg) {
cerr << msg; std::cerr << msg;
return *this; return *this;
} }
LogMessage& operator<<(int x) { LogMessage& operator<<(int x) {
cerr << x; std::cerr << x;
return *this; return *this;
} }
}; };
@@ -541,18 +524,29 @@ class LogMessage {
// and ones that are always active. // and ones that are always active.
#define CRASH_UNLESS(condition) \ #define CRASH_UNLESS(condition) \
PREDICT_TRUE(condition) ? (void)0 : \ SNAPPY_PREDICT_TRUE(condition) ? (void)0 : \
snappy::LogMessageVoidify() & snappy::LogMessageCrash() snappy::LogMessageVoidify() & snappy::LogMessageCrash()
#ifdef _MSC_VER
// ~LogMessageCrash calls abort() and therefore never exits. This is by design
// so temporarily disable warning C4722.
#pragma warning(push)
#pragma warning(disable:4722)
#endif
class LogMessageCrash : public LogMessage { class LogMessageCrash : public LogMessage {
public: public:
LogMessageCrash() { } LogMessageCrash() { }
~LogMessageCrash() { ~LogMessageCrash() {
cerr << endl; std::cerr << std::endl;
abort(); abort();
} }
}; };
#ifdef _MSC_VER
#pragma warning(pop)
#endif
// This class is used to explicitly ignore values in the conditional // This class is used to explicitly ignore values in the conditional
// logging macros. This avoids compiler warnings like "value computed // logging macros. This avoids compiler warnings like "value computed
// is not used" and "statement has no effect". // is not used" and "statement has no effect".
@@ -572,11 +566,8 @@ class LogMessageVoidify {
#define CHECK_NE(a, b) CRASH_UNLESS((a) != (b)) #define CHECK_NE(a, b) CRASH_UNLESS((a) != (b))
#define CHECK_LT(a, b) CRASH_UNLESS((a) < (b)) #define CHECK_LT(a, b) CRASH_UNLESS((a) < (b))
#define CHECK_GT(a, b) CRASH_UNLESS((a) > (b)) #define CHECK_GT(a, b) CRASH_UNLESS((a) > (b))
#define CHECK_OK(cond) (cond).CheckSuccess()
} // namespace } // namespace snappy
using snappy::CompressFile; #endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
using snappy::UncompressFile;
using snappy::MeasureFile;
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_

855
src/snappy/snappy/snappy.cc
View File

File diff suppressed because it is too large Load Diff

27
src/snappy/snappy/snappy.h
View File

@@ -36,8 +36,8 @@
// using BMDiff and then compressing the output of BMDiff with // using BMDiff and then compressing the output of BMDiff with
// Snappy. // Snappy.
#ifndef UTIL_SNAPPY_SNAPPY_H__ #ifndef THIRD_PARTY_SNAPPY_SNAPPY_H__
#define UTIL_SNAPPY_SNAPPY_H__ #define THIRD_PARTY_SNAPPY_SNAPPY_H__
#include <stddef.h> #include <stddef.h>
#include <string> #include <string>
@@ -84,6 +84,18 @@ namespace snappy {
bool Uncompress(const char* compressed, size_t compressed_length, bool Uncompress(const char* compressed, size_t compressed_length,
string* uncompressed); string* uncompressed);
// Decompresses "compressed" to "*uncompressed".
//
// returns false if the message is corrupted and could not be decompressed
bool Uncompress(Source* compressed, Sink* uncompressed);
// This routine uncompresses as much of the "compressed" as possible
// into sink. It returns the number of valid bytes added to sink
// (extra invalid bytes may have been added due to errors; the caller
// should ignore those). The emitted data typically has length
// GetUncompressedLength(), but may be shorter if an error is
// encountered.
size_t UncompressAsMuchAsPossible(Source* compressed, Sink* uncompressed);
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Lower-level character array based routines. May be useful for // Lower-level character array based routines. May be useful for
@@ -164,6 +176,14 @@ namespace snappy {
bool IsValidCompressedBuffer(const char* compressed, bool IsValidCompressedBuffer(const char* compressed,
size_t compressed_length); size_t compressed_length);
// Returns true iff the contents of "compressed" can be uncompressed
// successfully. Does not return the uncompressed data. Takes
// time proportional to *compressed length, but is usually at least
// a factor of four faster than actual decompression.
// On success, consumes all of *compressed. On failure, consumes an
// unspecified prefix of *compressed.
bool IsValidCompressed(Source* compressed);
// The size of a compression block. Note that many parts of the compression // The size of a compression block. Note that many parts of the compression
// code assumes that kBlockSize <= 65536; in particular, the hash table // code assumes that kBlockSize <= 65536; in particular, the hash table
// can only store 16-bit offsets, and EmitCopy() also assumes the offset // can only store 16-bit offsets, and EmitCopy() also assumes the offset
@@ -180,5 +200,4 @@ namespace snappy {
static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits; static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
} // end namespace snappy } // end namespace snappy
#endif // THIRD_PARTY_SNAPPY_SNAPPY_H__
#endif // UTIL_SNAPPY_SNAPPY_H__

425
src/snappy/snappy/snappy_unittest.cc
View File

@@ -32,6 +32,7 @@
#include <algorithm> #include <algorithm>
#include <string> #include <string>
#include <utility>
#include <vector> #include <vector>
#include "snappy.h" #include "snappy.h"
@@ -50,25 +51,19 @@ DEFINE_bool(zlib, false,
"Run zlib compression (http://www.zlib.net)"); "Run zlib compression (http://www.zlib.net)");
DEFINE_bool(lzo, false, DEFINE_bool(lzo, false,
"Run LZO compression (http://www.oberhumer.com/opensource/lzo/)"); "Run LZO compression (http://www.oberhumer.com/opensource/lzo/)");
DEFINE_bool(quicklz, false,
"Run quickLZ compression (http://www.quicklz.com/)");
DEFINE_bool(liblzf, false,
"Run libLZF compression "
"(http://www.goof.com/pcg/marc/liblzf.html)");
DEFINE_bool(fastlz, false,
"Run FastLZ compression (http://www.fastlz.org/");
DEFINE_bool(snappy, true, "Run snappy compression"); DEFINE_bool(snappy, true, "Run snappy compression");
DEFINE_bool(write_compressed, false, DEFINE_bool(write_compressed, false,
"Write compressed versions of each file to <file>.comp"); "Write compressed versions of each file to <file>.comp");
DEFINE_bool(write_uncompressed, false, DEFINE_bool(write_uncompressed, false,
"Write uncompressed versions of each file to <file>.uncomp"); "Write uncompressed versions of each file to <file>.uncomp");
DEFINE_bool(snappy_dump_decompression_table, false,
"If true, we print the decompression table during tests.");
namespace snappy { namespace snappy {
#if defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF)
#ifdef HAVE_FUNC_MMAP
// To test against code that reads beyond its input, this class copies a // To test against code that reads beyond its input, this class copies a
// string to a newly allocated group of pages, the last of which // string to a newly allocated group of pages, the last of which
@@ -79,7 +74,7 @@ namespace snappy {
class DataEndingAtUnreadablePage { class DataEndingAtUnreadablePage {
public: public:
explicit DataEndingAtUnreadablePage(const string& s) { explicit DataEndingAtUnreadablePage(const string& s) {
const size_t page_size = getpagesize(); const size_t page_size = sysconf(_SC_PAGESIZE);
const size_t size = s.size(); const size_t size = s.size();
// Round up space for string to a multiple of page_size. // Round up space for string to a multiple of page_size.
size_t space_for_string = (size + page_size - 1) & ~(page_size - 1); size_t space_for_string = (size + page_size - 1) & ~(page_size - 1);
@@ -97,8 +92,9 @@ class DataEndingAtUnreadablePage {
} }
~DataEndingAtUnreadablePage() { ~DataEndingAtUnreadablePage() {
const size_t page_size = sysconf(_SC_PAGESIZE);
// Undo the mprotect. // Undo the mprotect.
CHECK_EQ(0, mprotect(protected_page_, getpagesize(), PROT_READ|PROT_WRITE)); CHECK_EQ(0, mprotect(protected_page_, page_size, PROT_READ|PROT_WRITE));
CHECK_EQ(0, munmap(mem_, alloc_size_)); CHECK_EQ(0, munmap(mem_, alloc_size_));
} }
@@ -113,7 +109,7 @@ class DataEndingAtUnreadablePage {
size_t size_; size_t size_;
}; };
#else // HAVE_FUNC_MMAP #else // defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF)
// Fallback for systems without mmap. // Fallback for systems without mmap.
typedef string DataEndingAtUnreadablePage; typedef string DataEndingAtUnreadablePage;
@@ -121,11 +117,11 @@ typedef string DataEndingAtUnreadablePage;
#endif #endif
enum CompressorType { enum CompressorType {
ZLIB, LZO, LIBLZF, QUICKLZ, FASTLZ, SNAPPY ZLIB, LZO, SNAPPY
}; };
const char* names[] = { const char* names[] = {
"ZLIB", "LZO", "LIBLZF", "QUICKLZ", "FASTLZ", "SNAPPY" "ZLIB", "LZO", "SNAPPY"
}; };
static size_t MinimumRequiredOutputSpace(size_t input_size, static size_t MinimumRequiredOutputSpace(size_t input_size,
@@ -141,26 +137,12 @@ static size_t MinimumRequiredOutputSpace(size_t input_size,
return input_size + input_size/64 + 16 + 3; return input_size + input_size/64 + 16 + 3;
#endif // LZO_VERSION #endif // LZO_VERSION
#ifdef LZF_VERSION
case LIBLZF:
return input_size;
#endif // LZF_VERSION
#ifdef QLZ_VERSION_MAJOR
case QUICKLZ:
return input_size + 36000; // 36000 is used for scratch.
#endif // QLZ_VERSION_MAJOR
#ifdef FASTLZ_VERSION
case FASTLZ:
return max(static_cast<int>(ceil(input_size * 1.05)), 66);
#endif // FASTLZ_VERSION
case SNAPPY: case SNAPPY:
return snappy::MaxCompressedLength(input_size); return snappy::MaxCompressedLength(input_size);
default: default:
LOG(FATAL) << "Unknown compression type number " << comp; LOG(FATAL) << "Unknown compression type number " << comp;
return 0;
} }
} }
@@ -214,58 +196,6 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
} }
#endif // LZO_VERSION #endif // LZO_VERSION
#ifdef LZF_VERSION
case LIBLZF: {
int destlen = lzf_compress(input,
input_size,
string_as_array(compressed),
input_size);
if (destlen == 0) {
// lzf *can* cause lots of blowup when compressing, so they
// recommend to limit outsize to insize, and just not compress
// if it's bigger. Ideally, we'd just swap input and output.
compressed->assign(input, input_size);
destlen = input_size;
}
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
break;
}
#endif // LZF_VERSION
#ifdef QLZ_VERSION_MAJOR
case QUICKLZ: {
qlz_state_compress *state_compress = new qlz_state_compress;
int destlen = qlz_compress(input,
string_as_array(compressed),
input_size,
state_compress);
delete state_compress;
CHECK_NE(0, destlen);
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
break;
}
#endif // QLZ_VERSION_MAJOR
#ifdef FASTLZ_VERSION
case FASTLZ: {
// Use level 1 compression since we mostly care about speed.
int destlen = fastlz_compress_level(
1,
input,
input_size,
string_as_array(compressed));
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
CHECK_NE(destlen, 0);
break;
}
#endif // FASTLZ_VERSION
case SNAPPY: { case SNAPPY: {
size_t destlen; size_t destlen;
snappy::RawCompress(input, input_size, snappy::RawCompress(input, input_size,
@@ -278,7 +208,6 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
break; break;
} }
default: { default: {
return false; // the asked-for library wasn't compiled in return false; // the asked-for library wasn't compiled in
} }
@@ -321,56 +250,12 @@ static bool Uncompress(const string& compressed, CompressorType comp,
} }
#endif // LZO_VERSION #endif // LZO_VERSION
#ifdef LZF_VERSION
case LIBLZF: {
output->resize(size);
int destlen = lzf_decompress(compressed.data(),
compressed.size(),
string_as_array(output),
output->size());
if (destlen == 0) {
// This error probably means we had decided not to compress,
// and thus have stored input in output directly.
output->assign(compressed.data(), compressed.size());
destlen = compressed.size();
}
CHECK_EQ(destlen, size);
break;
}
#endif // LZF_VERSION
#ifdef QLZ_VERSION_MAJOR
case QUICKLZ: {
output->resize(size);
qlz_state_decompress *state_decompress = new qlz_state_decompress;
int destlen = qlz_decompress(compressed.data(),
string_as_array(output),
state_decompress);
delete state_decompress;
CHECK_EQ(destlen, size);
break;
}
#endif // QLZ_VERSION_MAJOR
#ifdef FASTLZ_VERSION
case FASTLZ: {
output->resize(size);
int destlen = fastlz_decompress(compressed.data(),
compressed.length(),
string_as_array(output),
size);
CHECK_EQ(destlen, size);
break;
}
#endif // FASTLZ_VERSION
case SNAPPY: { case SNAPPY: {
snappy::RawUncompress(compressed.data(), compressed.size(), snappy::RawUncompress(compressed.data(), compressed.size(),
string_as_array(output)); string_as_array(output));
break; break;
} }
default: { default: {
return false; // the asked-for library wasn't compiled in return false; // the asked-for library wasn't compiled in
} }
@@ -392,13 +277,13 @@ static void Measure(const char* data,
{ {
// Chop the input into blocks // Chop the input into blocks
int num_blocks = (length + block_size - 1) / block_size; int num_blocks = (length + block_size - 1) / block_size;
vector<const char*> input(num_blocks); std::vector<const char*> input(num_blocks);
vector<size_t> input_length(num_blocks); std::vector<size_t> input_length(num_blocks);
vector<string> compressed(num_blocks); std::vector<string> compressed(num_blocks);
vector<string> output(num_blocks); std::vector<string> output(num_blocks);
for (int b = 0; b < num_blocks; b++) { for (int b = 0; b < num_blocks; b++) {
int input_start = b * block_size; int input_start = b * block_size;
int input_limit = min<int>((b+1)*block_size, length); int input_limit = std::min<int>((b+1)*block_size, length);
input[b] = data+input_start; input[b] = data+input_start;
input_length[b] = input_limit-input_start; input_length[b] = input_limit-input_start;
@@ -448,13 +333,13 @@ static void Measure(const char* data,
} }
compressed_size = 0; compressed_size = 0;
for (int i = 0; i < compressed.size(); i++) { for (size_t i = 0; i < compressed.size(); i++) {
compressed_size += compressed[i].size(); compressed_size += compressed[i].size();
} }
} }
sort(ctime, ctime + kRuns); std::sort(ctime, ctime + kRuns);
sort(utime, utime + kRuns); std::sort(utime, utime + kRuns);
const int med = kRuns/2; const int med = kRuns/2;
float comp_rate = (length / ctime[med]) * repeats / 1048576.0; float comp_rate = (length / ctime[med]) * repeats / 1048576.0;
@@ -469,12 +354,11 @@ static void Measure(const char* data,
x.c_str(), x.c_str(),
block_size/(1<<20), block_size/(1<<20),
static_cast<int>(length), static_cast<uint32>(compressed_size), static_cast<int>(length), static_cast<uint32>(compressed_size),
(compressed_size * 100.0) / max<int>(1, length), (compressed_size * 100.0) / std::max<int>(1, length),
comp_rate, comp_rate,
urate.c_str()); urate.c_str());
} }
static int VerifyString(const string& input) { static int VerifyString(const string& input) {
string compressed; string compressed;
DataEndingAtUnreadablePage i(input); DataEndingAtUnreadablePage i(input);
@@ -491,6 +375,23 @@ static int VerifyString(const string& input) {
return uncompressed.size(); return uncompressed.size();
} }
static void VerifyStringSink(const string& input) {
string compressed;
DataEndingAtUnreadablePage i(input);
const size_t written = snappy::Compress(i.data(), i.size(), &compressed);
CHECK_EQ(written, compressed.size());
CHECK_LE(compressed.size(),
snappy::MaxCompressedLength(input.size()));
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
string uncompressed;
uncompressed.resize(input.size());
snappy::UncheckedByteArraySink sink(string_as_array(&uncompressed));
DataEndingAtUnreadablePage c(compressed);
snappy::ByteArraySource source(c.data(), c.size());
CHECK(snappy::Uncompress(&source, &sink));
CHECK_EQ(uncompressed, input);
}
static void VerifyIOVec(const string& input) { static void VerifyIOVec(const string& input) {
string compressed; string compressed;
@@ -505,13 +406,13 @@ static void VerifyIOVec(const string& input) {
// ranging from 1 to 10. // ranging from 1 to 10.
char* buf = new char[input.size()]; char* buf = new char[input.size()];
ACMRandom rnd(input.size()); ACMRandom rnd(input.size());
int num = rnd.Next() % 10 + 1; size_t num = rnd.Next() % 10 + 1;
if (input.size() < num) { if (input.size() < num) {
num = input.size(); num = input.size();
} }
struct iovec* iov = new iovec[num]; struct iovec* iov = new iovec[num];
int used_so_far = 0; int used_so_far = 0;
for (int i = 0; i < num; ++i) { for (size_t i = 0; i < num; ++i) {
iov[i].iov_base = buf + used_so_far; iov[i].iov_base = buf + used_so_far;
if (i == num - 1) { if (i == num - 1) {
iov[i].iov_len = input.size() - used_so_far; iov[i].iov_len = input.size() - used_so_far;
@@ -562,6 +463,28 @@ static void VerifyNonBlockedCompression(const string& input) {
CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncomp_str)); CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncomp_str));
CHECK_EQ(uncomp_str, input); CHECK_EQ(uncomp_str, input);
// Uncompress using source/sink
string uncomp_str2;
uncomp_str2.resize(input.size());
snappy::UncheckedByteArraySink sink(string_as_array(&uncomp_str2));
snappy::ByteArraySource source(compressed.data(), compressed.size());
CHECK(snappy::Uncompress(&source, &sink));
CHECK_EQ(uncomp_str2, input);
// Uncompress into iovec
{
static const int kNumBlocks = 10;
struct iovec vec[kNumBlocks];
const int block_size = 1 + input.size() / kNumBlocks;
string iovec_data(block_size * kNumBlocks, 'x');
for (int i = 0; i < kNumBlocks; i++) {
vec[i].iov_base = string_as_array(&iovec_data) + i * block_size;
vec[i].iov_len = block_size;
}
CHECK(snappy::RawUncompressToIOVec(compressed.data(), compressed.size(),
vec, kNumBlocks));
CHECK_EQ(string(iovec_data.data(), input.size()), input);
}
} }
// Expand the input so that it is at least K times as big as block size // Expand the input so that it is at least K times as big as block size
@@ -580,6 +503,8 @@ static int Verify(const string& input) {
// Compress using string based routines // Compress using string based routines
const int result = VerifyString(input); const int result = VerifyString(input);
// Verify using sink based routines
VerifyStringSink(input);
VerifyNonBlockedCompression(input); VerifyNonBlockedCompression(input);
VerifyIOVec(input); VerifyIOVec(input);
@@ -589,12 +514,9 @@ static int Verify(const string& input) {
VerifyIOVec(input); VerifyIOVec(input);
} }
return result; return result;
} }
// This test checks to ensure that snappy doesn't coredump if it gets
// corrupted data.
static bool IsValidCompressedBuffer(const string& c) { static bool IsValidCompressedBuffer(const string& c) {
return snappy::IsValidCompressedBuffer(c.data(), c.size()); return snappy::IsValidCompressedBuffer(c.data(), c.size());
@@ -603,11 +525,13 @@ static bool Uncompress(const string& c, string* u) {
return snappy::Uncompress(c.data(), c.size(), u); return snappy::Uncompress(c.data(), c.size(), u);
} }
TYPED_TEST(CorruptedTest, VerifyCorrupted) { // This test checks to ensure that snappy doesn't coredump if it gets
// corrupted data.
TEST(CorruptedTest, VerifyCorrupted) {
string source = "making sure we don't crash with corrupted input"; string source = "making sure we don't crash with corrupted input";
VLOG(1) << source; VLOG(1) << source;
string dest; string dest;
TypeParam uncmp; string uncmp;
snappy::Compress(source.data(), source.size(), &dest); snappy::Compress(source.data(), source.size(), &dest);
// Mess around with the data. It's hard to simulate all possible // Mess around with the data. It's hard to simulate all possible
@@ -616,19 +540,19 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
dest[1]--; dest[1]--;
dest[3]++; dest[3]++;
// this really ought to fail. // this really ought to fail.
CHECK(!IsValidCompressedBuffer(TypeParam(dest))); CHECK(!IsValidCompressedBuffer(dest));
CHECK(!Uncompress(TypeParam(dest), &uncmp)); CHECK(!Uncompress(dest, &uncmp));
// This is testing for a security bug - a buffer that decompresses to 100k // This is testing for a security bug - a buffer that decompresses to 100k
// but we lie in the snappy header and only reserve 0 bytes of memory :) // but we lie in the snappy header and only reserve 0 bytes of memory :)
source.resize(100000); source.resize(100000);
for (int i = 0; i < source.length(); ++i) { for (size_t i = 0; i < source.length(); ++i) {
source[i] = 'A'; source[i] = 'A';
} }
snappy::Compress(source.data(), source.size(), &dest); snappy::Compress(source.data(), source.size(), &dest);
dest[0] = dest[1] = dest[2] = dest[3] = 0; dest[0] = dest[1] = dest[2] = dest[3] = 0;
CHECK(!IsValidCompressedBuffer(TypeParam(dest))); CHECK(!IsValidCompressedBuffer(dest));
CHECK(!Uncompress(TypeParam(dest), &uncmp)); CHECK(!Uncompress(dest, &uncmp));
if (sizeof(void *) == 4) { if (sizeof(void *) == 4) {
// Another security check; check a crazy big length can't DoS us with an // Another security check; check a crazy big length can't DoS us with an
@@ -637,20 +561,20 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
// where 3 GB might be an acceptable allocation size, Uncompress() // where 3 GB might be an acceptable allocation size, Uncompress()
// attempts to decompress, and sometimes causes the test to run out of // attempts to decompress, and sometimes causes the test to run out of
// memory. // memory.
dest[0] = dest[1] = dest[2] = dest[3] = 0xff; dest[0] = dest[1] = dest[2] = dest[3] = '\xff';
// This decodes to a really large size, i.e., about 3 GB. // This decodes to a really large size, i.e., about 3 GB.
dest[4] = 'k'; dest[4] = 'k';
CHECK(!IsValidCompressedBuffer(TypeParam(dest))); CHECK(!IsValidCompressedBuffer(dest));
CHECK(!Uncompress(TypeParam(dest), &uncmp)); CHECK(!Uncompress(dest, &uncmp));
} else { } else {
LOG(WARNING) << "Crazy decompression lengths not checked on 64-bit build"; LOG(WARNING) << "Crazy decompression lengths not checked on 64-bit build";
} }
// This decodes to about 2 MB; much smaller, but should still fail. // This decodes to about 2 MB; much smaller, but should still fail.
dest[0] = dest[1] = dest[2] = 0xff; dest[0] = dest[1] = dest[2] = '\xff';
dest[3] = 0x00; dest[3] = 0x00;
CHECK(!IsValidCompressedBuffer(TypeParam(dest))); CHECK(!IsValidCompressedBuffer(dest));
CHECK(!Uncompress(TypeParam(dest), &uncmp)); CHECK(!Uncompress(dest, &uncmp));
// try reading stuff in from a bad file. // try reading stuff in from a bad file.
for (int i = 1; i <= 3; ++i) { for (int i = 1; i <= 3; ++i) {
@@ -665,8 +589,8 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
snappy::ByteArraySource source(data.data(), data.size()); snappy::ByteArraySource source(data.data(), data.size());
CHECK(!snappy::GetUncompressedLength(&source, &ulen2) || CHECK(!snappy::GetUncompressedLength(&source, &ulen2) ||
(ulen2 < (1<<20))); (ulen2 < (1<<20)));
CHECK(!IsValidCompressedBuffer(TypeParam(data))); CHECK(!IsValidCompressedBuffer(data));
CHECK(!Uncompress(TypeParam(data), &uncmp)); CHECK(!Uncompress(data, &uncmp));
} }
} }
@@ -764,7 +688,7 @@ TEST(Snappy, RandomData) {
} }
string x; string x;
int len = rnd.Uniform(4096); size_t len = rnd.Uniform(4096);
if (i < 100) { if (i < 100) {
len = 65536 + rnd.Uniform(65536); len = 65536 + rnd.Uniform(65536);
} }
@@ -929,7 +853,6 @@ TEST(Snappy, IOVecCopyOverflow) {
} }
} }
static bool CheckUncompressedLength(const string& compressed, static bool CheckUncompressedLength(const string& compressed,
size_t* ulength) { size_t* ulength) {
const bool result1 = snappy::GetUncompressedLength(compressed.data(), const bool result1 = snappy::GetUncompressedLength(compressed.data(),
@@ -956,11 +879,11 @@ TEST(SnappyCorruption, TruncatedVarint) {
TEST(SnappyCorruption, UnterminatedVarint) { TEST(SnappyCorruption, UnterminatedVarint) {
string compressed, uncompressed; string compressed, uncompressed;
size_t ulength; size_t ulength;
compressed.push_back(128); compressed.push_back('\x80');
compressed.push_back(128); compressed.push_back('\x80');
compressed.push_back(128); compressed.push_back('\x80');
compressed.push_back(128); compressed.push_back('\x80');
compressed.push_back(128); compressed.push_back('\x80');
compressed.push_back(10); compressed.push_back(10);
CHECK(!CheckUncompressedLength(compressed, &ulength)); CHECK(!CheckUncompressedLength(compressed, &ulength));
CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
@@ -968,6 +891,20 @@ TEST(SnappyCorruption, UnterminatedVarint) {
&uncompressed)); &uncompressed));
} }
TEST(SnappyCorruption, OverflowingVarint) {
string compressed, uncompressed;
size_t ulength;
compressed.push_back('\xfb');
compressed.push_back('\xff');
compressed.push_back('\xff');
compressed.push_back('\xff');
compressed.push_back('\x7f');
CHECK(!CheckUncompressedLength(compressed, &ulength));
CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
CHECK(!snappy::Uncompress(compressed.data(), compressed.size(),
&uncompressed));
}
TEST(Snappy, ReadPastEndOfBuffer) { TEST(Snappy, ReadPastEndOfBuffer) {
// Check that we do not read past end of input // Check that we do not read past end of input
@@ -998,11 +935,13 @@ TEST(Snappy, ZeroOffsetCopyValidation) {
EXPECT_FALSE(snappy::IsValidCompressedBuffer(compressed, 4)); EXPECT_FALSE(snappy::IsValidCompressedBuffer(compressed, 4));
} }
namespace { namespace {
int TestFindMatchLength(const char* s1, const char *s2, unsigned length) { int TestFindMatchLength(const char* s1, const char *s2, unsigned length) {
return snappy::internal::FindMatchLength(s1, s2, s2 + length); std::pair<size_t, bool> p =
snappy::internal::FindMatchLength(s1, s2, s2 + length);
CHECK_EQ(p.first < 8, p.second);
return p.first;
} }
} // namespace } // namespace
@@ -1112,8 +1051,7 @@ TEST(Snappy, FindMatchLengthRandom) {
} }
DataEndingAtUnreadablePage u(s); DataEndingAtUnreadablePage u(s);
DataEndingAtUnreadablePage v(t); DataEndingAtUnreadablePage v(t);
int matched = snappy::internal::FindMatchLength( int matched = TestFindMatchLength(u.data(), v.data(), t.size());
u.data(), v.data(), v.data() + t.size());
if (matched == t.size()) { if (matched == t.size()) {
EXPECT_EQ(s, t); EXPECT_EQ(s, t);
} else { } else {
@@ -1125,21 +1063,113 @@ TEST(Snappy, FindMatchLengthRandom) {
} }
} }
static uint16 MakeEntry(unsigned int extra,
unsigned int len,
unsigned int copy_offset) {
// Check that all of the fields fit within the allocated space
assert(extra == (extra & 0x7)); // At most 3 bits
assert(copy_offset == (copy_offset & 0x7)); // At most 3 bits
assert(len == (len & 0x7f)); // At most 7 bits
return len | (copy_offset << 8) | (extra << 11);
}
// Check that the decompression table is correct, and optionally print out
// the computed one.
TEST(Snappy, VerifyCharTable) {
using snappy::internal::LITERAL;
using snappy::internal::COPY_1_BYTE_OFFSET;
using snappy::internal::COPY_2_BYTE_OFFSET;
using snappy::internal::COPY_4_BYTE_OFFSET;
using snappy::internal::char_table;
uint16 dst[256];
// Place invalid entries in all places to detect missing initialization
int assigned = 0;
for (int i = 0; i < 256; i++) {
dst[i] = 0xffff;
}
// Small LITERAL entries. We store (len-1) in the top 6 bits.
for (unsigned int len = 1; len <= 60; len++) {
dst[LITERAL | ((len-1) << 2)] = MakeEntry(0, len, 0);
assigned++;
}
// Large LITERAL entries. We use 60..63 in the high 6 bits to
// encode the number of bytes of length info that follow the opcode.
for (unsigned int extra_bytes = 1; extra_bytes <= 4; extra_bytes++) {
// We set the length field in the lookup table to 1 because extra
// bytes encode len-1.
dst[LITERAL | ((extra_bytes+59) << 2)] = MakeEntry(extra_bytes, 1, 0);
assigned++;
}
// COPY_1_BYTE_OFFSET.
//
// The tag byte in the compressed data stores len-4 in 3 bits, and
// offset/256 in 5 bits. offset%256 is stored in the next byte.
//
// This format is used for length in range [4..11] and offset in
// range [0..2047]
for (unsigned int len = 4; len < 12; len++) {
for (unsigned int offset = 0; offset < 2048; offset += 256) {
dst[COPY_1_BYTE_OFFSET | ((len-4)<<2) | ((offset>>8)<<5)] =
MakeEntry(1, len, offset>>8);
assigned++;
}
}
// COPY_2_BYTE_OFFSET.
// Tag contains len-1 in top 6 bits, and offset in next two bytes.
for (unsigned int len = 1; len <= 64; len++) {
dst[COPY_2_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(2, len, 0);
assigned++;
}
// COPY_4_BYTE_OFFSET.
// Tag contents len-1 in top 6 bits, and offset in next four bytes.
for (unsigned int len = 1; len <= 64; len++) {
dst[COPY_4_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(4, len, 0);
assigned++;
}
// Check that each entry was initialized exactly once.
EXPECT_EQ(256, assigned) << "Assigned only " << assigned << " of 256";
for (int i = 0; i < 256; i++) {
EXPECT_NE(0xffff, dst[i]) << "Did not assign byte " << i;
}
if (FLAGS_snappy_dump_decompression_table) {
printf("static const uint16 char_table[256] = {\n ");
for (int i = 0; i < 256; i++) {
printf("0x%04x%s",
dst[i],
((i == 255) ? "\n" : (((i%8) == 7) ? ",\n " : ", ")));
}
printf("};\n");
}
// Check that computed table matched recorded table.
for (int i = 0; i < 256; i++) {
EXPECT_EQ(dst[i], char_table[i]) << "Mismatch in byte " << i;
}
}
static void CompressFile(const char* fname) { static void CompressFile(const char* fname) {
string fullinput; string fullinput;
file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess(); CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
string compressed; string compressed;
Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false); Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false);
file::SetContents(string(fname).append(".comp"), compressed, file::Defaults()) CHECK_OK(file::SetContents(string(fname).append(".comp"), compressed,
.CheckSuccess(); file::Defaults()));
} }
static void UncompressFile(const char* fname) { static void UncompressFile(const char* fname) {
string fullinput; string fullinput;
file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess(); CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
size_t uncompLength; size_t uncompLength;
CHECK(CheckUncompressedLength(fullinput, &uncompLength)); CHECK(CheckUncompressedLength(fullinput, &uncompLength));
@@ -1148,28 +1178,25 @@ static void UncompressFile(const char* fname) {
uncompressed.resize(uncompLength); uncompressed.resize(uncompLength);
CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed)); CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed));
file::SetContents(string(fname).append(".uncomp"), uncompressed, CHECK_OK(file::SetContents(string(fname).append(".uncomp"), uncompressed,
file::Defaults()).CheckSuccess(); file::Defaults()));
} }
static void MeasureFile(const char* fname) { static void MeasureFile(const char* fname) {
string fullinput; string fullinput;
file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess(); CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
printf("%-40s :\n", fname); printf("%-40s :\n", fname);
int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len; int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len;
int end_len = fullinput.size(); int end_len = fullinput.size();
if (FLAGS_end_len >= 0) { if (FLAGS_end_len >= 0) {
end_len = min<int>(fullinput.size(), FLAGS_end_len); end_len = std::min<int>(fullinput.size(), FLAGS_end_len);
} }
for (int len = start_len; len <= end_len; len++) { for (int len = start_len; len <= end_len; len++) {
const char* const input = fullinput.data(); const char* const input = fullinput.data();
int repeats = (FLAGS_bytes + len) / (len + 1); int repeats = (FLAGS_bytes + len) / (len + 1);
if (FLAGS_zlib) Measure(input, len, ZLIB, repeats, 1024<<10); if (FLAGS_zlib) Measure(input, len, ZLIB, repeats, 1024<<10);
if (FLAGS_lzo) Measure(input, len, LZO, repeats, 1024<<10); if (FLAGS_lzo) Measure(input, len, LZO, repeats, 1024<<10);
if (FLAGS_liblzf) Measure(input, len, LIBLZF, repeats, 1024<<10);
if (FLAGS_quicklz) Measure(input, len, QUICKLZ, repeats, 1024<<10);
if (FLAGS_fastlz) Measure(input, len, FASTLZ, repeats, 1024<<10);
if (FLAGS_snappy) Measure(input, len, SNAPPY, repeats, 4096<<10); if (FLAGS_snappy) Measure(input, len, SNAPPY, repeats, 4096<<10);
// For block-size based measurements // For block-size based measurements
@@ -1298,6 +1325,37 @@ static void BM_UIOVec(int iters, int arg) {
} }
BENCHMARK(BM_UIOVec)->DenseRange(0, 4); BENCHMARK(BM_UIOVec)->DenseRange(0, 4);
static void BM_UFlatSink(int iters, int arg) {
StopBenchmarkTiming();
// Pick file to process based on "arg"
CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files));
string contents = ReadTestDataFile(files[arg].filename,
files[arg].size_limit);
string zcontents;
snappy::Compress(contents.data(), contents.size(), &zcontents);
char* dst = new char[contents.size()];
SetBenchmarkBytesProcessed(static_cast<int64>(iters) *
static_cast<int64>(contents.size()));
SetBenchmarkLabel(files[arg].label);
StartBenchmarkTiming();
while (iters-- > 0) {
snappy::ByteArraySource source(zcontents.data(), zcontents.size());
snappy::UncheckedByteArraySink sink(dst);
CHECK(snappy::Uncompress(&source, &sink));
}
StopBenchmarkTiming();
string s(dst, contents.size());
CHECK_EQ(contents, s);
delete[] dst;
}
BENCHMARK(BM_UFlatSink)->DenseRange(0, ARRAYSIZE(files) - 1);
static void BM_ZFlat(int iters, int arg) { static void BM_ZFlat(int iters, int arg) {
StopBenchmarkTiming(); StopBenchmarkTiming();
@@ -1329,23 +1387,20 @@ static void BM_ZFlat(int iters, int arg) {
} }
BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1); BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
} // namespace snappy } // namespace snappy
int main(int argc, char** argv) { int main(int argc, char** argv) {
InitGoogle(argv[0], &argc, &argv, true); InitGoogle(argv[0], &argc, &argv, true);
RunSpecifiedBenchmarks(); RunSpecifiedBenchmarks();
if (argc >= 2) { if (argc >= 2) {
for (int arg = 1; arg < argc; arg++) { for (int arg = 1; arg < argc; arg++) {
if (FLAGS_write_compressed) { if (FLAGS_write_compressed) {
CompressFile(argv[arg]); snappy::CompressFile(argv[arg]);
} else if (FLAGS_write_uncompressed) { } else if (FLAGS_write_uncompressed) {
UncompressFile(argv[arg]); snappy::UncompressFile(argv[arg]);
} else { } else {
MeasureFile(argv[arg]); snappy::MeasureFile(argv[arg]);
} }
} }
return 0; return 0;