ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit 'df26c08a34b4e07235d33fcd7e2fb311d83f069f' into HEAD

Browse Source 
Synchronize LZ4 with upstream
This commit is contained in:
JoelKatz
2015-05-29 15:03:27 -07:00
parent 6675ee7f5c df26c08a34
commit 4ff845ac91
62 changed files with 6254 additions and 2490 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
src/lz4/programs/Makefile
View File

@@ -19,32 +19,32 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# You can contact the author at :
# - LZ4 source repository : http://code.google.com/p/lz4/
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# - LZ4 source repository : https://github.com/Cyan4973/lz4
# - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
# ##########################################################################
# lz4 : Command Line Utility, supporting gzip-like arguments
# lz4c : CLU, supporting also legacy lz4demo arguments
# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
# fuzzer : Test tool, to check lz4 integrity on target platform
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
# frametest : Test tool, to check lz4frame integrity on target platform
# frametest32: Same as frametest, but forced to compile in 32-bits mode
# fullbench : Precisely measure speed for each LZ4 function variant
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# datagen : generates synthetic data samples for tests & benchmarks
# ##########################################################################
RELEASE?= r126
RELEASE?= r130
DESTDIR?=
PREFIX ?= /usr
PREFIX ?= /usr/local
CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
BINDIR=$(PREFIX)/bin
MANDIR=$(PREFIX)/share/man/man1
LZ4DIR=../lib
TEST_FILES = COPYING
TEST_TARGETS=test-native
BINDIR := $(PREFIX)/bin
MANDIR := $(PREFIX)/share/man/man1
LZ4DIR := ../lib
# Define *.exe as extension for Windows systems
@@ -58,12 +58,19 @@ endif
# Select test target for Travis CI's Build Matrix
TRAVIS_TARGET=$(LZ4_TRAVIS_CI_ENV)
TRAVIS_TARGET:= $(LZ4_TRAVIS_CI_ENV)
TEST_FILES := COPYING
TEST_TARGETS := test-native
FUZZER_TIME := -T9mn
default: lz4 lz4c
default: lz4
all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 frametest frametest32 datagen
m32: lz4c32 fullbench32 fuzzer32 frametest32
bins: lz4 lz4c fullbench fuzzer frametest datagen
all: bins m32
lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -92,12 +99,11 @@ frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxha
frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
datagen : datagen.c
datagen : datagen.c datagencli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
clean:
@rm -f core *.o *.test \
@rm -f core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) \
@@ -113,22 +119,26 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
install: lz4 lz4c
@echo Installing binaries
@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
@install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT)
@ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat
@install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT)
@install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4
@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat
@ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4
@install -m 755 lz4c $(DESTDIR)$(BINDIR)/lz4c
@echo Installing man pages
@install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
@install -m 644 lz4c.1 $(DESTDIR)$(MANDIR)/lz4c.1
@install -m 644 lz4cat.1 $(DESTDIR)$(MANDIR)/lz4cat.1
@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1
@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1
@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1
@echo lz4 installation completed
uninstall:
rm -f $(DESTDIR)$(BINDIR)/lz4cat
[ -x $(DESTDIR)$(BINDIR)/lz4$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4$(EXT)
[ -x $(DESTDIR)$(BINDIR)/lz4c$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4c$(EXT)
rm -f $(DESTDIR)$(BINDIR)/unlz4
[ -x $(DESTDIR)$(BINDIR)/lz4 ] && rm -f $(DESTDIR)$(BINDIR)/lz4
[ -x $(DESTDIR)$(BINDIR)/lz4c ] && rm -f $(DESTDIR)$(BINDIR)/lz4c
[ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1
[ -f $(DESTDIR)$(MANDIR)/lz4c.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4c.1
[ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
rm -f $(DESTDIR)$(MANDIR)/lz4c.1
rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
rm -f $(DESTDIR)$(MANDIR)/unlz4.1
@echo lz4 programs successfully uninstalled
test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-mem
@@ -139,13 +149,56 @@ test-all: test test32
test-travis: $(TRAVIS_TARGET)
test-lz4: lz4 datagen
./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID)
./datagen | ./lz4 | ./lz4 -vdq > $(VOID)
./datagen -g6M -p100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID)
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID)
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID)
# test frame concatenation with null-length frame
test-lz4-sparse: lz4 datagen
@echo "\n ---- test sparse file support ----"
./datagen -g5M -P100 > tmpSrc
./lz4 -B4D tmpSrc | ./lz4 -dv --sparse > tmpB4
diff -s tmpSrc tmpB4
./lz4 -B5D tmpSrc | ./lz4 -dv --sparse > tmpB5
diff -s tmpSrc tmpB5
./lz4 -B6D tmpSrc | ./lz4 -dv --sparse > tmpB6
diff -s tmpSrc tmpB6
./lz4 -B7D tmpSrc | ./lz4 -dv --sparse > tmpB7
diff -s tmpSrc tmpB7
./lz4 tmpSrc | ./lz4 -dv --no-sparse > tmpNoSparse
diff -s tmpSrc tmpNoSparse
ls -ls tmp*
./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dv --sparse > tmpOdd # Odd size file (to not finish on an exact nb of blocks)
./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd
ls -ls tmpOdd
@rm tmp*
@echo "\n Compatibility with Console :"
echo "Hello World 1 !" | ./lz4 | ./lz4 -d -c
echo "Hello World 2 !" | ./lz4 | ./lz4 -d | cat
echo "Hello World 3 !" | ./lz4 --no-frame-crc | ./lz4 -d -c
@echo "\n Compatibility with Append :"
./datagen -P100 -g1M > tmp1M
cat tmp1M > tmp2M
cat tmp1M >> tmp2M
./lz4 -B5 -v tmp1M tmpC
./lz4 -d -v tmpC tmpR
./lz4 -d -v tmpC >> tmpR
ls -ls tmp*
diff tmp2M tmpR
@rm tmp*
test-lz4-contentSize: lz4 datagen
@echo "\n ---- test original size support ----"
./datagen -g15M > tmp
./lz4 -v tmp | ./lz4 -t
./lz4 -v --content-size tmp | ./lz4 -d > tmp2
diff -s tmp tmp2
# test large size [2-4] GB
@./datagen -g3G -P100 | ./lz4 | ./lz4 --decompress --force --sparse - tmp
@ls -ls tmp
./lz4 --quiet --content-size tmp | ./lz4 --verbose --decompress --force --sparse - tmp2
@ls -ls tmp2
@rm tmp*
test-lz4-frame-concatenation: lz4 datagen
@echo "\n ---- test frame concatenation ----"
@echo -n > empty.test
@echo hi > nonempty.test
cat nonempty.test empty.test nonempty.test > orig.test
@@ -156,49 +209,101 @@ test-lz4: lz4 datagen
sdiff orig.test result.test
@rm *.test
@echo frame concatenation test completed
# test frame concatenation with null-length frame
test-lz4-multiple: lz4 datagen
@echo "\n ---- test multiple files ----"
@./datagen -s1 > tmp1 2> $(VOID)
@./datagen -s2 -g100K > tmp2 2> $(VOID)
@./datagen -s3 -g1M > tmp3 2> $(VOID)
./lz4 -f -m tmp*
ls -ls tmp*
rm tmp1 tmp2 tmp3
./lz4 -df -m *.lz4
ls -ls tmp*
./lz4 -f -m tmp1 notHere tmp2; echo $$?
@rm tmp*
test-lz4-basic: lz4 datagen
@echo "\n ---- test lz4 basic compression/decompression ----"
./datagen -g0 | ./lz4 -v | ./lz4 -t
./datagen -g16KB | ./lz4 -9 | ./lz4 -t
./datagen -g20KB > tmpSrc
./lz4 < tmpSrc | ./lz4 -d > tmpRes
diff -q tmpSrc tmpRes
./lz4 --no-frame-crc < tmpSrc | ./lz4 -d > tmpRes
diff -q tmpSrc tmpRes
./datagen | ./lz4 | ./lz4 -t
./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t
./datagen -g17M | ./lz4 -9v | ./lz4 -qt
./datagen -g33M | ./lz4 --no-frame-crc | ./lz4 -t
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -qt
./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -qt
@rm tmp*
test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
@echo "\n ---- test pass-through ----"
./datagen | ./lz4 -tf
test-lz4c: lz4c datagen
./datagen -g256MB | ./lz4c -l -v | ./lz4c -vdq > $(VOID)
@echo "\n ---- test lz4c version ----"
./datagen -g256MB | ./lz4c -l -v | ./lz4c -t
test-lz4c32: lz4 lz4c32 lz4 datagen
./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -vdq > $(VOID)
./datagen -g16KB | ./lz4c32 -9 | ./lz4 -vdq > $(VOID)
./datagen | ./lz4c32 | ./lz4c32 -vdq > $(VOID)
./datagen | ./lz4c32 | ./lz4 -vdq > $(VOID)
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -vdq > $(VOID)
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -vdq > $(VOID)
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID)
test-interop-32-64: lz4 lz4c32 datagen
@echo "\n ---- test interoperability 32-bits -vs- 64 bits ----"
./datagen -g16KB | ./lz4c32 -9 | ./lz4 -t
./datagen -P10 | ./lz4 -9B4 | ./lz4c32 -t
./datagen | ./lz4c32 | ./lz4 -t
./datagen -g1M | ./lz4 -3B5 | ./lz4c32 -t
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -qt
./datagen -g1G -P90 | ./lz4 | ./lz4c32 -t
./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -qt
test-lz4c32-basic: lz4c32 datagen
@echo "\n ---- test lz4c32 32-bits version ----"
./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -t
./datagen | ./lz4c32 | ./lz4c32 -t
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -qt
test-lz4c32: test-lz4c32-basic test-interop-32-64
test-fullbench: fullbench
./fullbench --no-prompt $(TEST_FILES)
./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fullbench32: fullbench32
./fullbench32 --no-prompt $(TEST_FILES)
./fullbench32 --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fuzzer: fuzzer
./fuzzer
./fuzzer $(FUZZER_TIME)
test-fuzzer32: fuzzer32
./fuzzer32
./fuzzer32 $(FUZZER_TIME)
test-frametest: frametest
./frametest
./frametest $(FUZZER_TIME)
test-frametest32: frametest32
./frametest32
./frametest32 $(FUZZER_TIME)
test-mem: lz4 datagen fuzzer frametest
test-mem: lz4 datagen fuzzer frametest fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
./datagen -g16KB > tmp
valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -BD -f tmp $(VOID)
./datagen -g16KB -s2 > tmp2
./datagen -g16KB -s3 > tmp3
valgrind --leak-check=yes --error-exitcode=1 ./lz4 --force --multiple tmp tmp2 tmp3
./datagen -g16MB > tmp
valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp /dev/null
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -B5D -f tmp tmp2
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -t tmp2
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -bi1 tmp
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2
./datagen -g256MB > tmp
valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp /dev/null
rm tmp
valgrind --leak-check=yes ./fuzzer -i64 -t1
valgrind --leak-check=yes ./frametest -i256
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -B4D -f -vq tmp $(VOID)
rm tmp*
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
test-mem32: lz4c32 datagen
# unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...

78
src/lz4/programs/bench.c
View File

@@ -1,6 +1,7 @@
/*
bench.c - Demo program to benchmark open-source compression algorithm
Copyright (C) Yann Collet 2012-2014
bench.c - Demo program to benchmark open-source compression algorithms
Copyright (C) Yann Collet 2012-2015
GPL v2 License
This program is free software; you can redistribute it and/or modify
@@ -18,16 +19,18 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
* Compiler Options
***************************************/
/* Disable some Visual warning messages */
#define _CRT_SECURE_NO_WARNINGS
#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
#if defined(_MSC_VER) || defined(_WIN32)
# define _CRT_SECURE_NO_WARNINGS
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
# define BMK_LEGACY_TIMER 1 /* S_ISREG & gettimeofday() are not supported by MSVC */
#endif
/* Unix Large Files support (>4GB) */
#define _FILE_OFFSET_BITS 64
@@ -37,11 +40,6 @@
# define _LARGEFILE64_SOURCE
#endif
/* S_ISREG & gettimeofday() are not supported by MSVC */
#if defined(_MSC_VER) || defined(_WIN32)
# define BMK_LEGACY_TIMER 1
#endif
/**************************************
* Includes
@@ -60,9 +58,9 @@
#include "lz4.h"
#define COMPRESSOR0 LZ4_compress_local
static int LZ4_compress_local(const char* src, char* dst, int size, int clevel) { (void)clevel; return LZ4_compress(src, dst, size); }
static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) { (void)clevel; return LZ4_compress_default(src, dst, srcSize, dstSize); }
#include "lz4hc.h"
#define COMPRESSOR1 LZ4_compressHC2
#define COMPRESSOR1 LZ4_compress_HC
#define DEFAULTCOMPRESSOR COMPRESSOR0
#include "xxhash.h"
@@ -123,8 +121,8 @@ struct chunkParameters
struct compressionParameters
{
int (*compressionFunction)(const char*, char*, int, int);
int (*decompressionFunction)(const char*, char*, int);
int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
int (*decompressionFunction)(const char* src, char* dst, int dstSize);
};
@@ -141,15 +139,15 @@ static int chunkSize = DEFAULT_CHUNKSIZE;
static int nbIterations = NBLOOPS;
static int BMK_pause = 0;
void BMK_SetBlocksize(int bsize) { chunkSize = bsize; }
void BMK_setBlocksize(int bsize) { chunkSize = bsize; }
void BMK_SetNbIterations(int nbLoops)
void BMK_setNbIterations(int nbLoops)
{
nbIterations = nbLoops;
DISPLAY("- %i iterations -\n", nbIterations);
}
void BMK_SetPause(void) { BMK_pause = 1; }
void BMK_setPause(void) { BMK_pause = 1; }
/*********************************************************
@@ -206,16 +204,21 @@ static size_t BMK_findMaxMem(U64 requiredMem)
while (!testmem)
{
requiredMem -= step;
if (requiredMem > step) requiredMem -= step;
else requiredMem >>= 1;
testmem = (BYTE*) malloc ((size_t)requiredMem);
}
free (testmem);
return (size_t) (requiredMem - step);
/* keep some space available */
if (requiredMem > step) requiredMem -= step;
else requiredMem >>= 1;
return (size_t)requiredMem;
}
static U64 BMK_GetFileSize(char* infilename)
static U64 BMK_GetFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
@@ -234,7 +237,7 @@ static U64 BMK_GetFileSize(char* infilename)
* Public function
**********************************************************/
int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
{
int fileIdx=0;
char* orig_buff;
@@ -265,7 +268,7 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
while (fileIdx<nbFiles)
{
FILE* inFile;
char* inFileName;
const char* inFileName;
U64 inFileSize;
size_t benchedSize;
int nbChunks;
@@ -278,15 +281,13 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
/* Check file existence */
inFileName = fileNamesTable[fileIdx++];
inFile = fopen( inFileName, "rb" );
if (inFile==NULL)
{
DISPLAY( "Pb opening %s\n", inFileName);
return 11;
}
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
/* Memory allocation & restrictions */
inFileSize = BMK_GetFileSize(inFileName);
if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; }
benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2;
if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; }
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
{
@@ -295,12 +296,11 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
/* Alloc */
chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters));
orig_buff = (char*)malloc((size_t )benchedSize);
orig_buff = (char*)malloc((size_t)benchedSize);
nbChunks = (int) ((int)benchedSize / chunkSize) + 1;
maxCompressedChunkSize = LZ4_compressBound(chunkSize);
compressedBuffSize = nbChunks * maxCompressedChunkSize;
compressedBuffer = (char*)malloc((size_t )compressedBuffSize);
compressedBuffer = (char*)malloc((size_t)compressedBuffSize);
if (!orig_buff || !compressedBuffer)
{
@@ -371,11 +371,12 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, cLevel);
chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize, cLevel);
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
nbLoops += !nbLoops; /* avoid division by zero */
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops;
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
ratio = (double)cSize/(double)benchedSize*100.;
@@ -397,8 +398,9 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
}
milliTime = BMK_GetMilliSpan(milliTime);
nbLoops += !nbLoops; /* avoid division by zero */
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops;
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
/* CRC Checking */
crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0);
@@ -408,9 +410,9 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
if (crcOrig==crcCheck)
{
if (ratio<100.)
DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
else
DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
}
totals += benchedSize;
totalz += cSize;
@@ -426,7 +428,7 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
if (nbFiles > 1)
DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.);
if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); }
if (BMK_pause) { DISPLAY("\npress enter...\n"); (void)getchar(); }
return 0;
}

23
src/lz4/programs/bench.h
View File

@@ -1,6 +1,6 @@
/*
bench.h - Demo program to benchmark open-source compression algorithm
Copyright (C) Yann Collet 2012-2014
Copyright (C) Yann Collet 2012-2015
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -17,26 +17,17 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 source repository : http://code.google.com/p/lz4/
- LZ4 public forum : https://group.google.com/forum/#!forum/lz4c
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/* Main function */
int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel);
int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel);
/* Set Parameters */
void BMK_SetBlocksize(int bsize);
void BMK_SetNbIterations(int nbLoops);
void BMK_SetPause(void);
void BMK_setBlocksize(int bsize);
void BMK_setNbIterations(int nbLoops);
void BMK_setPause(void);
#if defined (__cplusplus)
}
#endif

347
src/lz4/programs/datagen.c
View File

@@ -19,26 +19,20 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 source repository : http://code.google.com/p/lz4
- LZ4 source mirror : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
- ZSTD source repository : https://github.com/Cyan4973/zstd
- Public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
Remove Visual warning messages
* Includes
**************************************/
#define _CRT_SECURE_NO_WARNINGS // fgets
#include <stdlib.h> /* malloc */
#include <stdio.h> /* FILE, fwrite */
#include <string.h> /* memcpy */
/**************************************
Includes
**************************************/
#include <stdio.h> // fgets, sscanf
#include <string.h> // strcmp
/**************************************
Basic Types
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
@@ -57,230 +51,173 @@
/**************************************
Constants
* OS-specific Includes
**************************************/
#ifndef LZ4_VERSION
# define LZ4_VERSION "r125"
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
#else
# define SET_BINARY_MODE(file)
#endif
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define CDG_SIZE_DEFAULT (64 KB)
#define CDG_SEED_DEFAULT 0
#define CDG_COMPRESSIBILITY_DEFAULT 50
/**************************************
* Constants
**************************************/
#define KB *(1 <<10)
#define PRIME1 2654435761U
#define PRIME2 2246822519U
/**************************************
Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
/**************************************
Local Parameters
**************************************/
static unsigned no_prompt = 0;
static char* programName;
static unsigned displayLevel = 2;
/*********************************************************
functions
* Local Functions
*********************************************************/
#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned int CDG_rand(U32* src)
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned int RDG_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
rand32 += PRIME2;
rand32 = CDG_rotl32(rand32, 13);
rand32 ^= PRIME2;
rand32 = RDG_rotl32(rand32, 13);
*src = rand32;
return rand32;
}
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15)
#define CDG_RANDCHAR (((CDG_rand(seed) >> 9) & 63) + '0')
static void CDG_generate(U64 size, U32* seed, double proba)
#define LTSIZE 8192
#define LTMASK (LTSIZE-1)
static void* RDG_createLiteralDistrib(double ld)
{
BYTE fullbuff[32 KB + 128 KB + 1];
BYTE* buff = fullbuff + 32 KB;
U64 total=0;
U32 P32 = (U32)(32768 * proba);
U32 pos=1;
U32 genBlockSize = 128 KB;
BYTE* lt = (BYTE*)malloc(LTSIZE);
U32 i = 0;
BYTE character = '0';
BYTE firstChar = '(';
BYTE lastChar = '}';
// Build initial prefix
fullbuff[0] = CDG_RANDCHAR;
while (pos<32 KB)
if (ld==0.0)
{
// Select : Literal (char) or Match (within 32K)
if (CDG_RAND15BITS < P32)
character = 0;
firstChar = 0;
lastChar =255;
}
while (i<LTSIZE)
{
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
U32 end;
if (weight + i > LTSIZE) weight = LTSIZE-i;
end = i + weight;
while (i < end) lt[i++] = character;
character++;
if (character > lastChar) character = firstChar;
}
return lt;
}
static char RDG_genChar(U32* seed, const void* ltctx)
{
const BYTE* lt = (const BYTE*)ltctx;
U32 id = RDG_rand(seed) & LTMASK;
return lt[id];
}
#define RDG_DICTSIZE (32 KB)
#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767)
#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15)
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, void* litTable, unsigned* seedPtr)
{
BYTE* buffPtr = (BYTE*)buffer;
const U32 matchProba32 = (U32)(32768 * matchProba);
size_t pos = prefixSize;
void* ldctx = litTable;
U32* seed = seedPtr;
/* special case */
while (matchProba >= 1.0)
{
size_t size0 = RDG_rand(seed) & 3;
size0 = (size_t)1 << (16 + size0 * 2);
size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/
if (buffSize < pos + size0)
{
// Copy (within 64K)
U32 d;
int ref;
int length = CDG_RANDLENGTH + 4;
U32 offset = CDG_RAND15BITS + 1;
if (offset > pos) offset = pos;
ref = pos - offset;
memset(buffPtr+pos, 0, buffSize-pos);
return;
}
memset(buffPtr+pos, 0, size0);
pos += size0;
buffPtr[pos-1] = RDG_genChar(seed, ldctx);
}
/* init */
if (pos==0) buffPtr[0] = RDG_genChar(seed, ldctx), pos=1;
/* Generate compressible data */
while (pos < buffSize)
{
/* Select : Literal (char) or Match (within 32K) */
if (RDG_RAND15BITS < matchProba32)
{
/* Copy (within 32K) */
size_t match;
size_t d;
int length = RDG_RANDLENGTH + 4;
U32 offset = RDG_RAND15BITS + 1;
if (offset > pos) offset = (U32)pos;
match = pos - offset;
d = pos + length;
while (pos < d) fullbuff[pos++] = fullbuff[ref++];
if (d > buffSize) d = buffSize;
while (pos < d) buffPtr[pos++] = buffPtr[match++];
}
else
{
// Literal (noise)
U32 d = pos + CDG_RANDLENGTH;
while (pos < d) fullbuff[pos++] = CDG_RANDCHAR;
/* Literal (noise) */
size_t d;
size_t length = RDG_RANDLENGTH;
d = pos + length;
if (d > buffSize) d = buffSize;
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldctx);
}
}
}
// Generate compressible data
pos = 0;
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
{
void* ldctx;
if (litProba==0.0) litProba = matchProba / 4.5;
ldctx = RDG_createLiteralDistrib(litProba);
RDG_genBlock(buffer, size, 0, matchProba, ldctx, &seed);
free(ldctx);
}
#define RDG_BLOCKSIZE (128 KB)
void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed)
{
BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE];
U64 total = 0;
size_t genBlockSize = RDG_BLOCKSIZE;
void* ldctx;
/* init */
if (litProba==0.0) litProba = matchProba / 4.5;
ldctx = RDG_createLiteralDistrib(litProba);
SET_BINARY_MODE(stdout);
/* Generate dict */
RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed);
/* Generate compressible data */
while (total < size)
{
if (size-total < 128 KB) genBlockSize = (U32)(size-total);
RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed);
if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total);
total += genBlockSize;
buff[genBlockSize] = 0;
pos = 0;
while (pos<genBlockSize)
{
// Select : Literal (char) or Match (within 32K)
if (CDG_RAND15BITS < P32)
{
// Copy (within 64K)
int ref;
U32 d;
int length = CDG_RANDLENGTH + 4;
U32 offset = CDG_RAND15BITS + 1;
if (pos + length > genBlockSize ) length = genBlockSize - pos;
ref = pos - offset;
d = pos + length;
while (pos < d) buff[pos++] = buff[ref++];
}
else
{
// Literal (noise)
U32 d;
int length = CDG_RANDLENGTH;
if (pos + length > genBlockSize) length = genBlockSize - pos;
d = pos + length;
while (pos < d) buff[pos++] = CDG_RANDCHAR;
}
}
// output datagen
pos=0;
for (;pos+512<=genBlockSize;pos+=512)
printf("%512.512s", buff+pos);
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
// Regenerate prefix
memcpy(fullbuff, buff + 96 KB, 32 KB);
}
}
int CDG_usage(void)
{
DISPLAY( "Compressible data generator\n");
DISPLAY( "Usage :\n");
DISPLAY( " %s [size] [args]\n", programName);
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -g# : generate # data (default:%i)\n", CDG_SIZE_DEFAULT);
DISPLAY( " -s# : Select seed (default:%i)\n", CDG_SEED_DEFAULT);
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", CDG_COMPRESSIBILITY_DEFAULT);
DISPLAY( " -h : display help and exit\n");
return 0;
}
int main(int argc, char** argv)
{
int argNb;
int proba = CDG_COMPRESSIBILITY_DEFAULT;
U64 size = CDG_SIZE_DEFAULT;
U32 seed = CDG_SEED_DEFAULT;
// Check command line
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
if(!argument) continue; // Protection if argument empty
// Decode command (note : aggregated commands are allowed)
if (*argument=='-')
{
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; }
argument++;
while (*argument!=0)
{
switch(*argument)
{
case 'h':
return CDG_usage();
case 'g':
argument++;
size=0;
while ((*argument>='0') && (*argument<='9'))
{
size *= 10;
size += *argument - '0';
argument++;
}
if (*argument=='K') { size <<= 10; argument++; }
if (*argument=='M') { size <<= 20; argument++; }
if (*argument=='G') { size <<= 30; argument++; }
if (*argument=='B') { argument++; }
break;
case 's':
argument++;
seed=0;
while ((*argument>='0') && (*argument<='9'))
{
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 'p':
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9'))
{
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
case 'v':
displayLevel = 4;
argument++;
break;
default: ;
}
}
}
fwrite(buff, 1, genBlockSize, stdout);
/* update dict */
memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE);
}
// Get Seed
DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
CDG_generate(size, &seed, ((double)proba) / 100);
return 0;
free(ldctx);
}

40
src/lz4/programs/datagen.h Normal file
View File

@@ -0,0 +1,40 @@
/*
datagen.h - compressible data generator header
Copyright (C) Yann Collet 2012-2015
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- ZSTD source repository : https://github.com/Cyan4973/zstd
- Public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#include <stddef.h> /* size_t */
void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed);
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed);
/* RDG_genOut
Generate 'size' bytes of compressible data into stdout.
Compressibility can be controlled using 'matchProba'.
'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used.
Generated data can be selected using 'seed'.
If (matchProba, litProba and seed) are equal, the function always generate the same content.
RDG_genBuffer
Same as RDG_genOut, but generate data into provided buffer
*/

193
src/lz4/programs/datagencli.c Normal file
View File

@@ -0,0 +1,193 @@
/*
datagencli.c
compressible data command line generator
Copyright (C) Yann Collet 2012-2015
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- ZSTD source repository : https://github.com/Cyan4973/zstd
- Public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
* Includes
**************************************/
#include <stdio.h> /* fprintf, stderr */
#include "datagen.h" /* RDG_generate */
/**************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
* Constants
**************************************/
#ifndef ZSTD_VERSION
# define ZSTD_VERSION "r1"
#endif
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define SIZE_DEFAULT (64 KB)
#define SEED_DEFAULT 0
#define COMPRESSIBILITY_DEFAULT 50
/**************************************
* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned displayLevel = 2;
/*********************************************************
* Command line
*********************************************************/
static int usage(char* programName)
{
DISPLAY( "Compressible data generator\n");
DISPLAY( "Usage :\n");
DISPLAY( " %s [size] [args]\n", programName);
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT);
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT);
DISPLAY( " -h : display help and exit\n");
DISPLAY( "Special values :\n");
DISPLAY( " -P0 : generate incompressible noise\n");
DISPLAY( " -P100 : generate sparse files\n");
return 0;
}
int main(int argc, char** argv)
{
int argNb;
double proba = (double)COMPRESSIBILITY_DEFAULT / 100;
double litProba = 0.0;
U64 size = SIZE_DEFAULT;
U32 seed = SEED_DEFAULT;
char* programName;
/* Check command line */
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
/* Handle commands. Aggregated commands are allowed */
if (*argument=='-')
{
argument++;
while (*argument!=0)
{
switch(*argument)
{
case 'h':
return usage(programName);
case 'g':
argument++;
size=0;
while ((*argument>='0') && (*argument<='9'))
{
size *= 10;
size += *argument - '0';
argument++;
}
if (*argument=='K') { size <<= 10; argument++; }
if (*argument=='M') { size <<= 20; argument++; }
if (*argument=='G') { size <<= 30; argument++; }
if (*argument=='B') { argument++; }
break;
case 's':
argument++;
seed=0;
while ((*argument>='0') && (*argument<='9'))
{
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 'P':
argument++;
proba=0.0;
while ((*argument>='0') && (*argument<='9'))
{
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba>100.) proba=100.;
proba /= 100.;
break;
case 'L': /* hidden argument : Literal distribution probability */
argument++;
litProba=0.;
while ((*argument>='0') && (*argument<='9'))
{
litProba *= 10;
litProba += *argument - '0';
argument++;
}
if (litProba>100.) litProba=100.;
litProba /= 100.;
break;
case 'v':
displayLevel = 4;
argument++;
break;
default:
return usage(programName);
}
}
}
}
DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
RDG_genOut(size, proba, litProba, seed);
DISPLAYLEVEL(1, "\n");
return 0;
}

406
src/lz4/programs/frametest.c
View File

@@ -1,6 +1,7 @@
/*
frameTest - test tool for lz4frame
Copyright (C) Yann Collet 2014
Copyright (C) Yann Collet 2014-2015
GPL v2 License
This program is free software; you can redistribute it and/or modify
@@ -18,38 +19,43 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 source repository : http://code.google.com/p/lz4/
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
Compiler specific
* Compiler specific
**************************************/
#define _CRT_SECURE_NO_WARNINGS // fgets
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
#endif
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#ifdef __GNUC__
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
/* S_ISREG & gettimeofday() are not supported by MSVC */
#if defined(_MSC_VER) || defined(_WIN32)
# define FUZ_LEGACY_TIMER 1
#endif
/**************************************
Includes
* Includes
**************************************/
#include <stdlib.h> // free
#include <stdio.h> // fgets, sscanf
#include <sys/timeb.h> // timeb
#include <string.h> // strcmp
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* fprintf */
#include <string.h> /* strcmp */
#include "lz4frame_static.h"
#include "xxhash.h" // XXH64
#include "xxhash.h" /* XXH64 */
/* Use ftime() if gettimeofday() is not available on your target */
#if defined(FUZ_LEGACY_TIMER)
# include <sys/timeb.h> /* timeb, ftime */
#else
# include <sys/time.h> /* gettimeofday */
#endif
/**************************************
Basic Types
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
@@ -67,13 +73,26 @@ typedef unsigned long long U64;
#endif
/* unoptimized version; solves endianess & alignment issues */
static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
{
BYTE* dstPtr = (BYTE*)dstVoidPtr;
dstPtr[0] = (BYTE)value32;
dstPtr[1] = (BYTE)(value32 >> 8);
dstPtr[2] = (BYTE)(value32 >> 16);
dstPtr[3] = (BYTE)(value32 >> 24);
}
/**************************************
Constants
* Constants
**************************************/
#ifndef LZ4_VERSION
# define LZ4_VERSION ""
#endif
#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
@@ -87,7 +106,7 @@ static const U32 prime2 = 2246822519U;
/**************************************
Macros
* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
@@ -100,7 +119,7 @@ static U32 g_time = 0;
/*****************************************
Local Parameters
* Local Parameters
*****************************************/
static U32 no_prompt = 0;
static char* programName;
@@ -109,8 +128,10 @@ static U32 pause = 0;
/*********************************************************
Fuzzer functions
* Fuzzer functions
*********************************************************/
#if defined(FUZ_LEGACY_TIMER)
static U32 FUZ_GetMilliStart(void)
{
struct timeb tb;
@@ -120,6 +141,19 @@ static U32 FUZ_GetMilliStart(void)
return nCount;
}
#else
static U32 FUZ_GetMilliStart(void)
{
struct timeval tv;
U32 nCount;
gettimeofday(&tv, NULL);
nCount = (U32) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
return nCount;
}
#endif
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
{
@@ -151,15 +185,15 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, d
unsigned pos = 0;
U32 P32 = (U32)(32768 * proba);
// First Byte
/* First Byte */
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
while (pos < bufferSize)
{
// Select : Literal (noise) or copy (within 64K)
/* Select : Literal (noise) or copy (within 64K) */
if (FUZ_RAND15BITS < P32)
{
// Copy (within 64K)
/* Copy (within 64K) */
unsigned match, end;
unsigned length = FUZ_RANDLENGTH + 4;
unsigned offset = FUZ_RAND15BITS + 1;
@@ -171,7 +205,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, d
}
else
{
// Literal (noise)
/* Literal (noise) */
unsigned end;
unsigned length = FUZ_RANDLENGTH;
if (pos + length > bufferSize) length = bufferSize - pos;
@@ -203,18 +237,20 @@ int basicTests(U32 seed, double compressibility)
void* decodedBuffer;
U32 randState = seed;
size_t cSize, testSize;
LZ4F_preferences_t prefs = { 0 };
LZ4F_decompressionContext_t dCtx;
LZ4F_preferences_t prefs;
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cctx = NULL;
U64 crcOrig;
// Create compressible test buffer
/* Create compressible test buffer */
memset(&prefs, 0, sizeof(prefs));
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
// Trivial tests : one-step frame
/* Trivial tests : one-step frame */
testSize = COMPRESSIBLE_NOISE_LENGTH;
DISPLAYLEVEL(3, "Using NULL preferences : \n");
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
@@ -236,16 +272,63 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "Single Block : \n");
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (crcDest != crcOrig) goto _output_error;
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
DISPLAYLEVEL(4, "Reusing decompression context \n");
{
size_t iSize = compressedBufferSize - 4;
const BYTE* cBuff = (const BYTE*) compressedBuffer;
DISPLAYLEVEL(3, "Missing last 4 bytes : ");
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
if (!errorCode) goto _output_error;
DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)errorCode);
cBuff += iSize;
iSize = errorCode;
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL);
if (errorCode != 0) goto _output_error;
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (crcDest != crcOrig) goto _output_error;
}
{
size_t oSize = 0;
size_t iSize = 0;
LZ4F_frameInfo_t fi;
DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
DISPLAYLEVEL(3, "get FrameInfo on null input : ");
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
DISPLAYLEVEL(3, "get FrameInfo on not enough input : ");
iSize = 6;
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
ip += iSize;
DISPLAYLEVEL(3, "get FrameInfo on enough input : ");
iSize = 15 - iSize;
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, " correctly decoded \n");
ip += iSize;
}
DISPLAYLEVEL(3, "Byte after byte : \n");
while (ip < iend)
{
size_t oSize = oend-op;
size_t iSize = 1;
//DISPLAY("%7i \n", (int)(ip-(BYTE*)compressedBuffer));
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += oSize;
@@ -253,28 +336,28 @@ int basicTests(U32 seed, double compressibility)
}
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (crcDest != crcOrig) goto _output_error;
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH);
errorCode = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(errorCode)) goto _output_error;
}
DISPLAYLEVEL(3, "Using 64 KB block : \n");
prefs.frameInfo.blockSizeID = max64KB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
prefs.frameInfo.blockSizeID = LZ4F_max64KB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 256 KB block : \n");
prefs.frameInfo.blockSizeID = max256KB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
prefs.frameInfo.blockSizeID = LZ4F_max256KB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
@@ -314,42 +397,165 @@ int basicTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 1 MB block : \n");
prefs.frameInfo.blockSizeID = max1MB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
prefs.frameInfo.blockSizeID = LZ4F_max1MB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 4 MB block : \n");
prefs.frameInfo.blockSizeID = max4MB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
prefs.frameInfo.blockSizeID = LZ4F_max4MB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
{
size_t errorCode;
BYTE* const ostart = (BYTE*)compressedBuffer;
BYTE* op = ostart;
errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, "compress without frameSize : \n");
memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
DISPLAYLEVEL(3, "compress with frameSize : \n");
prefs.frameInfo.contentSize = testSize;
op = ostart;
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
DISPLAYLEVEL(3, "compress with wrong frameSize : \n");
prefs.frameInfo.contentSize = testSize+1;
op = ostart;
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL);
if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); }
else
goto _output_error;
errorCode = LZ4F_freeCompressionContext(cctx);
if (LZ4F_isError(errorCode)) goto _output_error;
cctx = NULL;
}
DISPLAYLEVEL(3, "Skippable frame test : \n");
{
size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
BYTE* op = (BYTE*)decodedBuffer;
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
BYTE* ip = (BYTE*)compressedBuffer;
BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
/* generate skippable frame */
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
FUZ_writeLE32(ip+4, (U32)cSize);
DISPLAYLEVEL(3, "random segment sizes : \n");
while (ip < iend)
{
unsigned nbBits = FUZ_rand(&randState) % maxBits;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t oSize = oend-op;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += oSize;
ip += iSize;
}
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize);
/* generate zero-size skippable frame */
DISPLAYLEVEL(3, "zero-size skippable frame\n");
ip = (BYTE*)compressedBuffer;
op = (BYTE*)decodedBuffer;
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1);
FUZ_writeLE32(ip+4, 0);
iend = ip+8;
while (ip < iend)
{
unsigned nbBits = FUZ_rand(&randState) % maxBits;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t oSize = oend-op;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += oSize;
ip += iSize;
}
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
DISPLAYLEVEL(3, "Skippable frame header complete in first call \n");
ip = (BYTE*)compressedBuffer;
op = (BYTE*)decodedBuffer;
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2);
FUZ_writeLE32(ip+4, 10);
iend = ip+18;
while (ip < iend)
{
size_t iSize = 10;
size_t oSize = 10;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += oSize;
ip += iSize;
}
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
}
DISPLAY("Basic tests completed \n");
_end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
LZ4F_freeCompressionContext(cctx); cctx = NULL;
return testResult;
_output_error:
@@ -362,8 +568,8 @@ _output_error:
static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
{
int p=0;
BYTE* b1=(BYTE*)buff1;
BYTE* b2=(BYTE*)buff2;
const BYTE* b1=(const BYTE*)buff1;
const BYTE* b2=(const BYTE*)buff2;
if (nonContiguous)
{
DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
@@ -376,7 +582,7 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un
static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility)
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration)
{
unsigned testResult = 0;
unsigned testNb = 0;
@@ -387,11 +593,16 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cCtx = NULL;
size_t result;
const U32 startTime = FUZ_GetMilliStart();
XXH64_state_t xxh64;
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
// Create buffers
/* Init */
duration *= 1000;
/* Create buffers */
result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
@@ -400,43 +611,56 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
compressedBuffer = malloc(LZ4F_compressFrameBound(srcDataLength, NULL));
CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
decodedBuffer = malloc(srcDataLength);
decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
// jump to requested testNb
for (testNb =0; testNb < startTest; testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
/* jump to requested testNb */
for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
// main fuzzer loop
for ( ; testNb < nbTests; testNb++)
/* main fuzzer test loop */
for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++)
{
U32 randState = coreRand ^ prime1;
unsigned BSId = 4 + (FUZ_rand(&randState) & 3);
unsigned BMId = FUZ_rand(&randState) & 1;
unsigned CCflag = FUZ_rand(&randState) & 1;
unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
LZ4F_preferences_t prefs = { 0 };
LZ4F_compressOptions_t cOptions = { 0 };
LZ4F_decompressOptions_t dOptions = { 0 };
LZ4F_preferences_t prefs;
LZ4F_compressOptions_t cOptions;
LZ4F_decompressOptions_t dOptions;
unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
size_t cSize;
U64 crcOrig, crcDecoded;
LZ4F_preferences_t* prefsPtr = &prefs;
(void)FUZ_rand(&coreRand); // update rand seed
prefs.frameInfo.blockMode = (blockMode_t)BMId;
prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId;
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag;
(void)FUZ_rand(&coreRand); /* update seed */
memset(&prefs, 0, sizeof(prefs));
memset(&cOptions, 0, sizeof(cOptions));
memset(&dOptions, 0, sizeof(dOptions));
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId;
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId;
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag;
prefs.frameInfo.contentSize = frameContentSize;
prefs.autoFlush = autoflush;
prefs.compressionLevel = FUZ_rand(&randState) % 5;
if ((FUZ_rand(&randState)&0xF) == 1) prefsPtr = NULL;
if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
DISPLAYUPDATE(2, "\r%5u ", testNb);
crcOrig = XXH64((BYTE*)srcBuffer+srcStart, (U32)srcSize, 1);
crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);
if ((FUZ_rand(&randState)&0xF) == 2)
if ((FUZ_rand(&randState) & 0xFFF) == 0)
{
/* create a skippable frame (rare case) */
BYTE* op = (BYTE*)compressedBuffer;
FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15));
FUZ_writeLE32(op+4, (U32)srcSize);
cSize = srcSize+8;
}
else if ((FUZ_rand(&randState) & 0xF) == 2)
{
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), (char*)srcBuffer + srcStart, srcSize, prefsPtr);
CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
@@ -483,10 +707,12 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
const BYTE* const iend = ip + cSize;
BYTE* op = (BYTE*)decodedBuffer;
BYTE* const oend = op + srcDataLength;
size_t totalOut = 0;
unsigned maxBits = FUZ_highbit((U32)cSize);
unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1;
nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */
XXH64_reset(&xxh64, 1);
if (maxBits < 3) maxBits = 3;
while (ip < iend)
{
unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
@@ -497,22 +723,24 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
if (oSize > (size_t)(oend-op)) oSize = oend-op;
dOptions.stableDst = FUZ_rand(&randState) & 1;
if (nonContiguousDst==2) dOptions.stableDst = 0;
//if (ip == compressedBuffer+62073) DISPLAY("oSize : %i : pos %i \n", (int)oSize, (int)(op-(BYTE*)decodedBuffer));
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
//if (op+oSize >= (BYTE*)decodedBuffer+94727) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
//if ((int)result<0) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid)
locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result));
XXH64_update(&xxh64, op, (U32)oSize);
totalOut += oSize;
op += oSize;
ip += iSize;
op += nonContiguousDst;
if (nonContiguousDst==2) op = decodedBuffer; // overwritten destination
if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */
}
CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
crcDecoded = XXH64_digest(&xxh64);
if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
CHECK(crcDecoded != crcOrig, "Decompression corruption");
if (totalOut) /* otherwise, it's a skippable frame */
{
crcDecoded = XXH64_digest(&xxh64);
if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
CHECK(crcDecoded != crcOrig, "Decompression corruption");
}
}
}
@@ -528,7 +756,7 @@ _end:
if (pause)
{
DISPLAY("press enter to finish \n");
getchar();
(void)getchar();
}
return testResult;
@@ -545,6 +773,7 @@ int FUZ_usage(void)
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
@@ -563,16 +792,17 @@ int main(int argc, char** argv)
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0;
U32 duration=0;
// Check command line
/* Check command line */
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
if(!argument) continue; // Protection if argument empty
if(!argument) continue; /* Protection if argument empty */
// Decode command (note : aggregated commands are allowed)
/* Decode command (note : aggregated commands are allowed) */
if (argument[0]=='-')
{
if (!strcmp(argument, "--no-prompt"))
@@ -605,7 +835,7 @@ int main(int argc, char** argv)
case 'i':
argument++;
nbTests=0;
nbTests=0; duration=0;
while ((*argument>='0') && (*argument<='9'))
{
nbTests *= 10;
@@ -613,6 +843,32 @@ int main(int argc, char** argv)
argument++;
}
break;
case 'T':
argument++;
nbTests = 0; duration = 0;
for (;;)
{
switch(*argument)
{
case 'm': duration *= 60; argument++; continue;
case 's':
case 'n': argument++; continue;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': duration *= 10; duration += *argument++ - '0'; continue;
}
break;
}
break;
case 's':
argument++;
seed=0;
@@ -654,7 +910,7 @@ int main(int argc, char** argv)
}
}
// Get Seed
/* Get Seed */
printf("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
@@ -665,5 +921,5 @@ int main(int argc, char** argv)
if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
if (result) return 1;
return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration);
}

714
src/lz4/programs/fullbench.c
View File

File diff suppressed because it is too large Load Diff

378
src/lz4/programs/fuzzer.c
View File

@@ -19,13 +19,12 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 source repository : http://code.google.com/p/lz4
- LZ4 source mirror : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
* Remove Visual warning messages
* Compiler options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS /* fgets */
@@ -34,21 +33,32 @@
# pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */
#endif
/* S_ISREG & gettimeofday() are not supported by MSVC */
#if defined(_MSC_VER) || defined(_WIN32)
# define FUZ_LEGACY_TIMER 1
#endif
/**************************************
* Includes
* Includes
**************************************/
#include <stdlib.h>
#include <stdio.h> /* fgets, sscanf */
#include <sys/timeb.h> /* timeb */
#include <string.h> /* strcmp */
#include "lz4.h"
#include "lz4hc.h"
#include "xxhash.h"
/* Use ftime() if gettimeofday() is not available on your target */
#if defined(FUZ_LEGACY_TIMER)
# include <sys/timeb.h> /* timeb, ftime */
#else
# include <sys/time.h> /* gettimeofday */
#endif
/**************************************
* Basic Types
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
@@ -67,7 +77,7 @@ typedef unsigned long long U64;
/**************************************
* Constants
* Constants
**************************************/
#ifndef LZ4_VERSION
# define LZ4_VERSION ""
@@ -88,7 +98,7 @@ typedef unsigned long long U64;
/*****************************************
* Macros
* Macros
*****************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
@@ -98,8 +108,10 @@ static U32 g_time = 0;
/*********************************************************
Fuzzer functions
* Fuzzer functions
*********************************************************/
#if defined(FUZ_LEGACY_TIMER)
static U32 FUZ_GetMilliStart(void)
{
struct timeb tb;
@@ -109,6 +121,20 @@ static U32 FUZ_GetMilliStart(void)
return nCount;
}
#else
static U32 FUZ_GetMilliStart(void)
{
struct timeval tv;
U32 nCount;
gettimeofday(&tv, NULL);
nCount = (U32) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
return nCount;
}
#endif
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
{
U32 nCurrent = FUZ_GetMilliStart();
@@ -127,10 +153,10 @@ static U32 FUZ_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
rand32 += PRIME2;
rand32 ^= PRIME2;
rand32 = FUZ_rotl32(rand32, 13);
*src = rand32;
return rand32 >> 3;
return rand32;
}
@@ -142,27 +168,28 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
size_t pos = 0;
U32 P32 = (U32)(32768 * proba);
// First Byte
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
/* First Bytes */
while (pos < 20)
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
while (pos < bufferSize)
{
// Select : Literal (noise) or copy (within 64K)
/* Select : Literal (noise) or copy (within 64K) */
if (FUZ_RAND15BITS < P32)
{
// Copy (within 64K)
/* Copy (within 64K) */
size_t match, d;
size_t length = FUZ_RANDLENGTH + 4;
size_t offset = FUZ_RAND15BITS + 1;
if (offset > pos) offset = pos;
while (offset > pos) offset >>= 1;
d = pos + length;
if (d > bufferSize) d = bufferSize;
while (d > bufferSize) d = bufferSize;
match = pos - offset;
while (pos < d) BBuffer[pos++] = BBuffer[match++];
}
else
{
// Literal (noise)
/* Literal (noise) */
size_t d;
size_t length = FUZ_RANDLENGTH;
d = pos + length;
@@ -177,13 +204,13 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
#define BLOCKSIZE_I134 (32 MB)
static int FUZ_AddressOverflow(void)
{
char* buffers[MAX_NB_BUFF_I134+1] = {0};
char* buffers[MAX_NB_BUFF_I134+1];
int i, nbBuff=0;
int highAddress = 0;
printf("Overflow tests : ");
// Only possible in 32-bits
/* Only possible in 32-bits */
if (sizeof(void*)==8)
{
printf("64 bits mode : no overflow \n");
@@ -273,7 +300,17 @@ static void FUZ_displayUpdate(unsigned testNb)
}
static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const double compressibility)
static void FUZ_findDiff(const void* buff1, const void* buff2)
{
const BYTE* b1 = (const BYTE*)buff1;
const BYTE* b2 = (const BYTE*)buff2;
size_t i=0;
while (b1[i]==b2[i]) i++;
DISPLAY("Wrong Byte at position %u\n", (unsigned)i);
}
static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration)
{
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
@@ -286,33 +323,35 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
int ret;
unsigned cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
LZ4_streamHC_t LZ4dictHC;
U32 crcOrig, crcCheck;
U32 coreRandState = seed;
U32 randState = coreRandState ^ PRIME3;
int result = 0;
const U32 startTime = FUZ_GetMilliStart();
// init
/* init */
memset(&LZ4dict, 0, sizeof(LZ4dict));
duration *= 1000;
// Create compressible test buffer
/* Create compressible test buffer */
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// move to startCycle
/* move to startCycle */
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
(void)FUZ_rand(&coreRandState);
if (0) // some problems related to dictionary re-use; in this case, enable this loop
if (0) /* some problems are related to dictionary re-use; in this case, enable this loop */
{
int dictSize, blockSize, blockStart;
char* dict;
@@ -334,8 +373,8 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
}
}
// Test loop
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
/* Main test loop */
for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetMilliSpan(startTime) < duration) ; cycleNb++)
{
U32 testNb = 0;
char* dict;
@@ -347,7 +386,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
(void)FUZ_rand(&coreRandState);
randState = coreRandState ^ PRIME3;
// Select block to test
/* Select block to test */
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
@@ -357,23 +396,58 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
/* Compression tests */
// Test compression HC
/* Test compression destSize */
FUZ_DISPLAYTEST;
{
int srcSize = blockSize;
int targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
char endCheck = FUZ_rand(&randState) & 255;
compressedBuffer[targetSize] = endCheck;
ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize);
FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !");
FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !");
FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !");
DISPLAY("destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
if (targetSize>0)
{
FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed");
/* check correctness */
FUZ_DISPLAYTEST;
crcOrig = XXH32(block, srcSize, 0);
compressedSize = ret;
endCheck = FUZ_rand(&randState) & 255;
decodedBuffer[srcSize] = endCheck;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize");
FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
FUZ_CHECKTEST(decodedBuffer[srcSize] != endCheck, "LZ4_decompress_safe() overwrite dst buffer !");
crcCheck = XXH32(decodedBuffer, srcSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe() corrupted decoded data");
DISPLAY(" OK \n");
}
else
DISPLAY(" \n");
}
/* Test compression HC */
FUZ_DISPLAYTEST;
ret = LZ4_compressHC(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret;
// Test compression HC using external state
/* Test compression HC using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
// Test compression using external state
/* Test compression using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
// Test compression
/* Test compression */
FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
@@ -383,7 +457,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
crcOrig = XXH32(block, blockSize, 0);
// Test decoding with output size being exactly what's necessary => must work
/* Test decoding with output size being exactly what's necessary => must work */
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
@@ -391,19 +465,19 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
// Test decoding with one byte missing => must fail
/* Test decoding with one byte missing => must fail */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
// Test decoding with one byte too much => must fail
/* Test decoding with one byte too much => must fail */
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
// Test decoding with output size exactly what's necessary => must work
/* Test decoding with output size exactly what's necessary => must work */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
@@ -517,16 +591,18 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
/* Compress using dictionary */
FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
free (LZ4continue);
{
LZ4_stream_t LZ4_stream;
LZ4_resetStream(&LZ4_stream);
LZ4_compress_continue (&LZ4_stream, dict, compressedBuffer, dictSize); /* Just to fill hash tables */
blockContinueCompressedSize = LZ4_compress_continue (&LZ4_stream, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
}
/* Decompress with dictionary as prefix */
FUZ_DISPLAYTEST;
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
if (crcCheck!=crcOrig)
@@ -539,14 +615,14 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
// Compress using External dictionary
/* Compress using External dictionary */
FUZ_DISPLAYTEST;
dict -= (FUZ_rand(&randState) & 0xF) + 1; // Separation, so it is an ExtDict
dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
@@ -555,7 +631,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
@@ -563,7 +639,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
// Decompress with dictionary as external
/* Decompress with dictionary as external */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
@@ -571,11 +647,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
@@ -589,7 +661,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
@@ -610,9 +682,9 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
}
}
// Compress HC using External dictionary
/* Compress HC using External dictionary */
FUZ_DISPLAYTEST;
dict -= (FUZ_rand(&randState) & 7); // even bigger separation
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7);
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
@@ -637,31 +709,27 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
// ***** End of tests *** //
// Fill stats
/* ***** End of tests *** */
/* Fill stats */
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
ccbytes += blockContinueCompressedSize;
}
if (nbCycles<=1) nbCycles = cycleNb; /* end by time */
bytes += !bytes; /* avoid division by 0 */
printf("\r%7u /%7u - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
// unalloc
/* release memory */
{
int result = 0;
_exit:
free(CNBuffer);
free(compressedBuffer);
@@ -692,13 +760,13 @@ static void FUZ_unitTests(void)
char ringBuffer[ringBufferSize];
U32 randState = 1;
// Init
/* Init */
FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
// 32-bits address space overflow test
/* 32-bits address space overflow test */
FUZ_AddressOverflow();
// LZ4 streaming tests
/* LZ4 streaming tests */
{
LZ4_stream_t* statePtr;
LZ4_stream_t streamingState;
@@ -706,12 +774,12 @@ static void FUZ_unitTests(void)
U64 crcNew;
int result;
// Allocation test
/* Allocation test */
statePtr = LZ4_createStream();
FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
LZ4_freeStream(statePtr);
// simple compression test
/* simple compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStream(&streamingState);
result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
@@ -722,7 +790,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// ring buffer test
/* ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
@@ -753,10 +821,10 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = crcOrig = XXH64_digest(&xxhNew);
crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// prepare next message
/* prepare next message */
iNext += messageSize;
rNext += messageSize;
dNext += messageSize;
@@ -767,21 +835,20 @@ static void FUZ_unitTests(void)
}
}
// LZ4 HC streaming tests
/* LZ4 HC streaming tests */
{
LZ4_streamHC_t* sp;
LZ4_streamHC_t sHC;
//XXH64_state_t xxh;
U64 crcOrig;
U64 crcNew;
int result;
// Allocation test
/* Allocation test */
sp = LZ4_createStreamHC();
FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
LZ4_freeStreamHC(sp);
// simple compression test
/* simple HC compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
@@ -792,7 +859,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// simple dictionary compression test
/* simple dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 64 KB);
@@ -804,7 +871,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption");
// multiple HC compression test with dictionary
/* multiple HC compression test with dictionary */
{
int result1, result2;
int segSize = testCompressedSize / 2;
@@ -824,7 +891,7 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
}
// remote dictionary HC compression test
/* remote dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 32 KB);
@@ -836,7 +903,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption");
// multiple HC compression with ext. dictionary
/* multiple HC compression with ext. dictionary */
{
XXH64_state_t crcOrigState;
XXH64_state_t crcNewState;
@@ -885,7 +952,7 @@ static void FUZ_unitTests(void)
}
}
// ring buffer test
/* ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
@@ -916,7 +983,7 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = crcOrig = XXH64_digest(&xxhNew);
crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// prepare next message
@@ -929,24 +996,51 @@ static void FUZ_unitTests(void)
}
}
// small decoder-side ring buffer test
/* small decoder-side ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
LZ4_streamDecode_t decodeState;
const U32 maxMessageSizeLog = 10;
const U32 maxMessageSizeLog = 12;
const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
U32 messageSize;
U32 totalMessageSize = 0;
U32 iNext = 0;
U32 dNext = 0;
const U32 dBufferSize = 64 KB + maxMessageSizeMask;
const U32 dBufferSize = 64 KB;
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNew, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_setStreamDecode(&decodeState, NULL, 0);
#define BSIZE1 65537
#define BSIZE2 16435
/* first block */
messageSize = BSIZE1;
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = BSIZE2;
iNext = 132000;
memcpy(testInput + iNext, testInput + 8, messageSize);
if (dNext > dBufferSize) dNext = 0;
while (totalMessageSize < 9 MB)
{
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
@@ -956,76 +1050,22 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = crcOrig = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
crcNew = XXH64_digest(&xxhNew);
if (crcOrig != crcNew)
FUZ_findDiff(testInput + iNext, testVerify + dNext);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test");
// prepare next message
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
iNext = (FUZ_rand(&randState) & 65535);
if (dNext + messageSize > dBufferSize) dNext = 0;
if (dNext > dBufferSize) dNext = 0;
}
}
// long stream test ; Warning : very long test !
if (1)
{
XXH64_state_t crcOrigState;
XXH64_state_t crcNewState;
const U64 totalTestSize = 6ULL << 30;
U64 totalTestDone = 0;
size_t oldStart = 0;
size_t oldSize = 0;
U32 segNb = 1;
DISPLAY("Long HC streaming test (%u MB)\n", (U32)(totalTestSize >> 20));
LZ4_resetStreamHC(&sHC, 0);
XXH64_reset(&crcOrigState, 0);
XXH64_reset(&crcNewState, 0);
while (totalTestDone < totalTestSize)
{
size_t testSize = (FUZ_rand(&randState) & 65535) + 1;
size_t testStart = FUZ_rand(&randState) & 65535;
FUZ_displayUpdate((U32)(totalTestDone >> 20));
if (testStart == oldStart + oldSize) // Corner case not covered by this test (LZ4_decompress_safe_usingDict() limitation)
testStart++;
XXH64_update(&crcOrigState, testInput + testStart, testSize);
crcOrig = XXH64_digest(&crcOrigState);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + testStart, testCompressed, (int)testSize, LZ4_compressBound((int)testSize));
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, (int)testSize, testInput + oldStart, (int)oldSize);
FUZ_CHECKTEST(result!=(int)testSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %u failed", segNb);
XXH64_update(&crcNewState, testVerify, testSize);
crcNew = XXH64_digest(&crcNewState);
if (crcOrig!=crcNew)
{
size_t c=0;
while (testVerify[c] == testInput[testStart+c]) c++;
DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)testSize);
}
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %u corruption", segNb);
oldStart = testStart;
oldSize = testSize;
totalTestDone += testSize;
segNb ++;
}
DISPLAY("\r");
}
}
printf("All unit tests completed successfully \n");
@@ -1042,6 +1082,7 @@ static int FUZ_usage(char* programName)
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS);
DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
@@ -1062,8 +1103,9 @@ int main(int argc, char** argv)
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int pause = 0;
char* programName = argv[0];
U32 duration = 0;
// Check command line
/* Check command line */
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
@@ -1095,7 +1137,7 @@ int main(int argc, char** argv)
case 'i':
argument++;
nbTests=0;
nbTests = 0; duration = 0;
while ((*argument>='0') && (*argument<='9'))
{
nbTests *= 10;
@@ -1104,6 +1146,31 @@ int main(int argc, char** argv)
}
break;
case 'T':
argument++;
nbTests = 0; duration = 0;
for (;;)
{
switch(*argument)
{
case 'm': duration *= 60; argument++; continue;
case 's':
case 'n': argument++; continue;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': duration *= 10; duration += *argument++ - '0'; continue;
}
break;
}
break;
case 's':
argument++;
seed=0; seedset=1;
@@ -1144,7 +1211,6 @@ int main(int argc, char** argv)
}
}
// Get Seed
printf("Starting LZ4 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
@@ -1156,11 +1222,11 @@ int main(int argc, char** argv)
if (nbTests<=0) nbTests=1;
{
int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100);
int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration);
if (pause)
{
DISPLAY("press enter ... \n");
getchar();
(void)getchar();
}
return result;
}

211
src/lz4/programs/lz4.1
View File

@@ -1,66 +1,173 @@
\"
\" lz4.1: This is a manual page for 'lz4' program. This file is part of the
\" lz4 <https://code.google.com/p/lz4/> project.
\" Author: Yann Collet
\"
\" No hyphenation
.hy 0
.nr HY 0
.TH lz4 "1" "2014-02-27" "lz4" "User Commands"
.TH lz4 "1" "2015-03-21" "lz4" "User Commands"
.SH NAME
\fBlz4\fR - Extremely fast compression algorithm
\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files
.SH SYNOPSIS
.TP 5
\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
.PP
.B unlz4
is equivalent to
.BR "lz4 \-d"
.br
.B lz4cat
is equivalent to
.BR "lz4 \-dc"
.br
.PP
When writing scripts that need to decompress files,
it is recommended to always use the name
.B lz4
with appropriate arguments
.RB ( "lz4 \-d"
or
.BR "lz4 \-dc" )
instead of the names
.B unlz4
and
.BR lz4cat .
.SH DESCRIPTION
.PP
\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on
the \fBLZ77\fR family of compression scheme. At the compression speed of 400
MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features
an extremely fast decoder, with speed in multiple GB/s per core, typically
reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports
following options
\fBlz4\fR is an extremely fast lossless compression algorithm,
based on \fBbyte-aligned LZ77\fR family of compression scheme.
\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs.
It features an extremely fast decoder, with speed in multiple GB/s per core,
typically reaching RAM speed limit on multi-core systems.
.B lz4
supports a command line syntax similar to
.BR gzip (1).
The native file format is the
.B .lz4
format.
.SS "Concatenation of .lz4 files"
It is possible to concatenate
.B .lz4
files as is.
.B lz4
will decompress such files as if they were a single
.B .lz4
file. For example:
lz4 file1 > foo.lz4
lz4 file2 >> foo.lz4
then
lz4cat foo.lz4
is equivalent to :
cat file1 file2
.PP
.SH OPTIONS
.
.SS "Short commands concatenation"
In some cases, some options can be expressed using short command
.B "-x"
or long command
.B "--long-word" .
Short commands can be concatenated together. For example,
.B "-d -c"
is equivalent to
.B "-dc" .
Long commands cannot be concatenated.
They must be clearly separated by a space.
.SS "Multiple commands"
When multiple contradictory commands are issued on a same command line,
only the latest one will be applied.
.
.SS "Operation mode"
.TP
.BR \-z ", " \-\-compress
Compress.
This is the default operation mode
when no operation mode option is specified ,
no other operation mode is implied from the command name
(for example,
.B unlz4
implies
.B \-\-decompress ),
nor from the input file name
(for example, a file extension
.B .lz4
implies
.B \-\-decompress
by default).
.B -z
can also be used to force compression of an already compressed
.B .lz4
file.
.TP
.BR \-d ", " \-\-decompress ", " \-\-uncompress
Decompress.
.B --decompress
is also the default operation when the input filename has an
.B .lz4
extensionq
.TP
.BR \-t ", " \-\-test
Test the integrity of compressed
.B .lz4
files.
The decompressed data is discarded.
No files are created nor removed.
.
.SS "Operation modifiers"
.TP
.B \-1
fast compression (default)
.TP
.B \-9
high compression
.TP
.B \-d
decompression
.BR \-f ", " --[no-]force
This option has several effects:
.RS
.IP \(bu 3
If the target file already exists,
overwrite it without prompting.
.IP \(bu 3
When used with
.B \-\-decompress
and
.B lz4
cannot recognize the type of the source file,
copy the source file as is to standard output.
This allows
.B lz4cat
.B \-\-force
to be used like
.BR cat (1)
for files that have not been compressed with
.BR lz4 .
.RE
.TP
.B \-f
overwrite output without prompting
.TP
.B \-h/\-H
display help/long help and exit
.TP
.B \-V
display Version number and exit
.TP
.B \-v
verbose mode
.TP
.B \-q
suppress warnings; specify twice to suppress errors too
.TP
.B \-c
.BR \-c ", " \--stdout ", " \--to-stdout
force write to standard output, even if it is the console
.TP
.B \-t
test compressed file integrity
.TP
.B \-z
force compression
.TP
.B \-l
use Legacy format (useful for Linux Kernel compression)
.BR \-m ", " \--multiple
Multiple file names.
By default, the second filename is used as the output filename for the compressed file.
With
.B -m
, you can specify any number of input filenames. Each of them will be compressed
independently, and the resulting name of the compressed file will be
.B filename.lz4
.
.TP
.B \-B#
block size [4-7](default : 7)
@@ -69,11 +176,37 @@ following options
.B \-BD
block dependency (improve compression ratio)
.TP
.B \-BX
enable block checksum (default:disabled)
.B \--[no-]frame-crc
select frame checksum (default:enabled)
.TP
.B \-Sx
disable stream checksum (default:enabled)
.B \--[no-]content-size
header includes original size (default:not present)
Note : this option can only be activated when the original size can be determined,
hence for a file. It won't work with unknown source size, such as stdin or pipe.
.TP
.B \--[no-]sparse
sparse file support (default:enabled)
.TP
.B \-l
use Legacy format (useful for Linux Kernel compression)
.
.SS "Other options"
.TP
.BR \-v ", " --verbose
verbose mode
.TP
.BR \-q ", " --quiet
suppress warnings; specify twice to suppress errors too
.TP
.B \-h/\-H
display help/long help and exit
.TP
.BR \-V ", " \--version
display Version number and exit
.TP
.BR \-k ", " \--keep
Don't delete source file.
This is default behavior anyway, so this option is just for compatibility with gzip/xz.
.TP
.B \-b
benchmark file(s)
@@ -82,7 +215,7 @@ following options
iteration loops [1-9](default : 3), benchmark mode only
.SH BUGS
Report bugs at:- https://code.google.com/p/lz4/
Report bugs at: https://github.com/Cyan4973/lz4
.SH AUTHOR
Yann Collet

33
src/lz4/programs/lz4c.1
View File

@@ -1,33 +0,0 @@
\"
\" lz4c.1: This is a manual page for 'lz4c' program. This file is part of the
\" lz4 <https://code.google.com/p/lz4/> project.
\"
\" No hyphenation
.hy 0
.nr HY 0
.TH lz4c "1" "2014-04-15" "lz4c" "User Commands"
.SH NAME
\fBlz4\fR - Extremely fast compression algorithm
.SH SYNOPSIS
.TP 5
\fBlz4c\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
.SH DESCRIPTION
.PP
\fBlz4c\fR is the legacy version of \fBlz4\fR.
As such, it supports older supplementary legacy commands.
\fBlz4c\fR is now deprecated.
It is recommended to use \fBlz4\fR instead whenever possible.
To get a list of commands specific to lz4c, do :
lz4c -h
.SH BUGS
Report bugs at:- https://code.google.com/p/lz4/
.SH AUTHOR
Yann Collet

32
src/lz4/programs/lz4cat.1
View File

@@ -1,32 +0,0 @@
\"
\" lz4cat.1: This is a manual page for 'lz4cat' program. This file is part of
\" the lz4 <https://code.google.com/p/lz4/> project.
\"
\" No hyphenation
.hy 0
.nr HY 0
.TH lz4cat "1" "2014-06-20" "lz4cat" "User Commands"
.SH NAME
\fBlz4cat\fR - Utility based on LZ4
.SH SYNOPSIS
.TP 5
\fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE]
.SH DESCRIPTION
.PP
\fBlz4cat\fR is an utility based on \fBlz4\fR, an extremely fast lossless compression algorithm.
\fBlz4cat\fR decompress input file or stream, redirecting its output to the console.
It is equivalent to \fBlz4 -cd\fR,
Available options are the same as \fBlz4\fR ones (man lz4).
.SH BUGS
Report bugs at:- https://code.google.com/p/lz4/
.SH AUTHOR
Yann Collet

175
src/lz4/programs/lz4cli.c
View File

@@ -1,6 +1,6 @@
/*
LZ4cli - LZ4 Command Line Interface
Copyright (C) Yann Collet 2011-2014
Copyright (C) Yann Collet 2011-2015
GPL v2 License
@@ -19,7 +19,7 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 source repository : http://code.google.com/p/lz4/
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/*
@@ -59,23 +59,20 @@
#include <stdlib.h> /* exit, calloc, free */
#include <string.h> /* strcmp, strlen */
#include "bench.h" /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */
#include "lz4io.h"
#include "lz4io.h" /* LZ4IO_compressFilename, LZ4IO_decompressFilename, LZ4IO_compressMultipleFilenames */
/****************************
* OS-specific Includes
*****************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <io.h> /* _isatty */
# ifdef __MINGW32__
int _fileno(FILE *stream); /* MINGW somehow forgets to include this prototype into <stdio.h> */
# endif
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#endif
@@ -85,12 +82,13 @@
******************************/
#define COMPRESSOR_NAME "LZ4 command line interface"
#ifndef LZ4_VERSION
# define LZ4_VERSION "r126"
# define LZ4_VERSION "r128"
#endif
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
#define LZ4_EXTENSION ".lz4"
#define LZ4_CAT "lz4cat"
#define LZ4CAT "lz4cat"
#define UNLZ4 "unlz4"
#define KB *(1U<<10)
#define MB *(1U<<20)
@@ -104,7 +102,7 @@
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned displayLevel = 2; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */
/**************************************
@@ -136,10 +134,10 @@ static char* programName;
#define EXTENDED_FORMAT
#define DEFAULT_COMPRESSOR LZ4IO_compressFilename
#define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename
int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel); /* hidden function */
int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */
/****************************
/*****************************
* Functions
*****************************/
static int usage(void)
@@ -170,11 +168,14 @@ static int usage_advanced(void)
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n");
DISPLAY( " -t : test compressed file integrity\n");
DISPLAY( " -m : multiple input files (implies automatic output filenames)\n");
DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n");
DISPLAY( " -B# : Block size [4-7](default : 7)\n");
DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
/* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */
DISPLAY( " -Sx : disable stream checksum (default:enabled)\n");
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n");
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
DISPLAY( "--[no-]sparse : sparse file support (default:enabled)\n");
DISPLAY( "Benchmark arguments :\n");
DISPLAY( " -b : benchmark file(s)\n");
DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");
@@ -184,7 +185,6 @@ static int usage_advanced(void)
DISPLAY( " -c1 : high compression\n");
DISPLAY( " -hc : high compression\n");
DISPLAY( " -y : overwrite output without prompting \n");
DISPLAY( " -s : suppress warnings \n");
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
EXTENDED_HELP;
return 0;
@@ -251,7 +251,7 @@ static int badusage(void)
static void waitEnter(void)
{
DISPLAY("Press enter to continue...\n");
getchar();
(void)getchar();
}
@@ -261,25 +261,29 @@ int main(int argc, char** argv)
cLevel=0,
decode=0,
bench=0,
filenamesStart=2,
legacy_format=0,
forceStdout=0,
forceCompress=0,
main_pause=0;
char* input_filename=0;
char* output_filename=0;
main_pause=0,
multiple_inputs=0,
operationResult=0;
const char* input_filename=0;
const char* output_filename=0;
char* dynNameSpace=0;
const char** inFileNames = NULL;
unsigned ifnIdx=0;
char nullOutput[] = NULL_OUTPUT;
char extension[] = LZ4_EXTENSION;
int blockSize;
int blockSize;
/* Init */
programName = argv[0];
LZ4IO_setOverwrite(0);
blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
/* lz4cat behavior */
if (!strcmp(programName, LZ4_CAT)) { decode=1; forceStdout=1; output_filename=stdoutmark; displayLevel=1; }
/* lz4cat predefined behavior */
if (!strcmp(programName, LZ4CAT)) { decode=1; forceStdout=1; output_filename=stdoutmark; displayLevel=1; }
if (!strcmp(programName, UNLZ4)) { decode=1; }
/* command switches */
for(i=1; i<argc; i++)
@@ -288,7 +292,29 @@ int main(int argc, char** argv)
if(!argument) continue; /* Protection if argument empty */
/* Decode command (note : aggregated commands are allowed) */
/* long commands (--long-word) */
if (!strcmp(argument, "--compress")) { forceCompress = 1; continue; }
if ((!strcmp(argument, "--decompress"))
|| (!strcmp(argument, "--uncompress"))) { decode = 1; continue; }
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; if (inFileNames==NULL) inFileNames = (const char**)malloc(argc * sizeof(char*)); continue; }
if (!strcmp(argument, "--test")) { decode = 1; LZ4IO_setOverwrite(1); output_filename=nulmark; continue; }
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; }
if ((!strcmp(argument, "--stdout"))
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; }
if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; }
if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; }
if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; }
if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
if (!strcmp(argument, "--keep")) { continue; } /* keep source file (default anyway; just for xz/lzma compatibility) */
/* Short commands (note : aggregated short commands are allowed) */
if (argument[0]=='-')
{
/* '-' means stdin/stdout */
@@ -308,7 +334,6 @@ int main(int argc, char** argv)
if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } /* -c1 (high compression) */
if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } /* -hc (high compression) */
if (*argument=='y') { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */
if (*argument=='s') { displayLevel=1; continue; } /* -s (silent mode) */
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
if ((*argument>='0') && (*argument<='9'))
@@ -353,7 +378,7 @@ int main(int argc, char** argv)
case 'v': displayLevel=4; break;
/* Quiet mode */
case 'q': displayLevel--; break;
case 'q': if (displayLevel) displayLevel--; break;
/* keep source file (default anyway, so useless) (for xz/lzma compatibility) */
case 'k': break;
@@ -372,36 +397,46 @@ int main(int argc, char** argv)
{
int B = argument[1] - '0';
blockSize = LZ4IO_setBlockSizeID(B);
BMK_SetBlocksize(blockSize);
BMK_setBlocksize(blockSize);
argument++;
break;
}
case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break;
case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* currently disables */
case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* currently disabled */
default : exitBlockProperties=1;
}
if (exitBlockProperties) break;
}
break;
/* Modify Stream properties */
case 'S': if (argument[1]=='x') { LZ4IO_setStreamChecksumMode(0); argument++; break; } else { badusage(); }
/* Benchmark */
case 'b': bench=1; break;
case 'b': bench=1; multiple_inputs=1;
if (inFileNames == NULL)
inFileNames = (const char**) malloc(argc * sizeof(char*));
break;
/* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */
case 'm': multiple_inputs=1;
if (inFileNames == NULL)
inFileNames = (const char**) malloc(argc * sizeof(char*));
break;
/* Modify Nb Iterations (benchmark only) */
case 'i':
if ((argument[1] >='1') && (argument[1] <='9'))
{
int iters = argument[1] - '0';
BMK_SetNbIterations(iters);
argument++;
unsigned iters = 0;
while ((argument[1] >='0') && (argument[1] <='9'))
{
iters *= 10;
iters += argument[1] - '0';
argument++;
}
BMK_setNbIterations(iters);
}
break;
/* Pause at the end (hidden option) */
case 'p': main_pause=1; BMK_SetPause(); break;
case 'p': main_pause=1; BMK_setPause(); break;
/* Specific commands for customized versions */
EXTENDED_ARGUMENTS;
@@ -413,29 +448,41 @@ int main(int argc, char** argv)
continue;
}
/* first provided filename is input */
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
/* Store in *inFileNames[] if -m is used. */
if (multiple_inputs) { inFileNames[ifnIdx++]=argument; continue; }
/* second provided filename is output */
/* Store first non-option arg in input_filename to preserve original cli logic. */
if (!input_filename) { input_filename=argument; continue; }
/* Second non-option arg in output_filename to preserve original cli logic. */
if (!output_filename)
{
output_filename=argument;
if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;
continue;
}
/* 3rd non-option arg should not exist */
DISPLAYLEVEL(1, "Warning : %s won't be used ! Do you want multiple input files (-m) ? \n", argument);
}
DISPLAYLEVEL(3, WELCOME_MESSAGE);
if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
/* No input filename ==> use stdin */
if (multiple_inputs) input_filename = inFileNames[0], output_filename = (const char*)(inFileNames[0]);
if(!input_filename) { input_filename=stdinmark; }
/* Check if input or output are defined as console; trigger an error in this case */
/* Check if input is defined as console; trigger an error in this case */
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
/* Check if benchmark is selected */
if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel);
if (bench)
{
int bmkResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel);
free((void*)inFileNames);
return bmkResult;
}
/* No output filename ==> try to select one automatically (when possible) */
while (!output_filename)
@@ -450,9 +497,9 @@ int main(int argc, char** argv)
{
size_t l = strlen(input_filename);
dynNameSpace = (char*)calloc(1,l+5);
strcpy(dynNameSpace, input_filename);
strcat(dynNameSpace, LZ4_EXTENSION);
output_filename = dynNameSpace;
strcpy(output_filename, input_filename);
strcpy(output_filename+l, LZ4_EXTENSION);
DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
break;
}
@@ -461,12 +508,12 @@ int main(int argc, char** argv)
size_t outl;
size_t inl = strlen(input_filename);
dynNameSpace = (char*)calloc(1,inl+1);
output_filename = dynNameSpace;
strcpy(output_filename, input_filename);
strcpy(dynNameSpace, input_filename);
outl = inl;
if (inl>4)
while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0;
while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0;
if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); }
output_filename = dynNameSpace;
DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
}
}
@@ -474,28 +521,40 @@ int main(int argc, char** argv)
/* Check if output is defined as console; trigger an error in this case */
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage();
/* No warning message in pure pipe mode (stdin + stdout) */
/* Downgrade notification level in pure pipe mode (stdin + stdout) and multiple file mode */
if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((multiple_inputs) && (displayLevel==2)) displayLevel=1;
/* IO Stream/File */
LZ4IO_setNotificationLevel(displayLevel);
if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename);
if (decode)
{
if (multiple_inputs)
operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION);
else
DEFAULT_DECOMPRESSOR(input_filename, output_filename);
}
else
{
/* compression is default action */
if (legacy_format)
{
DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
}
/* compression is default action */
if (legacy_format)
{
DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
}
else
{
if (multiple_inputs)
operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel);
else
{
DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
}
DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
}
}
if (main_pause) waitEnter();
free(dynNameSpace);
free((void*)inFileNames);
if (operationResult != 0) return operationResult;
return 0;
}

873
src/lz4/programs/lz4io.c
View File

File diff suppressed because it is too large Load Diff

30
src/lz4/programs/lz4io.h
View File

@@ -1,6 +1,6 @@
/*
LZ4io.h - LZ4 File/Stream Interface
Copyright (C) Yann Collet 2011-2014
Copyright (C) Yann Collet 2011-2015
GPL v2 License
This program is free software; you can redistribute it and/or modify
@@ -18,7 +18,7 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 source repository : http://code.google.com/p/lz4/
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/*
@@ -29,17 +29,18 @@
- The license of this source file is GPLv2.
*/
#pragma once
/* ************************************************** */
/* Special input/output values */
/* ************************************************** */
#define NULL_OUTPUT "null"
static char stdinmark[] = "stdin";
static char stdoutmark[] = "stdout";
static char const stdinmark[] = "stdin";
static char const stdoutmark[] = "stdout";
#ifdef _WIN32
static char nulmark[] = "nul";
static char const nulmark[] = "nul";
#else
static char nulmark[] = "/dev/null";
static char const nulmark[] = "/dev/null";
#endif
@@ -47,10 +48,13 @@ static char nulmark[] = "/dev/null";
/* ****************** Functions ********************* */
/* ************************************************** */
int LZ4IO_compressFilename (char* input_filename, char* output_filename, int compressionlevel);
int LZ4IO_decompressFilename(char* input_filename, char* output_filename);
int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel);
int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename);
int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel);
int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix);
/* ************************************************** */
/* ****************** Parameters ******************** */
/* ************************************************** */
@@ -67,11 +71,17 @@ int LZ4IO_setBlockSizeID(int blockSizeID);
typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t;
int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode);
/* Default setting : no checksum */
/* Default setting : no block checksum */
int LZ4IO_setBlockChecksumMode(int xxhash);
/* Default setting : checksum enabled */
/* Default setting : stream checksum enabled */
int LZ4IO_setStreamChecksumMode(int xxhash);
/* Default setting : 0 (no notification) */
int LZ4IO_setNotificationLevel(int level);
/* Default setting : 0 (disabled) */
int LZ4IO_setSparseFile(int enable);
/* Default setting : 0 (disabled) */
int LZ4IO_setContentSize(int enable);