mirror of
https://github.com/Xahau/xahaud.git
synced 2025-12-06 17:27:52 +00:00
Squashed 'src/lz4/' changes from baf78e7e4..c10863b98
c10863b98 fix : asan error in dctx, due to increased maximum frame header size, reported by Craig Young 1e92bb0af Merge pull request #382 from lz4/installVars edb434365 updated lz4frame manual a1f3a27e9 Merge branch 'dev' into installVars and fixed conflicts af9d72b7f Merge pull request #383 from lz4/blockChecksum 8593ba883 Merge pull request #380 from lz4/dictID 01cdbfb5f lz4c legacy commands are now enabled at runtime based on link/binary name "lz4c" 731cff120 fix minor markdown display issues aea7d521a better respect GNU standard Makefile conventions 930a69211 cli : restored command -BX to enable block checksum (#322) 77f99d292 restored block checksum capability at lz4frame API level 228e26ee6 Merge pull request #378 from deymo/dev 757497ae3 implemented lz4frame decompression API 4531637ec support dictionary compression with independent blocks 8d597d62d fixed gcc prototype warning d8aafe2c5 dictionary compression correctly uses compression level ca2fb166a fixed C++ conversion warnings bf8daa2fd fixed uninitialization error in lz4frame 31f2cdf4d implemented dictionary compression in lz4frame 1d1737aaf fixed frameCompress example a82dadfba added dictID inside LZ4F_frameInfo_t e98a52857 updated Frame specification 76ef6d0ab Allow to predefine FORCE_INLINE macro. 9a967030d Merge branch 'dev' of github.com:Cyan4973/lz4 into dev c5a015bc3 Merge pull request #376 from ido/patch-1 4891f909e Merge branch 'dev' of github.com:Cyan4973/lz4 into dev dba4d70a1 updated man page 8aeaf6bb3 Fix typos preventing installation of static lib. ac5ed8e4b Merge pull request #374 from ferdnyc/patch-2 63d500568 Generate updated man page from Markdown source ef0655510 Merge pull request #373 from ferdnyc/patch-2 872f767fe Fix formatting of concatenation example 6ad3a983d fix #369 2ef4afeeb Merge pull request #371 from jku/tests-LIBDIR e14b4c5a3 tests/Makefile: don't use LIBDIR as variable 1525fd1f5 Merge pull request #367 from Chocobo1/fallthrough 7982a905e Merge pull request #368 from Chocobo1/readme 9d453a188 Update README.md a4ec7fa7b Fix gcc7 Wimplicit-fallthrough warnings 89b9f026c made level 10 a bit faster a6fd0f9d0 -g compilation flag not by default for lz4 cli bfc180660 clarified lz4frame api comment (#350) d18084c65 report where decompression ends (#313) 139a387d1 updated NEWS 0beaa356f fixed minor scan-build warning 775e63ee0 refactored simple_buffer.c example (#363) 7e15e240a added a paragraph on overlap matches 03d8586fc cli accept block sizes with KB / MB prefixes 8c355d17f Merge pull request #360 from Chocobo1/md 1c9d4091d [Doc] Fix markdown 6614be7cc Merge pull request #359 from PierreNav/dev 4e359f5c3 Add DLL files to the INSTALL target 5c97cdfa0 Merge pull request #352 from lz4/resetDCtx 2600a154b fix (minor) g++ compatibility for frametest e60cbb5ca added test for LZ4F_resetDecompressionContext() 2012e4de9 fixed c_standards tests a90212761 bumped version number to 1.8.0 b8575f2d2 updated Makefile fe932c452 expose LZ4F_resetDecompressionContext() 1efa48831 minor readability changes a8dd86d93 changed macro HEAPMODE into LZ4_HEAPMODE 11bfedb6c Merge pull request #349 from svpv/master 6cf1f7d75 lz4hc.c: clamp compression levels > 12 a41df535f lz4cli.c: fix a comment: LZ4HC_DEFAULT_CLEVEL -> LZ4HC_CLEVEL_MAX f3460fc14 liz4hc.h: fix a comment: LZ4HC_MAX_CLEVEL -> LZ4HC_CLEVEL_MAX 37ef330cd Merge pull request #348 from terrelln/deprecate 0b3e80704 [LZ4F] Allow users to disable LZ4F_DEPRECATE 31e9ed612 Merge pull request #347 from lz4/negativeCLevels e2c9b1912 lz4frame : Added negative compression levels 7eecd32c0 ensure lz4f_cctx internal buffer size remain valid in case of malloc error e169edac0 lz4frame : control lz4 context creation success e2827775e make __packed memory access default for gcc 3d4ee35da Merge pull request #344 from lz4/LZ4F_getFrameInfo 6226d52ea fixed minor Visual warning b88df6b1b Improved comments on LZ4F_getFrameInfo() fc31257ab added LZ4F_resetDecompressionContext() f0a7651fc Safer LZ4_getFrameInfo() ab547a0ef Merge pull request #342 from iburinoc/isatty 9b1f00056 Fix inline compile errors afde27acf Fix IS_CONSOLE returning 1 for NUL on windows 84246b974 Merge pull request #341 from iburinoc/exematch 4567d0def Merge pull request #340 from lz4/optlz4opt e9c3b14f2 Ignore extensions in exe name matching 38362c1d6 Merge pull request #338 from ldv-alt/dev 60b6d2907 improved level 10 speed for degenerated cases 8f8fc52fc Merge branch 'dev' into optlz4opt 0863931a1 fixed API comment for LZ4F_createCompressionContext() dab3590fc Export deprecated symbols declared in lz4hc.h 13f5cb2d5 Merge pull request #337 from ldv-alt/dev b1daffc4e cli: add GNU separator -- specifying that all following arguments are files 68044acfe Merge branch 'optlz4opt' of github.com:Cyan4973/lz4 into optlz4opt b8bc70022 minor refactor cea8f60df slight btopt speed improvement f513020a6 slight btopt speed improvement 8bd32a17b made SET_PRICE macro more usable 118b47f3d improved lz4opt speed (~4%) eff6166eb minor price function optimization 0d073d4d2 added `extern C` for lz4.h static section aae447fff LZ4_compress_HC_continue_destSize() works as intended up to level 10 cd35f0d98 LZ4_compress_HC_destSize() uses LZ4HC_compress_generic() code path 6b2a1b360 updated lz4 version number 430b7d32b created LZ4_HC_STATIC_LINKING_ONLY section 9144520dc fixed Visual compilation error a33bf89c6 fix #332 : do not modify /dev/null permissions e945a27f9 fix #333 : expose obsolete decoding functions e1b2b370e added Oleg @remittor contribution b9132bedc Merge pull request #331 from ldv-alt/dev 50fe87f13 Merge pull request #329 from remittor/dev-hc2 883ebdcee Export only those symbols that are part of public API 25b243588 Export deprecated symbols 7aeecbff7 Explicitly set visibility of public API functions when gcc is used 93b3b08ef Merge pull request #330 from inikep/dev cb1671955 bench.c: respect LZ4_MAX_INPUT_SIZE limit 517d446d7 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 36842ebb1 lz4hc: Fix LZ4HC_compress_hashChain for backward compatibility baa155088 lz4hc: Fix LZ4HC_compress_hashChain for full support destSize variant 66b26a389 tests: fuzzer: Add test for LZ4_compressHC_destSize f007153e3 lz4hc: Add LZ4_compressHC_destSize and LZ4_compress_HC_continue_destSize 45b592b7e lz4hc: Cleanup function LZ4HC_encodeSequence 534f8fa5d lz4hc: Cleanup function LZ4HC_compress_hashChain 9141ada8d Merge pull request #328 from ldv-alt/dev 0dfb0b9da Fix test-lz4-basic 97df1c978 updated LICENSE 5bfdff96b Merge pull request #320 from inikep/dev d3eaf7883 remove MOREFLAGS from test-lz4c32 5bb6ba794 Merge pull request #319 from inikep/dev ca2c5fd3c CFLAGS=-I/usr/include/x86_64-linux-gnu ace22424d circle.yml: fixed lz4c32 target (2) 0d77c85d9 circle.yml: fixed lz4c32 target 24007bb4e Merge pull request #318 from esiegerman/eks/mklib 4ee9bd5c3 improved Circle CI tests c0a8d0ad8 added circle.yml 3ec1591db Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 68a358769 Avoid fseek()'s 2GiB barrier with MinGW 40ad1e85d upgraded util.h eb51b2b8d Explicitly create $(DESTDIR)$(LIBDIR)/ at install time 69dbafc1c Merge pull request #317 from esiegerman/eks/no-andand-oror b89cac7b2 Don't use "foo && false || true" 7e57c398c Merge pull request #316 from inikep/dev f4b9c1d03 Merge pull request #315 from bebehei/patch-1 c139eb40b added OpenBSD NetBSD SunOS to list of supported `make install` OSes 5e1a925be update repolink in makefile 272699a4e Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev c7b14967a updated platform.h 4f261b828 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 04e5eaf9b added DragonFly to list of supported `make install` OS 04ec09269 updated comments on block sizes 7ed257191 Merge pull request #312 from mgorny/cmake-pc-version 96e150e7d [cmake] Fix substituting version in liblz4.pc 67abd75e6 Merge pull request #307 from inikep/dev 8e1fd97d1 lz4frame.h: added Introduction d56ee32b3 updated gen-lz4-manual.sh f54c7e0e7 added lz4frame_manual.html 9071df0fa lz4frame.h: prepared to generate manual 6a5633e99 lz4 manual updated to v1.7.5 f9f48f8ed lz4.h: improved manual generation 128244371 improved gen_manual 489d1618c added "This Makefile is validated for" 339e79320 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev ca083dd4d Merge pull request #304 from Chocobo1/cppcheck db6f733ec Use logical or instead of bitwise or 44f95e92e Fix printf specifier 9683a1ae8 LZ4_MEMORY_USAGE can be modified from compilation command line 16d69c8d0 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev 05e27ade6 updated NEWS c18a81c02 Merge pull request #296 from mgorny/cmake-fixes d7969e49a cmake: Install manpages 1380c33b7 cmake: Install lz4cat and unlz4 symlinks 28db4acc9 cmake: Support building shared & static libs simultaneously 9e867db90 cmake: Fix SOVERSION to match Makefiles cb9599449 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 8c4de60d0 Merge pull request #295 from inikep/opt-parser d2b51c22d fuzzer: tests more compression levels 312d88249 removed nextToUpdateBT 3d5bb3897 clean logging 1c80b9af4 LZ4HC_getSearchNum 581da2bf8 Merge pull request #293 from inikep/dev 98f9d6c72 improved logging 12aae846b fixed table update in LZ4_loadDictHC c8b31263c added FUZ_CLEVEL_DEFAULT 7a73c5c1f changed default PREFIX and MANDIR d57ff6456 changed default PREFIX and MANDIR a3d61cf3d improved test-lz4-opt-parser 20381a2fc fixed -BD compression f4575f4f1 added test-lz4-opt-parser e6536faf3 lib\Makefile: fixed INSTALL_DATA efaa9b7bc Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev b22e2bed5 BSD: improved "make install" ea51ad0ce Solaris: working "make install" 19c0f21b0 updated Makefile : lz4 no longer recompiled when already up-to-date 9b4b081c1 "make test" is now compatible with Solaris b41f9bb13 Merge pull request #292 from inikep/dev 7eb16d97d updated NEWS 52cac9a97 updated a few macros names fea95c156 use FindFirstFileA and FindNextFileA on Windows e6af952f8 improved comments 581245063 use fseeko for 32-bit MacOS f084b6538 test Large File support for Mac OS-X in 32-bits mode 973bc7974 util.h and platform.h based on zstd 70526a11e fixed lib/clean bc0839c5d minor fix for travis-install test 7cf0bb97b LZ4F_compressBound(0) provides upper bound for LZ4F_flush() and LZ4F_compressEnd() [#290, suggested by @vtermanis] 385cb4f53 minor update Makefile 4f61505cc datagen.vcxproj: CharacterSet is MultiByte 253ac12cb VS projects: CharacterSet changed to MultiByte 287f7d3d6 added IncludePath 2fd7eb554 fixed Visual Studio compilation 3c6ce9c66 fixed gcc 4.4 support 64cbc4e1d improved MinGW support 9546ba62d executables use basic types from util.h 452cc970d reordered definitons in platform.h 090cb887a improved formatting (2) e1332241a improved formatting 5f0cc8e2d fixed gcc warnings (2) 2efedaf7d fixed gcc warnings 6adf05d1e improved platform.h 2cfef1395 executables use platform.h bb69cdf25 updated util.h 45b14ab8e added platfom.h 913b98fd6 fix `make` concurrency build (#277) fec57e49e Merge pull request #287 from inikep/dev 4f050b68d tree update dependent from compression level 838ed341f fullUpdate as a parameter b3ecc3e3e last match starts at least 12 bytes before end of block d694bf911 faster opt compression e7c42d757 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev a22e71d4a full binary tree update 626552671 updated NEWS de93e9e5d fix #285 : lz4cat doesn't work with stdin (reported by @beiDei8z) fb6c98c85 slightly improved lz4opt.h 20183ad7a Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 0280cf40d fixed path-1, as suggested by @t-mat 7e2d4c4c0 fixed test-lz4c32 750a50cf6 fix #284, reported by @beiDei8z : added test cases for path/to/lz4cat 01ffed25a fix : invocation of path/to/unlz4 (#284), reported by @beiDei8z 8c2f23c13 fixed LZ4_STREAMHCSIZE 76228f064 LZ4HC_DICTIONARY_LOGSIZE 17 8bfa42e61 improved ratio 128903824 15-bit LZ4HC_HASH_LOG 410ba53db improved LZ4HC_BinTree_GetAllMatches 1833be1cf fixed gcc warnings f2ebf37bf slightly improved ratio 77b051ed7 improve code formatting e3fee9474 LZ4HC_MAX_CLEVEL = 12 757ef1d5f removed LZ4HC_GetAllMatches cfe6a24b0 updated LZ4HC_get_price 2113ead17 17-bit LZ4HC_MAXD c1ef7a177 introduced LZ4HC_compress_hashChain 3f430daf7 added lz4opt.h 5ae0e789f Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 5eb547a60 fix #283 : implement LZ4_versionString(). 9a6f7dfc5 updated NEWS a9a8dbf7c fixed minor display error when using -r 6b90cfbea Updated man page 009f06e01 minor man page update b3bc054d0 Merge pull request #279 from t-mat/md-to-man 544290d58 Add "man" target to Makefile 018ddf799 numbering 605512a7b updated NEWS f4c74e6fa library release build compatible with environment variable edc132b69 lz4 release build compatible with PIE 7819d5878 new test case with fPIE de0cf5de6 fixed LZ4IO_compressMultipleFilenames 65a542579 updated README.md 2cbd0233b Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev cc21115cb minor warning fix 8875e7dbb fix minor analyzer warning 8b233b228 added a few dates ba7a85b46 fixed minor analyzer warning 3bde02a05 added gcc 4.4 test 2fe3aa985 added examples to make all 742f2b683 added doc in code comments 2dbcebfda changes travis test order b4db9d7e3 Add C++14 standard [[deprecated]] attribute a38cbf9f6 fixed : shadow global variable on gcc 4.4 (minor) 19df3029e fixed #272 (compilation fails on gcc 4.4), reported by @totaam 519932afa attempt -mx32 tests on Travis CI a91983daa highly improved speed on -mx32 mode Now -mx32 is fastest mode on x64 CPU 6cabcd151 removed usan32 tests. Let's summarize : gcc-5 and gcc-6 on Travis do not support UB sanitizer. And clang on Travis is not able to generate -m32 code. Great. It makes the combination impossible to test on Travis ... :( 7a22904a7 moved usan32 tests to gcc-6 ffc8c18c1 usan32 relies on default compiler (by default) a4f4bf21e fixed gcc-5 conf 71814a26f switch usan32 tests to gcc-5 8e1ea59ad attempt to fix sanitize32 package dependency 8f536a43b added sanitize32 tests to Travis 3f6f57768 fixed strict aliasing warnings in fasttest c0e55ec05 fixed versionsTest 7d5354847 added v1.x.y to versions tests 279344a17 fixed Travis test cases 1abecbc33 fix 32-bits mode. Large File support for Mac OS-X in 32-bits mode Fixed potential undefined behavior Changed makefile for 32-bits mode 7fde7438d Merge branch 'dev' of github.com:Cyan4973/lz4 into dev a22ca9378 updated README for v1.7.3 e9bec0fdc Merge pull request #271 from inikep/dev b1519f861 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 2dee7bdaf include NEWS in lz4_x64.zip and lz4_x86.zip 632ab3b8d updated programs\README.md 8b48d5e44 updated example\README.md 11f6217cc created dll\example directory 873206e70 appveyor.yml: create only lz4_x64.zip and lz4_x86.zip 7b6a9d863 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 2e10aab20 MinGW doesn't require the import library at all 7cecc7b08 fixed a limit case scenario 00fa1b9c0 fixed double ;; for Visual 929badd61 fixed sign conversion warning d613e2494 restored .travis.yml d2be69b14 fixed minor coverity warnings 8c32a12f1 .travis.yml for coverity 2eddc8ea2 removed .travis.yml, for better rebase 2906c373c fixed appveyor badges 0d6dc079f update appveyor tokens b44790cf9 Merge pull request #270 from inikep/dev 4235e5e25 updated visual\README.md 257b65a1d ToolsVersion="4.0" 200746427 removed _x64 and _x86 e583dd056 ImageHasSafeExceptionHandlers=false eb160e1a2 appveyor.yml: added liblz4_x86.zip 45b8623af updated lib\dll\README.md 5625cc21e appveyor.yml: generate DLL package c093e256d added files used by DLL package be471c69d added fullbench-dll.sln that works with gcc/MinGW f71340ce1 DLL dependencies moved to lib/dll/ f3f59bef4 updated appveyor.yml b0e994cc5 upload executables to GitHub d84a80e66 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 923cf4447 bump version number 4c5c711b7 fixed minor conversion warning f094f5314 silence a minor msan warning 0fe5c187a fixed minor msan warning 201149f8e fixed conversion warnings fa158b411 fixed nameless union warnings 84af32d49 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev 5a5ebeec1 clang and g++ tests restricted to native only on Travis CI Ubuntu v12 environment 569897ddd Merge pull request #269 from inikep/dev 1b24cc115 fixed conversion warnings 9ad7508db Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev ecc55d19b fixed uintptr_t on pre-c99 compilers 3d3ccc154 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev d5c9b39dd minor code refactoring 4bd9db1dd Merge pull request #268 from nemequ/dev f42130ee4 cli : add --help command 84b97f90b fuzzer code refactoring fix g++ conversion warning 151021a3a cmake: add public domain dedication, and a note about the maintainer b6bcb84ea cmake: make PIC optional when generating a static library 94542d845 made lz4hc explicitly dependent on lz4 bb97e8a6f fixed __GNUC__ macro cb2230826 fixed fuzzer error 3580d9698 enabled deprecation warnings on remaining obsolete functions 874f3e095 update code comments e528a82f0 Merge pull request #267 from terrelln/strict-aliasing 3d456bdf2 fixed conversion warnings bf6dda69e updated frametest 85aeb0e4b Expose internal types to remove strict aliasing dbfdd5131 Merge pull request #266 from terrelln/frame-ex 7aad356ff Add frame decompression example ec6fb477b added visual\README.md 9d075580d Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 4f9db1383 updated xxhash lib c72d2f5b1 minor typo 7d2e4b3cf Merge pull request #265 from terrelln/rm-zstd 5b37837e6 Fix license and remove references to zstd 1f246a989 Fixed #178 fullbench on small input 5e13a6ec4 improved man on compression level f9c9bb364 Merge pull request #263 from terrelln/dict-ex cd4462629 Merge pull request #264 from inikep/dev 52251d970 Using MinGW+MSYS to create DLL 72a4a84df udated README.md 2ecce6f15 lz4.h: added Introduction section 93e2477ca added lz4_manual.html a340b734b added contrib/gen_manual 2278d1f02 custom block size d57d3e4b4 better MSYS/MinGW support 74f127787 NB_LOOPS ?= -i1 cf8ca581d reordered Travis CI tests 767a23160 changed FUZZER_TIME 6a160b72e Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 924f00c86 fix empty body 94917c9a0 Add dictionary random access example f34808e4a fixed some static analyzer warning 46f74d79b fixed sparse on stdout fe48cef50 fixed multiples files on lz4cat (#184, by @libor-m) 1b1680594 fix unused variable warning 1b95d60c6 fixed synthetic benchmark mode slightly optimized test mode acc48a3f8 fixed lz4cat changed lz4-test order 0c5e2e4d1 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev fafba3973 updated NEWS 2987eb417 Merge pull request #262 from inikep/dev 063b4d92e fixed make install 773ae50b4 DLL exports only functions defined in liblz4.def ca80a7c8f create DLL with MinGW/MSYS 67fd5dffd updated appveyor.yml c794829a2 update .gitignore 9e7fe1062 removed <PlatformToolset> dd14dca15 LZ4_DLL_IMPORT 325babddf added lz4.rc ef2fdfe3d rename liblz4-dll to lz4-dll 7df658ec1 removed LZ4_DLL_EXPORT=1 (2) 4a6e7327c removed LZ4_DLL_EXPORT=1 bde0d340b added fullbench-dll.vcxproj da48dbb57 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev fe5148417 introduced LZ4_COMPILED_AS_DLL d5903c8e5 Merge pull request #261 from terrelln/clean-fasttest d594d9d87 Remove fasttest when cleaning bd88e4007 Merge pull request #260 from nemequ/dev 661c48040 cmake: significant changes to accomodate a new "bundled" mode. ce2202690 Merge pull request #259 from inikep/dev 96ea2e363 updated IntDir 39157c9f3 added lz4-dll project ec63747de VS project: added IntDir c6fd78f4f VS project: added setargv.obj 1d22fae1e added test-lz4-speed.py 85f861fc4 bench.c: support notification level 2c788b372 joined arm and aarch64 tests 0a22bf7d6 err0r(LZ4F_errorCodes code) uses ptrdiff_t d06ab4f41 test powerpc64 230264913 tests\Makefile: added test-platform c201327f1 added platformTest 0f73ed572 new Travis tests 90981a35c Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev c7ab95faa slightly improved HC compression ratio e9089bcbe small compression ratio and speed improvement on small files 00c6370ad Fix #243 : detect and report fread() errors, by @iyokan f6a7c6b57 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev 301c4e94e Merge pull request #258 from inikep/dev2 839085f05 Merge pull request #257 from inikep/dev 9b3dcb325 MOREFLAGS="-Werror" 7fbb163f3 added "make c_standards" b6c407886 condensed travis.yml 2fbd7a512 updated travis.yml ef54f9f9d added --rm option d61bb74f2 updated declaration of LZ4F_freeDecompressionContext 7a98db266 added datagen.c to lz4.vcxproj 4b29af350 TreatWarningAsError=true only for debug 90d4401fe fixed appveyor.yml (3) 434d1230e fixed appveyor.yml (2) 58ead8dd2 fixed appveyor.yml 98be00847 Copy owner, file permissions and modification time 158156729 util.h requires CharacterSet=MultiByte ef9d1b2d4 fixed --test 112499a73 Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 58d7a619a Merge pull request #254 from terrelln/dev 920bf2171 Fix LZ4_decompress_fast_continue() bug 207f478ba minor refactor 8195ba8f7 Updated man page 96565816b restricted deprecation warning to fairly recent gcc versions (>=6.0) eaad740ac lz4frame obsolete enum should trigger deprecation warnings (on supported compilers) 079d5dd54 removed support for old error code enums from lz4frame_static.h, to free up namespace. note : lz4frame_static.h does not guaranteed API stability. note 2 : the macro to enable old error code enums is still present. Just needs to comment one line to re-enable them. 473acf5aa Merge pull request #252 from inikep/dev 10c5d5eee Merge pull request #253 from terrelln/exit-code 86a24c80f Fix typo in lz4 manpage f30c56c08 Quiet gcc-4.6.3 narrowing warning 374090c7a Return error if input file does not exist. fbede33fd fixed Travis tests 8ddaddc2d updated #include in util.h f878c08b7 better correctness on big-endian 64-bits platforms 26dec498c small compression speed improvement on 64-bits systems e8a6067e8 updated lz4cli.c 8b8c726a5 bench.c based on zstd e06fcd903 redefined VOID for Windows/MSYS e7648f4fc added -r option 6ebf8859e Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 4c496faba Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 6d6a3e0fb Merge pull request #250 from terrelln/test-mode 4c6610709 minor code refactoring 136caa552 Rename passThrough to testMode eda8c4eb1 fix #198 : no longer requires to restart streaming decompression from where it stopped. It permits relocating remaining data into another memory space. Still, the same content must be presented. 04d728b87 Fix tests for test mode 4791cda04 fixed strict warnings 258a5e7fa updated comments 3dcafd35d Fix test mode and write to null 6f68a1692 fixed travis badges 99d11120b Merge pull request #249 from inikep/dev 46b49af37 fixed clang conversion warning c66108ceb check fread() operation results (#243) 8847faffb updated programs/README.md f38adde9e versionsTests moved to tests/ 74ba377d7 updated .travis.yml 9f15a4c6e updated Visual Studio projects 84cedb463 updated links to LZ4 repository e624978ea test programs moved to tests/ 88cef262e documentation moved to doc/ b768da8ad Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev 59adfd82d fix for ARM platform 67500142c Clarified license (#115, #244) f17302769 moved cmake and debian directories to contrib (#245) d05ca2ff6 fixed #247, reported by Felix Bolte 7e112cfa5 fixes static compilation under Windows (#246) e64a92692 Merge pull request #242 from vapier/dev 92408d338 README: fix minor typo 52cdb4a5f Merge pull request #241 from vapier/dev c5cb4aa52 Merge pull request #238 from kou/add-lz4lib-api 544abaced Merge pull request #240 from kou/add-missin-ext 43b631223 add a flag to disable static libs 829941e5a Add missing $(EXT) to lz4 program name 59771f41c Use different prefix for each file to avoid conflict 1ca408105 Merge pull request #239 from kou/use-lower-case-for-header-name 012c66536 Add LZ4LIB_API 46a7b94e8 Use lower case for header file name 3b9439ccd updated uninstall 8eae1320c Merge pull request #236 from inikep/dev 33993542a removed test artefacts 3eb143fff fix bug #232 at https://github.com/Cyan4973/lz4/issues/232 c8a78dba1 datagen depends on lz4.h for version string 53515b05e Merge pull request #235 from inikep/dev 92ee3e0ea fixed LZ4 repository links 39be6f16a versionTests require 32-bit gcc libraries 122fdd13a versionTests require python3 6adf7392a updated .travis.yml 8ac619afe Merge pull request #233 from inikep/dev 58a1616d7 Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev c77fdb44f disabled <RunCodeAnalysis> for Win32 28f98647e lz4 version source from lz4.h dd51ca630 frametest depends on standard C time.h only f327a8bdd fullbench depends on standard C time.h only 32a31663c fuzzer depends on standard C time.h only 64f556e61 lz4cli version number from lz4.h 5871585b6 coding style change e5b0f1ea1 CFLAGS += $(MOREFLAGS) 829ae7587 remove EnablePREfast=true for Win32 547e1943c set OutDir ab707e8b0 update IncludePath 3a7b4457a restored lz4.vcxproj 81d90609b added appveyor.yml c200f7e09 renamed VS project f4111cf04 Merge pull request #231 from inikep/dev b3ae49c7b added <string.h> for strlen() 2cf32ef0f -b# and -e# options from zstd c1864f69e fix mingw64 compilation warnings 7f08131f9 Merge pull request #226 from gsauthof/port-make 3fa60044e don't use variable eval syntax in if() e1e3305db don't link the tools against the shared library b0d42cb7e remove additional -fPIC switch because 681d5010d use gnu c99 to get both c99 and POSIX 584e1fe92 request POSIX.2001 definitions for fileno() 5bd3eaa48 minor comments 9578ff539 minor refactor a7672dc4f minor decompression speed gains 182645ee0 lz4 as phony target 0214638eb Makefile : lz4 at root 63d22e7a1 minor comments clarifications 74e6afc5d minor comments clarification 2ac9ecec6 Fixed #157 : LZ4F_getFrameInfo() fails on valid null-content frame 45ea27949 Minor fuzzer code refactoring de69eca47 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev 828332d23 fixed : fuzzer test, reported by @setharnold (#203) b130fd592 minor refactoring changes 3c295ca4a Merge pull request #222 from kou/fix-build-error-with-mingw 0c63b7cbd Fix build error with MinGW on Linux 7316584ca Merge pull request #219 from BenWiederhake/dev d3e2d8026 Use https wherever possible cec38d1b7 Correctly reference self (github, not googlecode) 654003003 Use correct 'repository' hostname 91cce757f Updated xxhash library to v0.6.1 5540f4f93 stronger compilation flags 19e585cc4 modified : some more deprecated functions from lz4hc.h generate warnings 6b1600f41 changed : default benchmark LZ4_decompress_safe() 6923f3e23 Minor refactoring (code style) 410ec8154 Fixed : alignment warning bc342ab94 minor refactoring c8a573d32 very minor decompression speed gain a5508b1ad fixed display result issue when benchmarking multiple files e27e7316d minor code refactoring 2d7df8b30 Reduced bench dependency to standard C <time.h> Faster tests More accurate results on systems with high resolution clocks 8938e1074 minor compression speed improvement c77c0c108 minor refactoring 3c0332600 minor refactoring (coding style) 1f47f3f74 Merge pull request #208 from Nimloth/dev 7bfbccb97 Add FreeBSD to install targets 5bb5d6ee1 Merge pull request #204 from irwand/dev bc70a4b72 fixed non-C99 compilers 10e3eecc5 fixed non-C99 compiler 973baa2c1 Merge branch 'heapmode_alloc_failure_fix' into dev 3e5bb6753 alloc failure fix 8a42ee10a Merge pull request #201 from cernekee/cmake-pc 4bda09fc8 cmake: Install liblz4.pc 8b562a4f3 Merge pull request #197 from Cyan4973/fix-readme-travis 5cdf9a657 Fix wrong link for Travis 220e0b606 Merge pull request #194 from jzhuge/dev cf6652f9b lz4cli: print library version 60ba8638c Merge branch 'dev' of github.com:Cyan4973/lz4 into dev 219da951d Added zstd 2995a45e5 Merge pull request #185 from jwerner-chromium/dev dc868cd5b Allow for safe in-place decoding 4fcb2e17f Remove whitespace from ends of lines ad564c872 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev db53bbb94 updated hyperlink b84eb3349 Merge pull request #182 from kou/add-missing-ext b8abffc39 Merge pull request #181 from kou/fix-build-failure-on-mingw ab3e1d253 Add missing $(EXT) 14b40fb6d Fix build error on MinGW 62bbd102f Merge branch 'dev' of github.com:Cyan4973/lz4 into dev 2785eeedf clarified man page as suggested in #170 631dc8f2b Merge pull request #173 from flandr/liblz4 d67ae8fe4 Merge pull request #172 from psteinb/add_static_cmake 4c9d851eb added static build target for OSX and Linux to cmake 495ea43d9 Merge pull request #162 from KyleJHarper/20151021_examples 7aa004377 Forgot to make clean before commiting. Removing binary. 7f96babc4 Several changes to address a few concerns from Yann. See Google Group LZ4c topic 'Reusing compression/decompression resources' for details. d1be164e4 Added a note about why char* is used. Also removed a binary that wasn't supposed to be in there. 0d2518567 Finished the two example files. Decided to avoid adding anything to lz4.c/h to expose LZ4_compress_generic(). 02be6631b Took out the basics and placed them into basics.c. Added decompression and a wrapper for the generic call. I will likely break this file up into 2 examples before submission. 67b3a2470 Final tests and reporting are done. As expected there isn't much to be gained by jumping the chain. In most of my tests I did see a moderate performance gain when invoking LZ4_compress_generic() directly with normal text. This could very easily be an edge case. Either way it's interesting and worth sharing. bdd9143e0 Didn't add my file for some reason. Odd. 73dd936b9 Wrote the skeleton of the performance test. Need to finish tests for the rest of the call stack and then my own copy of generic(). 2c8013812 heapmode macro option for lz4hc ff3324b8d Fix : compiler-independent macro to remove deprecation warnings 64f8b037e clarified lz4 license 2df602568 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev 7d4575e54 fixed link typo 012ab2f52 Merge pull request #152 from dcolascione/dev 94a0954b1 Remove .dir-locals.el 4f1ff58f8 Tweak error message 975c06b1a Improve error messages for attempted console IO 6f1fe1b70 Add .dir-locals.el for setting tab-width 0011033e3 fixed #150 : blank lines when converting lz4.1 to .ps or .pdf 248553c5e minor : show in table that frame checksum is optional cd18ed308 create link to dynamic library without using `cp -a` (#147) fb6fece77 Updated Makefile and .travis 8a750aa35 Add static library target for msbuild 13f12aa02 Wrapped possibly empty string (Emscripten) with quotes (@Fordi) 4a5a003c4 updated cli help 9aa5504f0 New unalign access method e5aee601e updated xxhash e64345506 Merge pull request #142 from bioothod/master d15ac1f1a Slightly improved compression speed on ARMv6 0f2bf0c54 Improved performance on ARMv6 fb4d3ef2c Added .gitignore 4b4384772 Updated xxhash to r41 2a6ad2a7f debian: changed name lz4->liblz4 2f3e4bf32 debian: changed name lz4->liblz4 31fed3167 debian: changed name lz4->liblz4 309d1e087 debian: remove sbin dir 2131eedf7 debian: renamed lz4->liblz4, updated installed library files b47786b9e debian: updated debianization to cdbs a8f33076a cmake: when building library make it shared 33fa0bfb7 Added debianization b4b739c4b cmake: bump version to the latest release 5dd12b4a7 faster sanitize tests 60119e258 Makefile : generates *.o for faster processing 9517bd923 minor simplification lz4_readarch() a60d11e46 update on decompression speed 00eedb37d minor refactor 3e579afa6 Slightly improved decompression speed Slightly decreased decompression stack usage in 64-bits mode 795a2cdb1 sligthly improved decompression speed a3c6daba6 liblz4 : Moved XXH_NAMESPACE to CPPFLAGS d86dc9167 Merge pull request #131 from Cyan4973/dev dfabec2d1 Fixed a few minor visual analyzer warnings 4abc86e0b Increased stack memory considered normal by Visual static analyzer db59c7d33 Merge pull request #130 from Cyan4973/dev 8901dcdc9 removed one malloc 1098d897a lz4io : removed one malloc 81f19264a lz4cli : add one malloc test aa1350e13 lz4hc : added one malloc test 6fe48b918 changed datagen to remove one malloc 15603d956 Added static analyzer test in Visual 48feb909d Merge pull request #129 from Cyan4973/dev 57e14d4c1 renamed directory to versionsTest 14064beaf Updated NEWS 7273da38b minor example clarification 0cc38f9eb Updated lib readme 454d109c5 Fixed : fuzzer display messages caba2cc0f staticAnalyze use same compilation flags as normal build 8e8dc17ca Ensured make test is not confused with directory test 192ee724a Added namespace ability to xxhash c04df7e1b Changed : static library is no longer compiled with -fPIC by default (this option can still be added on the command line) See #53 cdef03389 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev 992faef5a Merge pull request #124 from toddlipcon/master 3d9adf7af Merge pull request #122 from keszybz/master 02ff90ac9 Merge pull request #121 from keszybz/add-gitignore 91138150c Fix cmake build to use gnu flags on clang 2ce2e6008 Minor doc clarification on sparse mode support b17d3220d Minor lib readme update 3bcf68ff3 Use lz4 binary to check output 386d216cd Add example which uses the frame api through the library 0e0c32dfb Make git ignore generated files 7a532eedd Minor update on source ports 8a7bcead9 Merge branch 'dev' of github.com:Cyan4973/lz4 into dev 1cec6cb03 Updated badges 7f3d82def Merge pull request #114 from lpsantil/djgpp f0a0e4430 READM fixup a3a479fc2 djgpp port now using the proposed contrib structure 00b6535b4 Reverted all `Makefile` changes 936f75b4d Renamed the djgpp README so that it gets converted to HTML by github. Fixed breakage of `make install`. 81d6b5634 Makefile cleanups to make building with Andrew Wu's build-djgpp cross compilers (and everyone else) simpler. faa0bce4e Fixup lib/Makefile so that is builds a library for djgpp using Andrew Wu's build-djgpp cross compilers b2bbb2790 Fixup programs/{Makefile,lz4cli.c,lz4io.c} to build with Andrew Wu's build-djgpp cross compilers. Add some documentation in README.DJ git-subtree-dir: src/lz4 git-subtree-split: c10863b98e1503af90616ae99725ecd120265dfb
This commit is contained in:
Brad Chase
20
programs/.gitignore
vendored
Normal file
20
programs/.gitignore
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
# local binary (Makefile)
|
||||
lz4
|
||||
unlz4
|
||||
lz4cat
|
||||
lz4c
|
||||
lz4c32
|
||||
datagen
|
||||
frametest
|
||||
frametest32
|
||||
fullbench
|
||||
fullbench32
|
||||
fuzzer
|
||||
fuzzer32
|
||||
*.exe
|
||||
|
||||
# tests files
|
||||
tmp*
|
||||
|
||||
# artefacts
|
||||
*.dSYM
|
||||
@@ -1,6 +1,8 @@
|
||||
# ##########################################################################
|
||||
# LZ4 programs - Makefile
|
||||
# Copyright (C) Yann Collet 2011-2015
|
||||
# Copyright (C) Yann Collet 2011-2016
|
||||
#
|
||||
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
|
||||
#
|
||||
# GPL v2 License
|
||||
#
|
||||
@@ -19,293 +21,152 @@
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
#
|
||||
# You can contact the author at :
|
||||
# - LZ4 homepage : http://www.lz4.org
|
||||
# - LZ4 source repository : https://github.com/Cyan4973/lz4
|
||||
# - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
# ##########################################################################
|
||||
# lz4 : Command Line Utility, supporting gzip-like arguments
|
||||
# lz4c : CLU, supporting also legacy lz4demo arguments
|
||||
# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
|
||||
# fuzzer : Test tool, to check lz4 integrity on target platform
|
||||
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
|
||||
# frametest : Test tool, to check lz4frame integrity on target platform
|
||||
# frametest32: Same as frametest, but forced to compile in 32-bits mode
|
||||
# fullbench : Precisely measure speed for each LZ4 function variant
|
||||
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
|
||||
# datagen : generates synthetic data samples for tests & benchmarks
|
||||
# ##########################################################################
|
||||
|
||||
RELEASE?= r130
|
||||
# Version numbers
|
||||
LZ4DIR := ../lib
|
||||
LIBVER_SRC := $(LZ4DIR)/lz4.h
|
||||
LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
|
||||
LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
|
||||
LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
|
||||
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
|
||||
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
|
||||
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
|
||||
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
|
||||
LIBVER := $(shell echo $(LIBVER_SCRIPT))
|
||||
|
||||
DESTDIR?=
|
||||
PREFIX ?= /usr/local
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
|
||||
FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c)
|
||||
OBJFILES := $(patsubst %.c,%.o,$(SRCFILES))
|
||||
|
||||
BINDIR := $(PREFIX)/bin
|
||||
MANDIR := $(PREFIX)/share/man/man1
|
||||
LZ4DIR := ../lib
|
||||
CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
|
||||
CFLAGS ?= -O3
|
||||
DEBUGFLAGS:=-Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
|
||||
-Wpointer-arith -Wstrict-aliasing=1
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
|
||||
|
||||
LZ4_VERSION=$(LIBVER)
|
||||
MD2ROFF = ronn
|
||||
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)"
|
||||
|
||||
|
||||
# Define *.exe as extension for Windows systems
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
EXT =.exe
|
||||
VOID = nul
|
||||
EXT :=.exe
|
||||
VOID := nul
|
||||
else
|
||||
EXT =
|
||||
VOID = /dev/null
|
||||
EXT :=
|
||||
VOID := /dev/null
|
||||
endif
|
||||
|
||||
|
||||
# Select test target for Travis CI's Build Matrix
|
||||
TRAVIS_TARGET:= $(LZ4_TRAVIS_CI_ENV)
|
||||
TEST_FILES := COPYING
|
||||
TEST_TARGETS := test-native
|
||||
FUZZER_TIME := -T9mn
|
||||
|
||||
default: lz4-release
|
||||
|
||||
default: lz4
|
||||
all: lz4 lz4c
|
||||
|
||||
m32: lz4c32 fullbench32 fuzzer32 frametest32
|
||||
all32: CFLAGS+=-m32
|
||||
all32: all
|
||||
|
||||
bins: lz4 lz4c fullbench fuzzer frametest datagen
|
||||
lz4: $(OBJFILES)
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
all: bins m32
|
||||
lz4-release: DEBUGFLAGS=
|
||||
lz4-release: lz4
|
||||
|
||||
lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
lz4c32: CFLAGS += -m32
|
||||
lz4c32 : $(SRCFILES)
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c
|
||||
$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
|
||||
lz4c: lz4
|
||||
ln -s lz4 lz4c
|
||||
|
||||
lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c
|
||||
$(CC) -m32 $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
|
||||
lz4.1: lz4.1.md
|
||||
cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
|
||||
|
||||
fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
man: lz4.1
|
||||
|
||||
fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
clean-man:
|
||||
rm lz4.1
|
||||
|
||||
fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
datagen : datagen.c datagencli.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
preview-man: clean-man man
|
||||
man ./lz4.1
|
||||
|
||||
clean:
|
||||
@rm -f core *.o *.test tmp* \
|
||||
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) \
|
||||
frametest$(EXT) frametest32$(EXT) \
|
||||
datagen$(EXT)
|
||||
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
|
||||
@$(RM) core *.o *.test tmp* \
|
||||
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
|
||||
#-----------------------------------------------------------------------------
|
||||
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
|
||||
#-----------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
||||
|
||||
install: lz4 lz4c
|
||||
unlz4: lz4
|
||||
ln -s lz4 unlz4
|
||||
|
||||
lz4cat: lz4
|
||||
ln -s lz4 lz4cat
|
||||
|
||||
ifneq (,$(filter $(shell uname),SunOS))
|
||||
INSTALL ?= ginstall
|
||||
else
|
||||
INSTALL ?= install
|
||||
endif
|
||||
|
||||
DESTDIR ?=
|
||||
# directory variables : GNU convention prefers lowercase
|
||||
# support both lower and uppercase (BSD), use uppercase in script
|
||||
prefix ?= /usr/local
|
||||
PREFIX ?= $(prefix)
|
||||
exec_prefix ?= $(PREFIX)
|
||||
bindir ?= $(exec_prefix)/bin
|
||||
BINDIR ?= $(bindir)
|
||||
datarootdir ?= $(PREFIX)/share
|
||||
mandir ?= $(datarootdir)/man
|
||||
|
||||
ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
||||
MANDIR ?= $(PREFIX)/man/man1
|
||||
else
|
||||
MANDIR ?= $(mandir)
|
||||
endif
|
||||
|
||||
INSTALL_PROGRAM ?= $(INSTALL) -m 755
|
||||
INSTALL_DATA ?= $(INSTALL) -m 644
|
||||
|
||||
|
||||
install: lz4$(EXT) lz4c$(EXT)
|
||||
@echo Installing binaries
|
||||
@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
|
||||
@install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4
|
||||
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
|
||||
@$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4
|
||||
@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat
|
||||
@ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4
|
||||
@install -m 755 lz4c $(DESTDIR)$(BINDIR)/lz4c
|
||||
@$(INSTALL_PROGRAM) lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c
|
||||
@echo Installing man pages
|
||||
@install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
|
||||
@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
|
||||
@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1
|
||||
@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1
|
||||
@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1
|
||||
@echo lz4 installation completed
|
||||
|
||||
uninstall:
|
||||
rm -f $(DESTDIR)$(BINDIR)/lz4cat
|
||||
rm -f $(DESTDIR)$(BINDIR)/unlz4
|
||||
[ -x $(DESTDIR)$(BINDIR)/lz4 ] && rm -f $(DESTDIR)$(BINDIR)/lz4
|
||||
[ -x $(DESTDIR)$(BINDIR)/lz4c ] && rm -f $(DESTDIR)$(BINDIR)/lz4c
|
||||
[ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1
|
||||
rm -f $(DESTDIR)$(MANDIR)/lz4c.1
|
||||
rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
|
||||
rm -f $(DESTDIR)$(MANDIR)/unlz4.1
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/lz4cat
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/unlz4
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/lz4
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/lz4c
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/lz4.1
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/lz4c.1
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/unlz4.1
|
||||
@echo lz4 programs successfully uninstalled
|
||||
|
||||
test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-mem
|
||||
|
||||
test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-mem32
|
||||
|
||||
test-all: test test32
|
||||
|
||||
test-travis: $(TRAVIS_TARGET)
|
||||
|
||||
test-lz4-sparse: lz4 datagen
|
||||
@echo "\n ---- test sparse file support ----"
|
||||
./datagen -g5M -P100 > tmpSrc
|
||||
./lz4 -B4D tmpSrc | ./lz4 -dv --sparse > tmpB4
|
||||
diff -s tmpSrc tmpB4
|
||||
./lz4 -B5D tmpSrc | ./lz4 -dv --sparse > tmpB5
|
||||
diff -s tmpSrc tmpB5
|
||||
./lz4 -B6D tmpSrc | ./lz4 -dv --sparse > tmpB6
|
||||
diff -s tmpSrc tmpB6
|
||||
./lz4 -B7D tmpSrc | ./lz4 -dv --sparse > tmpB7
|
||||
diff -s tmpSrc tmpB7
|
||||
./lz4 tmpSrc | ./lz4 -dv --no-sparse > tmpNoSparse
|
||||
diff -s tmpSrc tmpNoSparse
|
||||
ls -ls tmp*
|
||||
./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dv --sparse > tmpOdd # Odd size file (to not finish on an exact nb of blocks)
|
||||
./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd
|
||||
ls -ls tmpOdd
|
||||
@rm tmp*
|
||||
@echo "\n Compatibility with Console :"
|
||||
echo "Hello World 1 !" | ./lz4 | ./lz4 -d -c
|
||||
echo "Hello World 2 !" | ./lz4 | ./lz4 -d | cat
|
||||
echo "Hello World 3 !" | ./lz4 --no-frame-crc | ./lz4 -d -c
|
||||
@echo "\n Compatibility with Append :"
|
||||
./datagen -P100 -g1M > tmp1M
|
||||
cat tmp1M > tmp2M
|
||||
cat tmp1M >> tmp2M
|
||||
./lz4 -B5 -v tmp1M tmpC
|
||||
./lz4 -d -v tmpC tmpR
|
||||
./lz4 -d -v tmpC >> tmpR
|
||||
ls -ls tmp*
|
||||
diff tmp2M tmpR
|
||||
@rm tmp*
|
||||
|
||||
|
||||
|
||||
test-lz4-contentSize: lz4 datagen
|
||||
@echo "\n ---- test original size support ----"
|
||||
./datagen -g15M > tmp
|
||||
./lz4 -v tmp | ./lz4 -t
|
||||
./lz4 -v --content-size tmp | ./lz4 -d > tmp2
|
||||
diff -s tmp tmp2
|
||||
# test large size [2-4] GB
|
||||
@./datagen -g3G -P100 | ./lz4 | ./lz4 --decompress --force --sparse - tmp
|
||||
@ls -ls tmp
|
||||
./lz4 --quiet --content-size tmp | ./lz4 --verbose --decompress --force --sparse - tmp2
|
||||
@ls -ls tmp2
|
||||
@rm tmp*
|
||||
|
||||
test-lz4-frame-concatenation: lz4 datagen
|
||||
@echo "\n ---- test frame concatenation ----"
|
||||
@echo -n > empty.test
|
||||
@echo hi > nonempty.test
|
||||
cat nonempty.test empty.test nonempty.test > orig.test
|
||||
@./lz4 -zq empty.test > empty.lz4.test
|
||||
@./lz4 -zq nonempty.test > nonempty.lz4.test
|
||||
cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test
|
||||
./lz4 -d concat.lz4.test > result.test
|
||||
sdiff orig.test result.test
|
||||
@rm *.test
|
||||
@echo frame concatenation test completed
|
||||
|
||||
test-lz4-multiple: lz4 datagen
|
||||
@echo "\n ---- test multiple files ----"
|
||||
@./datagen -s1 > tmp1 2> $(VOID)
|
||||
@./datagen -s2 -g100K > tmp2 2> $(VOID)
|
||||
@./datagen -s3 -g1M > tmp3 2> $(VOID)
|
||||
./lz4 -f -m tmp*
|
||||
ls -ls tmp*
|
||||
rm tmp1 tmp2 tmp3
|
||||
./lz4 -df -m *.lz4
|
||||
ls -ls tmp*
|
||||
./lz4 -f -m tmp1 notHere tmp2; echo $$?
|
||||
@rm tmp*
|
||||
|
||||
test-lz4-basic: lz4 datagen
|
||||
@echo "\n ---- test lz4 basic compression/decompression ----"
|
||||
./datagen -g0 | ./lz4 -v | ./lz4 -t
|
||||
./datagen -g16KB | ./lz4 -9 | ./lz4 -t
|
||||
./datagen -g20KB > tmpSrc
|
||||
./lz4 < tmpSrc | ./lz4 -d > tmpRes
|
||||
diff -q tmpSrc tmpRes
|
||||
./lz4 --no-frame-crc < tmpSrc | ./lz4 -d > tmpRes
|
||||
diff -q tmpSrc tmpRes
|
||||
./datagen | ./lz4 | ./lz4 -t
|
||||
./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t
|
||||
./datagen -g17M | ./lz4 -9v | ./lz4 -qt
|
||||
./datagen -g33M | ./lz4 --no-frame-crc | ./lz4 -t
|
||||
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t
|
||||
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -qt
|
||||
./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -qt
|
||||
@rm tmp*
|
||||
|
||||
test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
|
||||
@echo "\n ---- test pass-through ----"
|
||||
./datagen | ./lz4 -tf
|
||||
|
||||
test-lz4c: lz4c datagen
|
||||
@echo "\n ---- test lz4c version ----"
|
||||
./datagen -g256MB | ./lz4c -l -v | ./lz4c -t
|
||||
|
||||
test-interop-32-64: lz4 lz4c32 datagen
|
||||
@echo "\n ---- test interoperability 32-bits -vs- 64 bits ----"
|
||||
./datagen -g16KB | ./lz4c32 -9 | ./lz4 -t
|
||||
./datagen -P10 | ./lz4 -9B4 | ./lz4c32 -t
|
||||
./datagen | ./lz4c32 | ./lz4 -t
|
||||
./datagen -g1M | ./lz4 -3B5 | ./lz4c32 -t
|
||||
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -qt
|
||||
./datagen -g1G -P90 | ./lz4 | ./lz4c32 -t
|
||||
./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -qt
|
||||
|
||||
test-lz4c32-basic: lz4c32 datagen
|
||||
@echo "\n ---- test lz4c32 32-bits version ----"
|
||||
./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -t
|
||||
./datagen | ./lz4c32 | ./lz4c32 -t
|
||||
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt
|
||||
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -qt
|
||||
|
||||
test-lz4c32: test-lz4c32-basic test-interop-32-64
|
||||
|
||||
test-fullbench: fullbench
|
||||
./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES)
|
||||
|
||||
test-fullbench32: fullbench32
|
||||
./fullbench32 --no-prompt $(NB_LOOPS) $(TEST_FILES)
|
||||
|
||||
test-fuzzer: fuzzer
|
||||
./fuzzer $(FUZZER_TIME)
|
||||
|
||||
test-fuzzer32: fuzzer32
|
||||
./fuzzer32 $(FUZZER_TIME)
|
||||
|
||||
test-frametest: frametest
|
||||
./frametest $(FUZZER_TIME)
|
||||
|
||||
test-frametest32: frametest32
|
||||
./frametest32 $(FUZZER_TIME)
|
||||
|
||||
test-mem: lz4 datagen fuzzer frametest fullbench
|
||||
@echo "\n ---- valgrind tests : memory analyzer ----"
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
|
||||
./datagen -g16KB > tmp
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -BD -f tmp $(VOID)
|
||||
./datagen -g16KB -s2 > tmp2
|
||||
./datagen -g16KB -s3 > tmp3
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./lz4 --force --multiple tmp tmp2 tmp3
|
||||
./datagen -g16MB > tmp
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -B5D -f tmp tmp2
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -t tmp2
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -bi1 tmp
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2
|
||||
./datagen -g256MB > tmp
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -B4D -f -vq tmp $(VOID)
|
||||
rm tmp*
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
|
||||
|
||||
test-mem32: lz4c32 datagen
|
||||
# unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...
|
||||
|
||||
endif
|
||||
|
||||
71
programs/README.md
Normal file
71
programs/README.md
Normal file
@@ -0,0 +1,71 @@
|
||||
Command Line Interface for LZ4 library
|
||||
============================================
|
||||
|
||||
Command Line Interface (CLI) can be created using the `make` command without any additional parameters.
|
||||
There are also multiple targets that create different variations of CLI:
|
||||
- `lz4` : default CLI, with a command line syntax close to gzip
|
||||
- `lz4c` : Same as `lz4` with additional support legacy lz4 commands (incompatible with gzip)
|
||||
- `lz4c32` : Same as `lz4c`, but forced to compile in 32-bits mode
|
||||
|
||||
|
||||
#### Aggregation of parameters
|
||||
CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`.
|
||||
|
||||
|
||||
|
||||
#### Benchmark in Command Line Interface
|
||||
CLI includes in-memory compression benchmark module for lz4.
|
||||
The benchmark is conducted using a given filename.
|
||||
The file is read into memory.
|
||||
It makes benchmark more precise as it eliminates I/O overhead.
|
||||
|
||||
The benchmark measures ratio, compressed size, compression and decompression speed.
|
||||
One can select compression levels starting from `-b` and ending with `-e`.
|
||||
The `-i` parameter selects a number of seconds used for each of tested levels.
|
||||
|
||||
|
||||
|
||||
#### Usage of Command Line Interface
|
||||
The full list of commands can be obtained with `-h` or `-H` parameter:
|
||||
```
|
||||
Usage :
|
||||
lz4 [arg] [input] [output]
|
||||
|
||||
input : a filename
|
||||
with no FILE, or when FILE is - or stdin, read standard input
|
||||
Arguments :
|
||||
-1 : Fast compression (default)
|
||||
-9 : High compression
|
||||
-d : decompression (default for .lz4 extension)
|
||||
-z : force compression
|
||||
-f : overwrite output without prompting
|
||||
--rm : remove source file(s) after successful de/compression
|
||||
-h/-H : display help/long help and exit
|
||||
|
||||
Advanced arguments :
|
||||
-V : display Version number and exit
|
||||
-v : verbose mode
|
||||
-q : suppress warnings; specify twice to suppress errors too
|
||||
-c : force write to standard output, even if it is the console
|
||||
-t : test compressed file integrity
|
||||
-m : multiple input files (implies automatic output filenames)
|
||||
-r : operate recursively on directories (sets also -m)
|
||||
-l : compress using Legacy format (Linux kernel compression)
|
||||
-B# : Block size [4-7] (default : 7)
|
||||
-BD : Block dependency (improve compression ratio)
|
||||
--no-frame-crc : disable stream checksum (default:enabled)
|
||||
--content-size : compressed frame includes original size (default:not present)
|
||||
--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)
|
||||
Benchmark arguments :
|
||||
-b# : benchmark file(s), using # compression level (default : 1)
|
||||
-e# : test all compression levels from -bX to # (default : 1)
|
||||
-i# : minimum evaluation time in seconds (default : 3s)
|
||||
-B# : cut file into independent blocks of size # bytes [32+]
|
||||
or predefined block size [4-7] (default: 7)
|
||||
```
|
||||
|
||||
#### License
|
||||
|
||||
All files in this directory are licensed under GPL-v2.
|
||||
See [COPYING](COPYING) for details.
|
||||
The text of the license is also included at the top of each source file.
|
||||
752
programs/bench.c
752
programs/bench.c
@@ -1,6 +1,6 @@
|
||||
/*
|
||||
bench.c - Demo program to benchmark open-source compression algorithms
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
Copyright (C) Yann Collet 2012-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
@@ -19,42 +19,32 @@
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : https://github.com/Cyan4973/lz4
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
- LZ4 homepage : http://www.lz4.org
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Compiler Options
|
||||
***************************************/
|
||||
#if defined(_MSC_VER) || defined(_WIN32)
|
||||
# define _CRT_SECURE_NO_WARNINGS
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
|
||||
# define BMK_LEGACY_TIMER 1 /* S_ISREG & gettimeofday() are not supported by MSVC */
|
||||
#endif
|
||||
|
||||
/* Unix Large Files support (>4GB) */
|
||||
#define _FILE_OFFSET_BITS 64
|
||||
#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
|
||||
# define _LARGEFILE_SOURCE
|
||||
#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
|
||||
# define _LARGEFILE64_SOURCE
|
||||
/*-************************************
|
||||
* Compiler options
|
||||
**************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <sys/types.h> /* stat64 */
|
||||
#include <sys/stat.h> /* stat64 */
|
||||
#include "platform.h" /* Compiler options */
|
||||
#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "xxhash.h"
|
||||
|
||||
/* Use ftime() if gettimeofday() is not available on your target */
|
||||
#if defined(BMK_LEGACY_TIMER)
|
||||
# include <sys/timeb.h> /* timeb, ftime */
|
||||
#else
|
||||
# include <sys/time.h> /* gettimeofday */
|
||||
#endif
|
||||
|
||||
#include "lz4.h"
|
||||
#define COMPRESSOR0 LZ4_compress_local
|
||||
@@ -62,134 +52,310 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi
|
||||
#include "lz4hc.h"
|
||||
#define COMPRESSOR1 LZ4_compress_HC
|
||||
#define DEFAULTCOMPRESSOR COMPRESSOR0
|
||||
|
||||
#include "xxhash.h"
|
||||
#define LZ4_isError(errcode) (errcode==0)
|
||||
|
||||
|
||||
/**************************************
|
||||
* Compiler specifics
|
||||
***************************************/
|
||||
#if !defined(S_ISREG)
|
||||
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Basic Types
|
||||
***************************************/
|
||||
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define NBLOOPS 3
|
||||
#define TIMELOOP 2000
|
||||
#ifndef LZ4_GIT_COMMIT_STRING
|
||||
# define LZ4_GIT_COMMIT_STRING ""
|
||||
#else
|
||||
# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT)
|
||||
#endif
|
||||
|
||||
#define NBSECONDS 3
|
||||
#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
|
||||
#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
|
||||
#define COOLPERIOD_SEC 10
|
||||
#define DECOMP_MULT 2 /* test decompression DECOMP_MULT times longer than compression */
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define MAX_MEM (2 GB - 64 MB)
|
||||
#define DEFAULT_CHUNKSIZE (4 MB)
|
||||
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
|
||||
|
||||
static U32 g_compressibilityDefault = 50;
|
||||
|
||||
|
||||
/**************************************
|
||||
* Local structures
|
||||
/* *************************************
|
||||
* console display
|
||||
***************************************/
|
||||
struct chunkParameters
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stdout); } }
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, "\n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Benchmark Parameters
|
||||
***************************************/
|
||||
static U32 g_nbSeconds = NBSECONDS;
|
||||
static size_t g_blockSize = 0;
|
||||
int g_additionalParam = 0;
|
||||
|
||||
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
|
||||
|
||||
void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
|
||||
|
||||
void BMK_SetNbSeconds(unsigned nbSeconds)
|
||||
{
|
||||
U32 id;
|
||||
char* origBuffer;
|
||||
char* compressedBuffer;
|
||||
int origSize;
|
||||
int compressedSize;
|
||||
};
|
||||
g_nbSeconds = nbSeconds;
|
||||
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
|
||||
}
|
||||
|
||||
void BMK_SetBlockSize(size_t blockSize)
|
||||
{
|
||||
g_blockSize = blockSize;
|
||||
}
|
||||
|
||||
|
||||
/* ********************************************************
|
||||
* Bench functions
|
||||
**********************************************************/
|
||||
typedef struct {
|
||||
const char* srcPtr;
|
||||
size_t srcSize;
|
||||
char* cPtr;
|
||||
size_t cRoom;
|
||||
size_t cSize;
|
||||
char* resPtr;
|
||||
size_t resSize;
|
||||
} blockParam_t;
|
||||
|
||||
struct compressionParameters
|
||||
{
|
||||
int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
|
||||
int (*decompressionFunction)(const char* src, char* dst, int dstSize);
|
||||
};
|
||||
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
|
||||
/**************************************
|
||||
* MACRO
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
|
||||
|
||||
/**************************************
|
||||
* Benchmark Parameters
|
||||
***************************************/
|
||||
static int chunkSize = DEFAULT_CHUNKSIZE;
|
||||
static int nbIterations = NBLOOPS;
|
||||
static int BMK_pause = 0;
|
||||
|
||||
void BMK_setBlocksize(int bsize) { chunkSize = bsize; }
|
||||
|
||||
void BMK_setNbIterations(int nbLoops)
|
||||
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const char* displayName, int cLevel,
|
||||
const size_t* fileSizes, U32 nbFiles)
|
||||
{
|
||||
nbIterations = nbLoops;
|
||||
DISPLAY("- %i iterations -\n", nbIterations);
|
||||
}
|
||||
size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
|
||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
|
||||
size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
||||
void* const compressedBuffer = malloc(maxCompressedSize);
|
||||
void* const resultBuffer = malloc(srcSize);
|
||||
U32 nbBlocks;
|
||||
UTIL_time_t ticksPerSecond;
|
||||
struct compressionParameters compP;
|
||||
int cfunctionId;
|
||||
|
||||
void BMK_setPause(void) { BMK_pause = 1; }
|
||||
/* checks */
|
||||
if (!compressedBuffer || !resultBuffer || !blockTable)
|
||||
EXM_THROW(31, "allocation error : not enough memory");
|
||||
|
||||
/* init */
|
||||
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
|
||||
UTIL_initTimer(&ticksPerSecond);
|
||||
|
||||
/*********************************************************
|
||||
* Private functions
|
||||
**********************************************************/
|
||||
|
||||
#if defined(BMK_LEGACY_TIMER)
|
||||
|
||||
static int BMK_GetMilliStart(void)
|
||||
{
|
||||
/* Based on Legacy ftime()
|
||||
Rolls over every ~ 12.1 days (0x100000/24/60/60)
|
||||
Use GetMilliSpan to correct for rollover */
|
||||
struct timeb tb;
|
||||
int nCount;
|
||||
ftime( &tb );
|
||||
nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static int BMK_GetMilliStart(void)
|
||||
{
|
||||
/* Based on newer gettimeofday()
|
||||
Use GetMilliSpan to correct for rollover */
|
||||
struct timeval tv;
|
||||
int nCount;
|
||||
gettimeofday(&tv, NULL);
|
||||
nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
/* Init */
|
||||
if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1;
|
||||
switch (cfunctionId)
|
||||
{
|
||||
#ifdef COMPRESSOR0
|
||||
case 0 : compP.compressionFunction = COMPRESSOR0; break;
|
||||
#endif
|
||||
#ifdef COMPRESSOR1
|
||||
case 1 : compP.compressionFunction = COMPRESSOR1; break;
|
||||
#endif
|
||||
default : compP.compressionFunction = DEFAULTCOMPRESSOR;
|
||||
}
|
||||
|
||||
/* Init blockTable data */
|
||||
{ const char* srcPtr = (const char*)srcBuffer;
|
||||
char* cPtr = (char*)compressedBuffer;
|
||||
char* resPtr = (char*)resultBuffer;
|
||||
U32 fileNb;
|
||||
for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
|
||||
size_t remaining = fileSizes[fileNb];
|
||||
U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
|
||||
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
|
||||
for ( ; nbBlocks<blockEnd; nbBlocks++) {
|
||||
size_t const thisBlockSize = MIN(remaining, blockSize);
|
||||
blockTable[nbBlocks].srcPtr = srcPtr;
|
||||
blockTable[nbBlocks].cPtr = cPtr;
|
||||
blockTable[nbBlocks].resPtr = resPtr;
|
||||
blockTable[nbBlocks].srcSize = thisBlockSize;
|
||||
blockTable[nbBlocks].cRoom = LZ4_compressBound((int)thisBlockSize);
|
||||
srcPtr += thisBlockSize;
|
||||
cPtr += blockTable[nbBlocks].cRoom;
|
||||
resPtr += thisBlockSize;
|
||||
remaining -= thisBlockSize;
|
||||
} } }
|
||||
|
||||
static int BMK_GetMilliSpan( int nTimeStart )
|
||||
{
|
||||
int nSpan = BMK_GetMilliStart() - nTimeStart;
|
||||
if ( nSpan < 0 )
|
||||
nSpan += 0x100000 * 1000;
|
||||
return nSpan;
|
||||
/* warmimg up memory */
|
||||
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
|
||||
|
||||
/* Bench */
|
||||
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
|
||||
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
|
||||
UTIL_time_t coolTime;
|
||||
U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100;
|
||||
U64 totalCTime=0, totalDTime=0;
|
||||
U32 cCompleted=0, dCompleted=0;
|
||||
# define NB_MARKS 4
|
||||
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
|
||||
U32 markNb = 0;
|
||||
size_t cSize = 0;
|
||||
double ratio = 0.;
|
||||
|
||||
UTIL_getTime(&coolTime);
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
while (!cCompleted || !dCompleted) {
|
||||
UTIL_time_t clockStart;
|
||||
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
|
||||
|
||||
/* overheat protection */
|
||||
if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
|
||||
DISPLAYLEVEL(2, "\rcooling down ... \r");
|
||||
UTIL_sleep(COOLPERIOD_SEC);
|
||||
UTIL_getTime(&coolTime);
|
||||
}
|
||||
|
||||
/* Compression */
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
|
||||
if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
|
||||
|
||||
UTIL_sleepMilli(1); /* give processor time to other processes */
|
||||
UTIL_waitForNextTick(ticksPerSecond);
|
||||
UTIL_getTime(&clockStart);
|
||||
|
||||
if (!cCompleted) { /* still some time to do compression tests */
|
||||
U32 nbLoops = 0;
|
||||
do {
|
||||
U32 blockNb;
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const rSize = compP.compressionFunction(blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom, cLevel);
|
||||
if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4_compress() failed");
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
}
|
||||
nbLoops++;
|
||||
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
|
||||
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
|
||||
if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
|
||||
totalCTime += clockSpan;
|
||||
cCompleted = totalCTime>maxTime;
|
||||
} }
|
||||
|
||||
cSize = 0;
|
||||
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
|
||||
cSize += !cSize; /* avoid div by 0 */
|
||||
ratio = (double)srcSize / (double)cSize;
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / fastestC );
|
||||
|
||||
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
|
||||
#if 1
|
||||
/* Decompression */
|
||||
if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
|
||||
|
||||
UTIL_sleepMilli(1); /* give processor time to other processes */
|
||||
UTIL_waitForNextTick(ticksPerSecond);
|
||||
UTIL_getTime(&clockStart);
|
||||
|
||||
if (!dCompleted) {
|
||||
U32 nbLoops = 0;
|
||||
do {
|
||||
U32 blockNb;
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
|
||||
if (LZ4_isError(regenSize)) {
|
||||
DISPLAY("LZ4_decompress_safe() failed on block %u \n", blockNb);
|
||||
clockLoop = 0; /* force immediate test end */
|
||||
break;
|
||||
}
|
||||
|
||||
blockTable[blockNb].resSize = regenSize;
|
||||
}
|
||||
nbLoops++;
|
||||
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < DECOMP_MULT*clockLoop);
|
||||
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
|
||||
if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
|
||||
totalDTime += clockSpan;
|
||||
dCompleted = totalDTime>(DECOMP_MULT*maxTime);
|
||||
} }
|
||||
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / fastestC,
|
||||
(double)srcSize / fastestD );
|
||||
|
||||
/* CRC Checking */
|
||||
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if (crcOrig!=crcCheck) {
|
||||
size_t u;
|
||||
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
for (u=0; u<srcSize; u++) {
|
||||
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
|
||||
U32 segNb, bNb, pos;
|
||||
size_t bacc = 0;
|
||||
DISPLAY("Decoding error at pos %u ", (U32)u);
|
||||
for (segNb = 0; segNb < nbBlocks; segNb++) {
|
||||
if (bacc + blockTable[segNb].srcSize > u) break;
|
||||
bacc += blockTable[segNb].srcSize;
|
||||
}
|
||||
pos = (U32)(u - bacc);
|
||||
bNb = pos / (128 KB);
|
||||
DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
|
||||
break;
|
||||
}
|
||||
if (u==srcSize-1) { /* should never happen */
|
||||
DISPLAY("no difference detected\n");
|
||||
} }
|
||||
break;
|
||||
} } /* CRC Checking */
|
||||
#endif
|
||||
} /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
|
||||
|
||||
if (g_displayLevel == 1) {
|
||||
double cSpeed = (double)srcSize / fastestC;
|
||||
double dSpeed = (double)srcSize / fastestD;
|
||||
if (g_additionalParam)
|
||||
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
|
||||
else
|
||||
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
|
||||
}
|
||||
DISPLAYLEVEL(2, "%2i#\n", cLevel);
|
||||
} /* Bench */
|
||||
|
||||
/* clean up */
|
||||
free(blockTable);
|
||||
free(compressedBuffer);
|
||||
free(resultBuffer);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -200,10 +366,9 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
|
||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||
requiredMem += 2*step;
|
||||
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
|
||||
if (requiredMem > maxMemory) requiredMem = maxMemory;
|
||||
|
||||
while (!testmem)
|
||||
{
|
||||
while (!testmem) {
|
||||
if (requiredMem > step) requiredMem -= step;
|
||||
else requiredMem >>= 1;
|
||||
testmem = (BYTE*) malloc ((size_t)requiredMem);
|
||||
@@ -218,220 +383,139 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
}
|
||||
|
||||
|
||||
static U64 BMK_GetFileSize(const char* infilename)
|
||||
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
const char* displayName, int cLevel, int cLevelLast,
|
||||
const size_t* fileSizes, unsigned nbFiles)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
#endif
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
|
||||
return (U64)statbuf.st_size;
|
||||
int l;
|
||||
|
||||
const char* pch = strrchr(displayName, '\\'); /* Windows */
|
||||
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
|
||||
if (pch) displayName = pch+1;
|
||||
|
||||
SET_REALTIME_PRIORITY;
|
||||
|
||||
if (g_displayLevel == 1 && !g_additionalParam)
|
||||
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LZ4_VERSION_STRING, LZ4_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
|
||||
|
||||
if (cLevelLast < cLevel) cLevelLast = cLevel;
|
||||
|
||||
for (l=cLevel; l <= cLevelLast; l++) {
|
||||
BMK_benchMem(srcBuffer, benchedSize,
|
||||
displayName, l,
|
||||
fileSizes, nbFiles);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Public function
|
||||
**********************************************************/
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
|
||||
/*! BMK_loadFiles() :
|
||||
Loads `buffer` with content of files listed within `fileNamesTable`.
|
||||
At most, fills `buffer` entirely */
|
||||
static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t* fileSizes,
|
||||
const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
int fileIdx=0;
|
||||
char* orig_buff;
|
||||
struct compressionParameters compP;
|
||||
int cfunctionId;
|
||||
|
||||
U64 totals = 0;
|
||||
U64 totalz = 0;
|
||||
double totalc = 0.;
|
||||
double totald = 0.;
|
||||
|
||||
|
||||
/* Init */
|
||||
if (cLevel <= 3) cfunctionId = 0; else cfunctionId = 1;
|
||||
switch (cfunctionId)
|
||||
{
|
||||
#ifdef COMPRESSOR0
|
||||
case 0 : compP.compressionFunction = COMPRESSOR0; break;
|
||||
#endif
|
||||
#ifdef COMPRESSOR1
|
||||
case 1 : compP.compressionFunction = COMPRESSOR1; break;
|
||||
#endif
|
||||
default : compP.compressionFunction = DEFAULTCOMPRESSOR;
|
||||
}
|
||||
compP.decompressionFunction = LZ4_decompress_fast;
|
||||
|
||||
/* Loop for each file */
|
||||
while (fileIdx<nbFiles)
|
||||
{
|
||||
FILE* inFile;
|
||||
const char* inFileName;
|
||||
U64 inFileSize;
|
||||
size_t benchedSize;
|
||||
int nbChunks;
|
||||
int maxCompressedChunkSize;
|
||||
size_t readSize;
|
||||
char* compressedBuffer; int compressedBuffSize;
|
||||
struct chunkParameters* chunkP;
|
||||
U32 crcOrig;
|
||||
|
||||
/* Check file existence */
|
||||
inFileName = fileNamesTable[fileIdx++];
|
||||
inFile = fopen( inFileName, "rb" );
|
||||
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
inFileSize = BMK_GetFileSize(inFileName);
|
||||
if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; }
|
||||
benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2;
|
||||
if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; }
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize)
|
||||
{
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
|
||||
}
|
||||
|
||||
/* Alloc */
|
||||
chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters));
|
||||
orig_buff = (char*)malloc((size_t)benchedSize);
|
||||
nbChunks = (int) ((int)benchedSize / chunkSize) + 1;
|
||||
maxCompressedChunkSize = LZ4_compressBound(chunkSize);
|
||||
compressedBuffSize = nbChunks * maxCompressedChunkSize;
|
||||
compressedBuffer = (char*)malloc((size_t)compressedBuffSize);
|
||||
|
||||
if (!orig_buff || !compressedBuffer)
|
||||
{
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
free(orig_buff);
|
||||
free(compressedBuffer);
|
||||
free(chunkP);
|
||||
fclose(inFile);
|
||||
return 12;
|
||||
}
|
||||
|
||||
/* Init chunks data */
|
||||
{
|
||||
int i;
|
||||
size_t remaining = benchedSize;
|
||||
char* in = orig_buff;
|
||||
char* out = compressedBuffer;
|
||||
for (i=0; i<nbChunks; i++)
|
||||
{
|
||||
chunkP[i].id = i;
|
||||
chunkP[i].origBuffer = in; in += chunkSize;
|
||||
if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
|
||||
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
|
||||
chunkP[i].compressedSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Fill input buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
readSize = fread(orig_buff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
|
||||
if (readSize != benchedSize)
|
||||
{
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(orig_buff);
|
||||
free(compressedBuffer);
|
||||
free(chunkP);
|
||||
return 13;
|
||||
}
|
||||
|
||||
/* Calculating input Checksum */
|
||||
crcOrig = XXH32(orig_buff, (unsigned int)benchedSize,0);
|
||||
|
||||
|
||||
/* Bench */
|
||||
{
|
||||
int loopNb, chunkNb;
|
||||
size_t cSize=0;
|
||||
double fastestC = 100000000., fastestD = 100000000.;
|
||||
double ratio=0.;
|
||||
U32 crcCheck=0;
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
|
||||
{
|
||||
int nbLoops;
|
||||
int milliTime;
|
||||
|
||||
/* Compression */
|
||||
DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, inFileName, (int)benchedSize);
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) compressedBuffer[i]=(char)i; } /* warmimg up memory */
|
||||
|
||||
nbLoops = 0;
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliStart() == milliTime);
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
|
||||
{
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
|
||||
chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize, cLevel);
|
||||
nbLoops++;
|
||||
}
|
||||
milliTime = BMK_GetMilliSpan(milliTime);
|
||||
|
||||
nbLoops += !nbLoops; /* avoid division by zero */
|
||||
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops;
|
||||
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
|
||||
ratio = (double)cSize/(double)benchedSize*100.;
|
||||
|
||||
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000.);
|
||||
|
||||
/* Decompression */
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing area, for CRC checking */
|
||||
|
||||
nbLoops = 0;
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliStart() == milliTime);
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
|
||||
{
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
|
||||
chunkP[chunkNb].compressedSize = LZ4_decompress_fast(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].origSize);
|
||||
nbLoops++;
|
||||
}
|
||||
milliTime = BMK_GetMilliSpan(milliTime);
|
||||
|
||||
nbLoops += !nbLoops; /* avoid division by zero */
|
||||
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops;
|
||||
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
|
||||
|
||||
/* CRC Checking */
|
||||
crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0);
|
||||
if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; }
|
||||
size_t pos = 0, totalSize = 0;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
FILE* f;
|
||||
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
if (UTIL_isDirectory(fileNamesTable[n])) {
|
||||
DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
|
||||
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
||||
pos += readSize; }
|
||||
fileSizes[n] = (size_t)fileSize;
|
||||
totalSize += (size_t)fileSize;
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
if (crcOrig==crcCheck)
|
||||
{
|
||||
if (ratio<100.)
|
||||
DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
|
||||
else
|
||||
DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
|
||||
}
|
||||
totals += benchedSize;
|
||||
totalz += cSize;
|
||||
totalc += fastestC;
|
||||
totald += fastestD;
|
||||
}
|
||||
if (totalSize == 0) EXM_THROW(12, "no data to bench");
|
||||
}
|
||||
|
||||
free(orig_buff);
|
||||
free(compressedBuffer);
|
||||
free(chunkP);
|
||||
}
|
||||
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
||||
int cLevel, int cLevelLast)
|
||||
{
|
||||
void* srcBuffer;
|
||||
size_t benchedSize;
|
||||
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
char mfName[20] = {0};
|
||||
|
||||
if (nbFiles > 1)
|
||||
DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.);
|
||||
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
|
||||
|
||||
if (BMK_pause) { DISPLAY("\npress enter...\n"); (void)getchar(); }
|
||||
/* Memory allocation & restrictions */
|
||||
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
|
||||
if (benchedSize==0) EXM_THROW(12, "not enough memory");
|
||||
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
||||
if (benchedSize > LZ4_MAX_INPUT_SIZE) {
|
||||
benchedSize = LZ4_MAX_INPUT_SIZE;
|
||||
DISPLAY("File(s) bigger than LZ4's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
||||
} else {
|
||||
if (benchedSize < totalSizeToLoad)
|
||||
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
||||
}
|
||||
srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */
|
||||
if (!srcBuffer) EXM_THROW(12, "not enough memory");
|
||||
|
||||
return 0;
|
||||
/* Load input buffer */
|
||||
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
|
||||
|
||||
/* Bench */
|
||||
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
|
||||
{ const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
|
||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||
displayName, cLevel, cLevelLast,
|
||||
fileSizes, nbFiles);
|
||||
}
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
free(fileSizes);
|
||||
}
|
||||
|
||||
|
||||
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
|
||||
{
|
||||
char name[20] = {0};
|
||||
size_t benchedSize = 10000000;
|
||||
void* const srcBuffer = malloc(benchedSize);
|
||||
|
||||
/* Memory allocation */
|
||||
if (!srcBuffer) EXM_THROW(21, "not enough memory");
|
||||
|
||||
/* Fill input buffer */
|
||||
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
|
||||
|
||||
/* Bench */
|
||||
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
|
||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1);
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
}
|
||||
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||
int cLevel, int cLevelLast)
|
||||
{
|
||||
double const compressibility = (double)g_compressibilityDefault / 100;
|
||||
|
||||
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
|
||||
if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
|
||||
if (cLevelLast < cLevel) cLevelLast = cLevel;
|
||||
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
|
||||
|
||||
if (nbFiles == 0)
|
||||
BMK_syntheticTest(cLevel, cLevelLast, compressibility);
|
||||
else
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/*
|
||||
bench.h - Demo program to benchmark open-source compression algorithm
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
Copyright (C) Yann Collet 2012-2016
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
@@ -17,17 +17,21 @@
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : https://github.com/Cyan4973/lz4
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
#pragma once
|
||||
#ifndef BENCH_H_125623623633
|
||||
#define BENCH_H_125623623633
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* Main function */
|
||||
int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel);
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||
int cLevel, int cLevelLast);
|
||||
|
||||
/* Set Parameters */
|
||||
void BMK_setBlocksize(int bsize);
|
||||
void BMK_setNbIterations(int nbLoops);
|
||||
void BMK_setPause(void);
|
||||
void BMK_SetNbSeconds(unsigned nbLoops);
|
||||
void BMK_SetBlockSize(size_t blockSize);
|
||||
void BMK_setAdditionalParam(int additionalParam);
|
||||
void BMK_setNotificationLevel(unsigned level);
|
||||
|
||||
#endif /* BENCH_H_125623623633 */
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/*
|
||||
datagen.c - compressible data generator test tool
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
Copyright (C) Yann Collet 2012-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
@@ -19,49 +19,20 @@
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD source repository : https://github.com/Cyan4973/zstd
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include "platform.h" /* Compiler options, SET_BINARY_MODE */
|
||||
#include "util.h" /* U32 */
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <stdio.h> /* FILE, fwrite */
|
||||
#include <string.h> /* memcpy */
|
||||
|
||||
|
||||
/**************************************
|
||||
* Basic Types
|
||||
**************************************/
|
||||
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* OS-specific Includes
|
||||
**************************************/
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
# include <io.h> /* _setmode, _isatty */
|
||||
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
|
||||
#else
|
||||
# define SET_BINARY_MODE(file)
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
@@ -71,9 +42,20 @@
|
||||
#define PRIME2 2246822519U
|
||||
|
||||
|
||||
/**************************************
|
||||
* Local types
|
||||
**************************************/
|
||||
#define LTLOG 13
|
||||
#define LTSIZE (1<<LTLOG)
|
||||
#define LTMASK (LTSIZE-1)
|
||||
typedef BYTE litDistribTable[LTSIZE];
|
||||
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Local Functions
|
||||
*********************************************************/
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) :(b) )
|
||||
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static unsigned int RDG_rand(U32* src)
|
||||
{
|
||||
@@ -86,51 +68,38 @@ static unsigned int RDG_rand(U32* src)
|
||||
}
|
||||
|
||||
|
||||
#define LTSIZE 8192
|
||||
#define LTMASK (LTSIZE-1)
|
||||
static void* RDG_createLiteralDistrib(double ld)
|
||||
static void RDG_fillLiteralDistrib(litDistribTable lt, double ld)
|
||||
{
|
||||
BYTE* lt = (BYTE*)malloc(LTSIZE);
|
||||
U32 i = 0;
|
||||
BYTE character = '0';
|
||||
BYTE firstChar = '(';
|
||||
BYTE lastChar = '}';
|
||||
BYTE const firstChar = ld <= 0.0 ? 0 : '(';
|
||||
BYTE const lastChar = ld <= 0.0 ? 255 : '}';
|
||||
BYTE character = ld <= 0.0 ? 0 : '0';
|
||||
U32 u = 0;
|
||||
|
||||
if (ld==0.0)
|
||||
{
|
||||
character = 0;
|
||||
firstChar = 0;
|
||||
lastChar =255;
|
||||
}
|
||||
while (i<LTSIZE)
|
||||
{
|
||||
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
|
||||
U32 end;
|
||||
if (weight + i > LTSIZE) weight = LTSIZE-i;
|
||||
end = i + weight;
|
||||
while (i < end) lt[i++] = character;
|
||||
while (u<LTSIZE) {
|
||||
U32 const weight = (U32)((double)(LTSIZE - u) * ld) + 1;
|
||||
U32 const end = MIN(u+weight, LTSIZE);
|
||||
while (u < end) lt[u++] = character;
|
||||
character++;
|
||||
if (character > lastChar) character = firstChar;
|
||||
}
|
||||
return lt;
|
||||
}
|
||||
|
||||
static char RDG_genChar(U32* seed, const void* ltctx)
|
||||
|
||||
static BYTE RDG_genChar(U32* seed, const litDistribTable lt)
|
||||
{
|
||||
const BYTE* lt = (const BYTE*)ltctx;
|
||||
U32 id = RDG_rand(seed) & LTMASK;
|
||||
return lt[id];
|
||||
return (lt[id]);
|
||||
}
|
||||
|
||||
|
||||
#define RDG_DICTSIZE (32 KB)
|
||||
#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767)
|
||||
#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15)
|
||||
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, void* litTable, unsigned* seedPtr)
|
||||
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr)
|
||||
{
|
||||
BYTE* buffPtr = (BYTE*)buffer;
|
||||
const U32 matchProba32 = (U32)(32768 * matchProba);
|
||||
size_t pos = prefixSize;
|
||||
void* ldctx = litTable;
|
||||
U32* seed = seedPtr;
|
||||
|
||||
/* special case */
|
||||
@@ -146,11 +115,11 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
|
||||
}
|
||||
memset(buffPtr+pos, 0, size0);
|
||||
pos += size0;
|
||||
buffPtr[pos-1] = RDG_genChar(seed, ldctx);
|
||||
buffPtr[pos-1] = RDG_genChar(seed, lt);
|
||||
}
|
||||
|
||||
/* init */
|
||||
if (pos==0) buffPtr[0] = RDG_genChar(seed, ldctx), pos=1;
|
||||
if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1;
|
||||
|
||||
/* Generate compressible data */
|
||||
while (pos < buffSize)
|
||||
@@ -176,7 +145,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
|
||||
size_t length = RDG_RANDLENGTH;
|
||||
d = pos + length;
|
||||
if (d > buffSize) d = buffSize;
|
||||
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldctx);
|
||||
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -184,11 +153,10 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
|
||||
|
||||
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
|
||||
{
|
||||
void* ldctx;
|
||||
litDistribTable lt;
|
||||
if (litProba==0.0) litProba = matchProba / 4.5;
|
||||
ldctx = RDG_createLiteralDistrib(litProba);
|
||||
RDG_genBlock(buffer, size, 0, matchProba, ldctx, &seed);
|
||||
free(ldctx);
|
||||
RDG_fillLiteralDistrib(lt, litProba);
|
||||
RDG_genBlock(buffer, size, 0, matchProba, lt, &seed);
|
||||
}
|
||||
|
||||
|
||||
@@ -198,26 +166,24 @@ void RDG_genOut(unsigned long long size, double matchProba, double litProba, uns
|
||||
BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE];
|
||||
U64 total = 0;
|
||||
size_t genBlockSize = RDG_BLOCKSIZE;
|
||||
void* ldctx;
|
||||
litDistribTable lt;
|
||||
|
||||
/* init */
|
||||
if (litProba==0.0) litProba = matchProba / 4.5;
|
||||
ldctx = RDG_createLiteralDistrib(litProba);
|
||||
RDG_fillLiteralDistrib(lt, litProba);
|
||||
SET_BINARY_MODE(stdout);
|
||||
|
||||
/* Generate dict */
|
||||
RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed);
|
||||
RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed);
|
||||
|
||||
/* Generate compressible data */
|
||||
while (total < size)
|
||||
{
|
||||
RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed);
|
||||
RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed);
|
||||
if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total);
|
||||
total += genBlockSize;
|
||||
fwrite(buff, 1, genBlockSize, stdout);
|
||||
/* update dict */
|
||||
memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE);
|
||||
}
|
||||
|
||||
free(ldctx);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/*
|
||||
datagen.h - compressible data generator header
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
Copyright (C) Yann Collet 2012-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD source repository : https://github.com/Cyan4973/zstd
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
/*
|
||||
datagencli.c
|
||||
compressible data command line generator
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD source repository : https://github.com/Cyan4973/zstd
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdio.h> /* fprintf, stderr */
|
||||
#include "datagen.h" /* RDG_generate */
|
||||
|
||||
|
||||
/**************************************
|
||||
* Basic Types
|
||||
**************************************/
|
||||
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION "r1"
|
||||
#endif
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define SIZE_DEFAULT (64 KB)
|
||||
#define SEED_DEFAULT 0
|
||||
#define COMPRESSIBILITY_DEFAULT 50
|
||||
|
||||
|
||||
/**************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static unsigned displayLevel = 2;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
static int usage(char* programName)
|
||||
{
|
||||
DISPLAY( "Compressible data generator\n");
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [size] [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
|
||||
DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT);
|
||||
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
DISPLAY( "Special values :\n");
|
||||
DISPLAY( " -P0 : generate incompressible noise\n");
|
||||
DISPLAY( " -P100 : generate sparse files\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int argNb;
|
||||
double proba = (double)COMPRESSIBILITY_DEFAULT / 100;
|
||||
double litProba = 0.0;
|
||||
U64 size = SIZE_DEFAULT;
|
||||
U32 seed = SEED_DEFAULT;
|
||||
char* programName;
|
||||
|
||||
/* Check command line */
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Handle commands. Aggregated commands are allowed */
|
||||
if (*argument=='-')
|
||||
{
|
||||
argument++;
|
||||
while (*argument!=0)
|
||||
{
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return usage(programName);
|
||||
case 'g':
|
||||
argument++;
|
||||
size=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
size *= 10;
|
||||
size += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (*argument=='K') { size <<= 10; argument++; }
|
||||
if (*argument=='M') { size <<= 20; argument++; }
|
||||
if (*argument=='G') { size <<= 30; argument++; }
|
||||
if (*argument=='B') { argument++; }
|
||||
break;
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 'P':
|
||||
argument++;
|
||||
proba=0.0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba>100.) proba=100.;
|
||||
proba /= 100.;
|
||||
break;
|
||||
case 'L': /* hidden argument : Literal distribution probability */
|
||||
argument++;
|
||||
litProba=0.;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
litProba *= 10;
|
||||
litProba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (litProba>100.) litProba=100.;
|
||||
litProba /= 100.;
|
||||
break;
|
||||
case 'v':
|
||||
displayLevel = 4;
|
||||
argument++;
|
||||
break;
|
||||
default:
|
||||
return usage(programName);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
|
||||
|
||||
RDG_genOut(size, proba, litProba, seed);
|
||||
DISPLAYLEVEL(1, "\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,925 +0,0 @@
|
||||
/*
|
||||
frameTest - test tool for lz4frame
|
||||
Copyright (C) Yann Collet 2014-2015
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : https://github.com/Cyan4973/lz4
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Compiler specific
|
||||
**************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
|
||||
#endif
|
||||
|
||||
/* S_ISREG & gettimeofday() are not supported by MSVC */
|
||||
#if defined(_MSC_VER) || defined(_WIN32)
|
||||
# define FUZ_LEGACY_TIMER 1
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <string.h> /* strcmp */
|
||||
#include "lz4frame_static.h"
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
|
||||
/* Use ftime() if gettimeofday() is not available on your target */
|
||||
#if defined(FUZ_LEGACY_TIMER)
|
||||
# include <sys/timeb.h> /* timeb, ftime */
|
||||
#else
|
||||
# include <sys/time.h> /* gettimeofday */
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Basic Types
|
||||
**************************************/
|
||||
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/* unoptimized version; solves endianess & alignment issues */
|
||||
static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
|
||||
{
|
||||
BYTE* dstPtr = (BYTE*)dstVoidPtr;
|
||||
dstPtr[0] = (BYTE)value32;
|
||||
dstPtr[1] = (BYTE)(value32 >> 8);
|
||||
dstPtr[2] = (BYTE)(value32 >> 16);
|
||||
dstPtr[3] = (BYTE)(value32 >> 24);
|
||||
}
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION ""
|
||||
#endif
|
||||
|
||||
#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const U32 nbTestsDefault = 256 KB;
|
||||
#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
|
||||
|
||||
|
||||
/**************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
|
||||
if ((FUZ_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=4)) \
|
||||
{ g_time = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel>=4) fflush(stdout); } }
|
||||
static const U32 refreshRate = 150;
|
||||
static U32 g_time = 0;
|
||||
|
||||
|
||||
/*****************************************
|
||||
* Local Parameters
|
||||
*****************************************/
|
||||
static U32 no_prompt = 0;
|
||||
static char* programName;
|
||||
static U32 displayLevel = 2;
|
||||
static U32 pause = 0;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#if defined(FUZ_LEGACY_TIMER)
|
||||
|
||||
static U32 FUZ_GetMilliStart(void)
|
||||
{
|
||||
struct timeb tb;
|
||||
U32 nCount;
|
||||
ftime( &tb );
|
||||
nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static U32 FUZ_GetMilliStart(void)
|
||||
{
|
||||
struct timeval tv;
|
||||
U32 nCount;
|
||||
gettimeofday(&tv, NULL);
|
||||
nCount = (U32) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
||||
{
|
||||
U32 nCurrent = FUZ_GetMilliStart();
|
||||
U32 nSpan = nCurrent - nTimeStart;
|
||||
if (nTimeStart > nCurrent)
|
||||
nSpan += 0x100000 * 1000;
|
||||
return nSpan;
|
||||
}
|
||||
|
||||
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* src)
|
||||
{
|
||||
U32 rand32 = *src;
|
||||
rand32 *= prime1;
|
||||
rand32 += prime2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
|
||||
#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF)
|
||||
#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
|
||||
static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, double proba, U32* seed)
|
||||
{
|
||||
BYTE* BBuffer = (BYTE*)buffer;
|
||||
unsigned pos = 0;
|
||||
U32 P32 = (U32)(32768 * proba);
|
||||
|
||||
/* First Byte */
|
||||
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
|
||||
|
||||
while (pos < bufferSize)
|
||||
{
|
||||
/* Select : Literal (noise) or copy (within 64K) */
|
||||
if (FUZ_RAND15BITS < P32)
|
||||
{
|
||||
/* Copy (within 64K) */
|
||||
unsigned match, end;
|
||||
unsigned length = FUZ_RANDLENGTH + 4;
|
||||
unsigned offset = FUZ_RAND15BITS + 1;
|
||||
if (offset > pos) offset = pos;
|
||||
if (pos + length > bufferSize) length = bufferSize - pos;
|
||||
match = pos - offset;
|
||||
end = pos + length;
|
||||
while (pos < end) BBuffer[pos++] = BBuffer[match++];
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Literal (noise) */
|
||||
unsigned end;
|
||||
unsigned length = FUZ_RANDLENGTH;
|
||||
if (pos + length > bufferSize) length = bufferSize - pos;
|
||||
end = pos + length;
|
||||
while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static unsigned FUZ_highbit(U32 v32)
|
||||
{
|
||||
unsigned nbBits = 0;
|
||||
if (v32==0) return 0;
|
||||
while (v32)
|
||||
{
|
||||
v32 >>= 1;
|
||||
nbBits ++;
|
||||
}
|
||||
return nbBits;
|
||||
}
|
||||
|
||||
|
||||
int basicTests(U32 seed, double compressibility)
|
||||
{
|
||||
int testResult = 0;
|
||||
void* CNBuffer;
|
||||
void* compressedBuffer;
|
||||
void* decodedBuffer;
|
||||
U32 randState = seed;
|
||||
size_t cSize, testSize;
|
||||
LZ4F_preferences_t prefs;
|
||||
LZ4F_decompressionContext_t dCtx = NULL;
|
||||
LZ4F_compressionContext_t cctx = NULL;
|
||||
U64 crcOrig;
|
||||
|
||||
/* Create compressible test buffer */
|
||||
memset(&prefs, 0, sizeof(prefs));
|
||||
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
|
||||
compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL));
|
||||
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
|
||||
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
|
||||
crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
|
||||
/* Trivial tests : one-step frame */
|
||||
testSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
DISPLAYLEVEL(3, "Using NULL preferences : \n");
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Decompression test : \n");
|
||||
{
|
||||
size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
size_t compressedBufferSize = cSize;
|
||||
BYTE* op = (BYTE*)decodedBuffer;
|
||||
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
|
||||
BYTE* ip = (BYTE*)compressedBuffer;
|
||||
BYTE* const iend = (BYTE*)compressedBuffer + cSize;
|
||||
U64 crcDest;
|
||||
|
||||
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
|
||||
DISPLAYLEVEL(3, "Single Block : \n");
|
||||
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
|
||||
|
||||
DISPLAYLEVEL(4, "Reusing decompression context \n");
|
||||
{
|
||||
size_t iSize = compressedBufferSize - 4;
|
||||
const BYTE* cBuff = (const BYTE*) compressedBuffer;
|
||||
DISPLAYLEVEL(3, "Missing last 4 bytes : ");
|
||||
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
if (!errorCode) goto _output_error;
|
||||
DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)errorCode);
|
||||
cBuff += iSize;
|
||||
iSize = errorCode;
|
||||
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL);
|
||||
if (errorCode != 0) goto _output_error;
|
||||
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
}
|
||||
|
||||
{
|
||||
size_t oSize = 0;
|
||||
size_t iSize = 0;
|
||||
LZ4F_frameInfo_t fi;
|
||||
|
||||
DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
|
||||
errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
|
||||
|
||||
DISPLAYLEVEL(3, "get FrameInfo on null input : ");
|
||||
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
|
||||
if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
|
||||
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
|
||||
|
||||
DISPLAYLEVEL(3, "get FrameInfo on not enough input : ");
|
||||
iSize = 6;
|
||||
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
|
||||
if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
|
||||
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
|
||||
ip += iSize;
|
||||
|
||||
DISPLAYLEVEL(3, "get FrameInfo on enough input : ");
|
||||
iSize = 15 - iSize;
|
||||
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
DISPLAYLEVEL(3, " correctly decoded \n");
|
||||
ip += iSize;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "Byte after byte : \n");
|
||||
while (ip < iend)
|
||||
{
|
||||
size_t oSize = oend-op;
|
||||
size_t iSize = 1;
|
||||
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
}
|
||||
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH);
|
||||
|
||||
errorCode = LZ4F_freeDecompressionContext(dCtx);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "Using 64 KB block : \n");
|
||||
prefs.frameInfo.blockSizeID = LZ4F_max64KB;
|
||||
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "without checksum : \n");
|
||||
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Using 256 KB block : \n");
|
||||
prefs.frameInfo.blockSizeID = LZ4F_max256KB;
|
||||
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Decompression test : \n");
|
||||
{
|
||||
size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
|
||||
BYTE* op = (BYTE*)decodedBuffer;
|
||||
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
|
||||
BYTE* ip = (BYTE*)compressedBuffer;
|
||||
BYTE* const iend = (BYTE*)compressedBuffer + cSize;
|
||||
U64 crcDest;
|
||||
|
||||
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
|
||||
DISPLAYLEVEL(3, "random segment sizes : \n");
|
||||
while (ip < iend)
|
||||
{
|
||||
unsigned nbBits = FUZ_rand(&randState) % maxBits;
|
||||
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
|
||||
size_t oSize = oend-op;
|
||||
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
|
||||
//DISPLAY("%7i : + %6i\n", (int)(ip-(BYTE*)compressedBuffer), (int)iSize);
|
||||
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
}
|
||||
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
|
||||
|
||||
errorCode = LZ4F_freeDecompressionContext(dCtx);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "without checksum : \n");
|
||||
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Using 1 MB block : \n");
|
||||
prefs.frameInfo.blockSizeID = LZ4F_max1MB;
|
||||
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "without checksum : \n");
|
||||
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Using 4 MB block : \n");
|
||||
prefs.frameInfo.blockSizeID = LZ4F_max4MB;
|
||||
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "without checksum : \n");
|
||||
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
{
|
||||
size_t errorCode;
|
||||
BYTE* const ostart = (BYTE*)compressedBuffer;
|
||||
BYTE* op = ostart;
|
||||
errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
|
||||
DISPLAYLEVEL(3, "compress without frameSize : \n");
|
||||
memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
|
||||
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += errorCode;
|
||||
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += errorCode;
|
||||
errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
|
||||
|
||||
DISPLAYLEVEL(3, "compress with frameSize : \n");
|
||||
prefs.frameInfo.contentSize = testSize;
|
||||
op = ostart;
|
||||
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += errorCode;
|
||||
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += errorCode;
|
||||
errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
|
||||
|
||||
DISPLAYLEVEL(3, "compress with wrong frameSize : \n");
|
||||
prefs.frameInfo.contentSize = testSize+1;
|
||||
op = ostart;
|
||||
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += errorCode;
|
||||
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += errorCode;
|
||||
errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); }
|
||||
else
|
||||
goto _output_error;
|
||||
|
||||
errorCode = LZ4F_freeCompressionContext(cctx);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
cctx = NULL;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "Skippable frame test : \n");
|
||||
{
|
||||
size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
|
||||
BYTE* op = (BYTE*)decodedBuffer;
|
||||
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
|
||||
BYTE* ip = (BYTE*)compressedBuffer;
|
||||
BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
|
||||
|
||||
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
|
||||
/* generate skippable frame */
|
||||
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
|
||||
FUZ_writeLE32(ip+4, (U32)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "random segment sizes : \n");
|
||||
while (ip < iend)
|
||||
{
|
||||
unsigned nbBits = FUZ_rand(&randState) % maxBits;
|
||||
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
|
||||
size_t oSize = oend-op;
|
||||
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
|
||||
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
}
|
||||
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize);
|
||||
|
||||
/* generate zero-size skippable frame */
|
||||
DISPLAYLEVEL(3, "zero-size skippable frame\n");
|
||||
ip = (BYTE*)compressedBuffer;
|
||||
op = (BYTE*)decodedBuffer;
|
||||
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1);
|
||||
FUZ_writeLE32(ip+4, 0);
|
||||
iend = ip+8;
|
||||
|
||||
while (ip < iend)
|
||||
{
|
||||
unsigned nbBits = FUZ_rand(&randState) % maxBits;
|
||||
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
|
||||
size_t oSize = oend-op;
|
||||
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
|
||||
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
}
|
||||
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
|
||||
|
||||
DISPLAYLEVEL(3, "Skippable frame header complete in first call \n");
|
||||
ip = (BYTE*)compressedBuffer;
|
||||
op = (BYTE*)decodedBuffer;
|
||||
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2);
|
||||
FUZ_writeLE32(ip+4, 10);
|
||||
iend = ip+18;
|
||||
while (ip < iend)
|
||||
{
|
||||
size_t iSize = 10;
|
||||
size_t oSize = 10;
|
||||
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
|
||||
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
}
|
||||
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
|
||||
}
|
||||
|
||||
DISPLAY("Basic tests completed \n");
|
||||
_end:
|
||||
free(CNBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
|
||||
LZ4F_freeCompressionContext(cctx); cctx = NULL;
|
||||
return testResult;
|
||||
|
||||
_output_error:
|
||||
testResult = 1;
|
||||
DISPLAY("Error detected ! \n");
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
||||
static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
|
||||
{
|
||||
int p=0;
|
||||
const BYTE* b1=(const BYTE*)buff1;
|
||||
const BYTE* b2=(const BYTE*)buff2;
|
||||
if (nonContiguous)
|
||||
{
|
||||
DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
|
||||
return;
|
||||
}
|
||||
while (b1[p]==b2[p]) p++;
|
||||
DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]);
|
||||
}
|
||||
|
||||
|
||||
static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
|
||||
|
||||
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration)
|
||||
{
|
||||
unsigned testResult = 0;
|
||||
unsigned testNb = 0;
|
||||
void* srcBuffer = NULL;
|
||||
void* compressedBuffer = NULL;
|
||||
void* decodedBuffer = NULL;
|
||||
U32 coreRand = seed;
|
||||
LZ4F_decompressionContext_t dCtx = NULL;
|
||||
LZ4F_compressionContext_t cCtx = NULL;
|
||||
size_t result;
|
||||
const U32 startTime = FUZ_GetMilliStart();
|
||||
XXH64_state_t xxh64;
|
||||
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
|
||||
/* Init */
|
||||
duration *= 1000;
|
||||
|
||||
/* Create buffers */
|
||||
result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
|
||||
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
|
||||
result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
|
||||
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
|
||||
srcBuffer = malloc(srcDataLength);
|
||||
CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
|
||||
compressedBuffer = malloc(LZ4F_compressFrameBound(srcDataLength, NULL));
|
||||
CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
|
||||
decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
|
||||
CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
|
||||
FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
|
||||
|
||||
/* jump to requested testNb */
|
||||
for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
|
||||
|
||||
/* main fuzzer test loop */
|
||||
for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++)
|
||||
{
|
||||
U32 randState = coreRand ^ prime1;
|
||||
unsigned BSId = 4 + (FUZ_rand(&randState) & 3);
|
||||
unsigned BMId = FUZ_rand(&randState) & 1;
|
||||
unsigned CCflag = FUZ_rand(&randState) & 1;
|
||||
unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
|
||||
LZ4F_preferences_t prefs;
|
||||
LZ4F_compressOptions_t cOptions;
|
||||
LZ4F_decompressOptions_t dOptions;
|
||||
unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
|
||||
size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
|
||||
size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
|
||||
U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
|
||||
size_t cSize;
|
||||
U64 crcOrig, crcDecoded;
|
||||
LZ4F_preferences_t* prefsPtr = &prefs;
|
||||
|
||||
(void)FUZ_rand(&coreRand); /* update seed */
|
||||
memset(&prefs, 0, sizeof(prefs));
|
||||
memset(&cOptions, 0, sizeof(cOptions));
|
||||
memset(&dOptions, 0, sizeof(dOptions));
|
||||
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId;
|
||||
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId;
|
||||
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag;
|
||||
prefs.frameInfo.contentSize = frameContentSize;
|
||||
prefs.autoFlush = autoflush;
|
||||
prefs.compressionLevel = FUZ_rand(&randState) % 5;
|
||||
if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
|
||||
|
||||
DISPLAYUPDATE(2, "\r%5u ", testNb);
|
||||
crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);
|
||||
|
||||
if ((FUZ_rand(&randState) & 0xFFF) == 0)
|
||||
{
|
||||
/* create a skippable frame (rare case) */
|
||||
BYTE* op = (BYTE*)compressedBuffer;
|
||||
FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15));
|
||||
FUZ_writeLE32(op+4, (U32)srcSize);
|
||||
cSize = srcSize+8;
|
||||
}
|
||||
else if ((FUZ_rand(&randState) & 0xF) == 2)
|
||||
{
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), (char*)srcBuffer + srcStart, srcSize, prefsPtr);
|
||||
CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
|
||||
}
|
||||
else
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)srcBuffer + srcStart;
|
||||
const BYTE* const iend = ip + srcSize;
|
||||
BYTE* op = (BYTE*)compressedBuffer;
|
||||
BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL);
|
||||
unsigned maxBits = FUZ_highbit((U32)srcSize);
|
||||
result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr);
|
||||
CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result);
|
||||
op += result;
|
||||
while (ip < iend)
|
||||
{
|
||||
unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits;
|
||||
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
|
||||
size_t oSize = LZ4F_compressBound(iSize, prefsPtr);
|
||||
unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
|
||||
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
|
||||
cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
|
||||
|
||||
result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
|
||||
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
|
||||
op += result;
|
||||
ip += iSize;
|
||||
|
||||
if (forceFlush)
|
||||
{
|
||||
result = LZ4F_flush(cCtx, op, oend-op, &cOptions);
|
||||
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
|
||||
op += result;
|
||||
}
|
||||
}
|
||||
result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions);
|
||||
CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result);
|
||||
op += result;
|
||||
cSize = op-(BYTE*)compressedBuffer;
|
||||
}
|
||||
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)compressedBuffer;
|
||||
const BYTE* const iend = ip + cSize;
|
||||
BYTE* op = (BYTE*)decodedBuffer;
|
||||
BYTE* const oend = op + srcDataLength;
|
||||
size_t totalOut = 0;
|
||||
unsigned maxBits = FUZ_highbit((U32)cSize);
|
||||
unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1;
|
||||
nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */
|
||||
XXH64_reset(&xxh64, 1);
|
||||
if (maxBits < 3) maxBits = 3;
|
||||
while (ip < iend)
|
||||
{
|
||||
unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
|
||||
unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1;
|
||||
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1;
|
||||
size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
|
||||
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
|
||||
if (oSize > (size_t)(oend-op)) oSize = oend-op;
|
||||
dOptions.stableDst = FUZ_rand(&randState) & 1;
|
||||
if (nonContiguousDst==2) dOptions.stableDst = 0;
|
||||
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
|
||||
if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid)
|
||||
locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
|
||||
CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result));
|
||||
XXH64_update(&xxh64, op, (U32)oSize);
|
||||
totalOut += oSize;
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
op += nonContiguousDst;
|
||||
if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */
|
||||
}
|
||||
CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
|
||||
if (totalOut) /* otherwise, it's a skippable frame */
|
||||
{
|
||||
crcDecoded = XXH64_digest(&xxh64);
|
||||
if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
|
||||
CHECK(crcDecoded != crcOrig, "Decompression corruption");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "\rAll tests completed \n");
|
||||
|
||||
_end:
|
||||
LZ4F_freeDecompressionContext(dCtx);
|
||||
LZ4F_freeCompressionContext(cCtx);
|
||||
free(srcBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
|
||||
if (pause)
|
||||
{
|
||||
DISPLAY("press enter to finish \n");
|
||||
(void)getchar();
|
||||
}
|
||||
return testResult;
|
||||
|
||||
_output_error:
|
||||
testResult = 1;
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
||||
int FUZ_usage(void)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
|
||||
DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
|
||||
DISPLAY( " -s# : Select seed (default:prompt user)\n");
|
||||
DISPLAY( " -t# : Select starting test number (default:0)\n");
|
||||
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -v : verbose\n");
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
U32 seed=0;
|
||||
int seedset=0;
|
||||
int argNb;
|
||||
int nbTests = nbTestsDefault;
|
||||
int testNb = 0;
|
||||
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
|
||||
int result=0;
|
||||
U32 duration=0;
|
||||
|
||||
/* Check command line */
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Decode command (note : aggregated commands are allowed) */
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
if (!strcmp(argument, "--no-prompt"))
|
||||
{
|
||||
no_prompt=1;
|
||||
seedset=1;
|
||||
displayLevel=1;
|
||||
continue;
|
||||
}
|
||||
argument++;
|
||||
|
||||
while (*argument!=0)
|
||||
{
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return FUZ_usage();
|
||||
case 'v':
|
||||
argument++;
|
||||
displayLevel=4;
|
||||
break;
|
||||
case 'q':
|
||||
argument++;
|
||||
displayLevel--;
|
||||
break;
|
||||
case 'p': /* pause at the end */
|
||||
argument++;
|
||||
pause = 1;
|
||||
break;
|
||||
|
||||
case 'i':
|
||||
argument++;
|
||||
nbTests=0; duration=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
nbTests *= 10;
|
||||
nbTests += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'T':
|
||||
argument++;
|
||||
nbTests = 0; duration = 0;
|
||||
for (;;)
|
||||
{
|
||||
switch(*argument)
|
||||
{
|
||||
case 'm': duration *= 60; argument++; continue;
|
||||
case 's':
|
||||
case 'n': argument++; continue;
|
||||
case '0':
|
||||
case '1':
|
||||
case '2':
|
||||
case '3':
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
case '8':
|
||||
case '9': duration *= 10; duration += *argument++ - '0'; continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
seedset=1;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 't':
|
||||
argument++;
|
||||
testNb=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
testNb *= 10;
|
||||
testNb += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 'P': /* compressibility % */
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba<0) proba=0;
|
||||
if (proba>100) proba=100;
|
||||
break;
|
||||
default:
|
||||
;
|
||||
return FUZ_usage();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Get Seed */
|
||||
printf("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);
|
||||
|
||||
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
|
||||
printf("Seed = %u\n", seed);
|
||||
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
|
||||
|
||||
if (nbTests<=0) nbTests=1;
|
||||
|
||||
if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
|
||||
if (result) return 1;
|
||||
return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration);
|
||||
}
|
||||
1006
programs/fullbench.c
1006
programs/fullbench.c
File diff suppressed because it is too large
Load Diff
1233
programs/fuzzer.c
1233
programs/fuzzer.c
File diff suppressed because it is too large
Load Diff
385
programs/lz4.1
385
programs/lz4.1
@@ -1,221 +1,220 @@
|
||||
\"
|
||||
\" lz4.1: This is a manual page for 'lz4' program. This file is part of the
|
||||
\" lz4 <https://code.google.com/p/lz4/> project.
|
||||
\" Author: Yann Collet
|
||||
\"
|
||||
.
|
||||
.TH "LZ4" "1" "July 2017" "lz4 1.8.0" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
|
||||
.
|
||||
.SH "SYNOPSIS"
|
||||
\fBlz4\fR [\fIOPTIONS\fR] [\-|INPUT\-FILE] \fIOUTPUT\-FILE\fR
|
||||
.
|
||||
.P
|
||||
\fBunlz4\fR is equivalent to \fBlz4 \-d\fR
|
||||
.
|
||||
.P
|
||||
\fBlz4cat\fR is equivalent to \fBlz4 \-dcfm\fR
|
||||
.
|
||||
.P
|
||||
When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\.
|
||||
.
|
||||
.SH "DESCRIPTION"
|
||||
\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\.
|
||||
.
|
||||
.SS "Difference between lz4 and gzip"
|
||||
\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are :
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fBlz4\fR preserves original files
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files)
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression)
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them)
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\.
|
||||
.
|
||||
.IP "" 0
|
||||
.
|
||||
.P
|
||||
Default behaviors can be modified by opt\-in commands, detailed below\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
Similarly, \fBlz4 \-m \-d\fR can decompress multiple \fB*\.lz4\fR files\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\.
|
||||
.
|
||||
.IP "" 0
|
||||
.
|
||||
.SS "Concatenation of \.lz4 files"
|
||||
It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example:
|
||||
.
|
||||
.IP "" 4
|
||||
.
|
||||
.nf
|
||||
|
||||
\" No hyphenation
|
||||
.hy 0
|
||||
.nr HY 0
|
||||
|
||||
.TH lz4 "1" "2015-03-21" "lz4" "User Commands"
|
||||
.SH NAME
|
||||
\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files
|
||||
|
||||
.SH SYNOPSIS
|
||||
.TP 5
|
||||
\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
|
||||
.PP
|
||||
.B unlz4
|
||||
is equivalent to
|
||||
.BR "lz4 \-d"
|
||||
.br
|
||||
.B lz4cat
|
||||
is equivalent to
|
||||
.BR "lz4 \-dc"
|
||||
.br
|
||||
.PP
|
||||
When writing scripts that need to decompress files,
|
||||
it is recommended to always use the name
|
||||
.B lz4
|
||||
with appropriate arguments
|
||||
.RB ( "lz4 \-d"
|
||||
or
|
||||
.BR "lz4 \-dc" )
|
||||
instead of the names
|
||||
.B unlz4
|
||||
and
|
||||
.BR lz4cat .
|
||||
|
||||
|
||||
.SH DESCRIPTION
|
||||
.PP
|
||||
\fBlz4\fR is an extremely fast lossless compression algorithm,
|
||||
based on \fBbyte-aligned LZ77\fR family of compression scheme.
|
||||
\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs.
|
||||
It features an extremely fast decoder, with speed in multiple GB/s per core,
|
||||
typically reaching RAM speed limit on multi-core systems.
|
||||
.B lz4
|
||||
supports a command line syntax similar to
|
||||
.BR gzip (1).
|
||||
The native file format is the
|
||||
.B .lz4
|
||||
format.
|
||||
|
||||
.SS "Concatenation of .lz4 files"
|
||||
It is possible to concatenate
|
||||
.B .lz4
|
||||
files as is.
|
||||
.B lz4
|
||||
will decompress such files as if they were a single
|
||||
.B .lz4
|
||||
file. For example:
|
||||
lz4 file1 > foo.lz4
|
||||
lz4 file2 >> foo.lz4
|
||||
then
|
||||
lz4cat foo.lz4
|
||||
is equivalent to :
|
||||
cat file1 file2
|
||||
|
||||
.PP
|
||||
|
||||
.SH OPTIONS
|
||||
lz4 file1 > foo\.lz4
|
||||
lz4 file2 >> foo\.lz4
|
||||
.
|
||||
.fi
|
||||
.
|
||||
.IP "" 0
|
||||
.
|
||||
.P
|
||||
Then \fBlz4cat foo\.lz4\fR is equivalent to \fBcat file1 file2\fR\.
|
||||
.
|
||||
.SH "OPTIONS"
|
||||
.
|
||||
.SS "Short commands concatenation"
|
||||
In some cases, some options can be expressed using short command
|
||||
.B "-x"
|
||||
or long command
|
||||
.B "--long-word" .
|
||||
Short commands can be concatenated together. For example,
|
||||
.B "-d -c"
|
||||
is equivalent to
|
||||
.B "-dc" .
|
||||
Long commands cannot be concatenated.
|
||||
They must be clearly separated by a space.
|
||||
In some cases, some options can be expressed using short command \fB\-x\fR or long command \fB\-\-long\-word\fR\. Short commands can be concatenated together\. For example, \fB\-d \-c\fR is equivalent to \fB\-dc\fR\. Long commands cannot be concatenated\. They must be clearly separated by a space\.
|
||||
.
|
||||
.SS "Multiple commands"
|
||||
When multiple contradictory commands are issued on a same command line,
|
||||
only the latest one will be applied.
|
||||
When multiple contradictory commands are issued on a same command line, only the latest one will be applied\.
|
||||
.
|
||||
.SS "Operation mode"
|
||||
.
|
||||
.TP
|
||||
.BR \-z ", " \-\-compress
|
||||
Compress.
|
||||
This is the default operation mode
|
||||
when no operation mode option is specified ,
|
||||
no other operation mode is implied from the command name
|
||||
(for example,
|
||||
.B unlz4
|
||||
implies
|
||||
.B \-\-decompress ),
|
||||
nor from the input file name
|
||||
(for example, a file extension
|
||||
.B .lz4
|
||||
implies
|
||||
.B \-\-decompress
|
||||
by default).
|
||||
.B -z
|
||||
can also be used to force compression of an already compressed
|
||||
.B .lz4
|
||||
file.
|
||||
\fB\-z\fR \fB\-\-compress\fR
|
||||
Compress\. This is the default operation mode when no operation mode option is specified, no other operation mode is implied from the command name (for example, \fBunlz4\fR implies \fB\-\-decompress\fR), nor from the input file name (for example, a file extension \fB\.lz4\fR implies \fB\-\-decompress\fR by default)\. \fB\-z\fR can also be used to force compression of an already compressed \fB\.lz4\fR file\.
|
||||
.
|
||||
.TP
|
||||
.BR \-d ", " \-\-decompress ", " \-\-uncompress
|
||||
Decompress.
|
||||
.B --decompress
|
||||
is also the default operation when the input filename has an
|
||||
.B .lz4
|
||||
extensionq
|
||||
\fB\-d\fR \fB\-\-decompress\fR \fB\-\-uncompress\fR
|
||||
Decompress\. \fB\-\-decompress\fR is also the default operation when the input filename has an \fB\.lz4\fR extension\.
|
||||
.
|
||||
.TP
|
||||
.BR \-t ", " \-\-test
|
||||
Test the integrity of compressed
|
||||
.B .lz4
|
||||
files.
|
||||
The decompressed data is discarded.
|
||||
No files are created nor removed.
|
||||
\fB\-t\fR \fB\-\-test\fR
|
||||
Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is discarded\. No files are created nor removed\.
|
||||
.
|
||||
.TP
|
||||
\fB\-b#\fR
|
||||
Benchmark mode, using \fB#\fR compression level\.
|
||||
.
|
||||
.SS "Operation modifiers"
|
||||
.TP
|
||||
.B \-1
|
||||
fast compression (default)
|
||||
.TP
|
||||
.B \-9
|
||||
high compression
|
||||
|
||||
.TP
|
||||
.BR \-f ", " --[no-]force
|
||||
This option has several effects:
|
||||
.RS
|
||||
.IP \(bu 3
|
||||
If the target file already exists,
|
||||
overwrite it without prompting.
|
||||
.IP \(bu 3
|
||||
When used with
|
||||
.B \-\-decompress
|
||||
and
|
||||
.B lz4
|
||||
cannot recognize the type of the source file,
|
||||
copy the source file as is to standard output.
|
||||
This allows
|
||||
.B lz4cat
|
||||
.B \-\-force
|
||||
to be used like
|
||||
.BR cat (1)
|
||||
for files that have not been compressed with
|
||||
.BR lz4 .
|
||||
.RE
|
||||
|
||||
.TP
|
||||
.BR \-c ", " \--stdout ", " \--to-stdout
|
||||
force write to standard output, even if it is the console
|
||||
|
||||
.TP
|
||||
.BR \-m ", " \--multiple
|
||||
Multiple file names.
|
||||
By default, the second filename is used as the output filename for the compressed file.
|
||||
With
|
||||
.B -m
|
||||
, you can specify any number of input filenames. Each of them will be compressed
|
||||
independently, and the resulting name of the compressed file will be
|
||||
.B filename.lz4
|
||||
.
|
||||
|
||||
.TP
|
||||
.B \-B#
|
||||
block size [4-7](default : 7)
|
||||
B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB
|
||||
\fB\-#\fR
|
||||
Compression level, with # being any value from 1 to 16\. Higher values trade compression speed for compression ratio\. Values above 16 are considered the same as 16\. Recommended values are 1 for fast compression (default), and 9 for high compression\. Speed/compression trade\-off will vary depending on data to compress\. Decompression speed remains fast at all settings\.
|
||||
.
|
||||
.TP
|
||||
.B \-BD
|
||||
block dependency (improve compression ratio)
|
||||
\fB\-f\fR \fB\-\-[no\-]force\fR
|
||||
This option has several effects:
|
||||
.
|
||||
.IP
|
||||
If the target file already exists, overwrite it without prompting\.
|
||||
.
|
||||
.IP
|
||||
When used with \fB\-\-decompress\fR and \fBlz4\fR cannot recognize the type of the source file, copy the source file as is to standard output\. This allows \fBlz4cat \-\-force\fR to be used like \fBcat (1)\fR for files that have not been compressed with \fBlz4\fR\.
|
||||
.
|
||||
.TP
|
||||
.B \--[no-]frame-crc
|
||||
select frame checksum (default:enabled)
|
||||
\fB\-c\fR \fB\-\-stdout\fR \fB\-\-to\-stdout\fR
|
||||
Force write to standard output, even if it is the console\.
|
||||
.
|
||||
.TP
|
||||
.B \--[no-]content-size
|
||||
header includes original size (default:not present)
|
||||
Note : this option can only be activated when the original size can be determined,
|
||||
hence for a file. It won't work with unknown source size, such as stdin or pipe.
|
||||
\fB\-m\fR \fB\-\-multiple\fR
|
||||
Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\.
|
||||
.
|
||||
.TP
|
||||
.B \--[no-]sparse
|
||||
sparse file support (default:enabled)
|
||||
\fB\-r\fR
|
||||
operate recursively on directories\. This mode also sets \fB\-m\fR (multiple input files)\.
|
||||
.
|
||||
.TP
|
||||
.B \-l
|
||||
use Legacy format (useful for Linux Kernel compression)
|
||||
\fB\-B#\fR
|
||||
Block size [4\-7](default : 7)
|
||||
.
|
||||
.br
|
||||
\fB\-B4\fR= 64KB ; \fB\-B5\fR= 256KB ; \fB\-B6\fR= 1MB ; \fB\-B7\fR= 4MB
|
||||
.
|
||||
.TP
|
||||
\fB\-BD\fR
|
||||
Block Dependency (improves compression ratio on small blocks)
|
||||
.
|
||||
.TP
|
||||
\fB\-\-[no\-]frame\-crc\fR
|
||||
Select frame checksum (default:enabled)
|
||||
.
|
||||
.TP
|
||||
\fB\-\-[no\-]content\-size\fR
|
||||
Header includes original size (default:not present)
|
||||
.
|
||||
.br
|
||||
Note : this option can only be activated when the original size can be determined, hence for a file\. It won\'t work with unknown source size, such as stdin or pipe\.
|
||||
.
|
||||
.TP
|
||||
\fB\-\-[no\-]sparse\fR
|
||||
Sparse mode support (default:enabled on file, disabled on stdout)
|
||||
.
|
||||
.TP
|
||||
\fB\-l\fR
|
||||
Use Legacy format (typically for Linux Kernel compression)
|
||||
.
|
||||
.br
|
||||
Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) nor \fB\-r\fR
|
||||
.
|
||||
.SS "Other options"
|
||||
.
|
||||
.TP
|
||||
.BR \-v ", " --verbose
|
||||
verbose mode
|
||||
\fB\-v\fR \fB\-\-verbose\fR
|
||||
Verbose mode
|
||||
.
|
||||
.TP
|
||||
.BR \-q ", " --quiet
|
||||
suppress warnings; specify twice to suppress errors too
|
||||
\fB\-q\fR \fB\-\-quiet\fR
|
||||
Suppress warnings and real\-time statistics; specify twice to suppress errors too
|
||||
.
|
||||
.TP
|
||||
.B \-h/\-H
|
||||
display help/long help and exit
|
||||
\fB\-h\fR \fB\-H\fR \fB\-\-help\fR
|
||||
Display help/long help and exit
|
||||
.
|
||||
.TP
|
||||
.BR \-V ", " \--version
|
||||
display Version number and exit
|
||||
\fB\-V\fR \fB\-\-version\fR
|
||||
Display Version number and exit
|
||||
.
|
||||
.TP
|
||||
.BR \-k ", " \--keep
|
||||
Don't delete source file.
|
||||
This is default behavior anyway, so this option is just for compatibility with gzip/xz.
|
||||
\fB\-k\fR \fB\-\-keep\fR
|
||||
Preserve source files (default behavior)
|
||||
.
|
||||
.TP
|
||||
.B \-b
|
||||
benchmark file(s)
|
||||
\fB\-\-rm\fR
|
||||
Delete source files on successful compression or decompression
|
||||
.
|
||||
.TP
|
||||
.B \-i#
|
||||
iteration loops [1-9](default : 3), benchmark mode only
|
||||
|
||||
.SH BUGS
|
||||
Report bugs at: https://github.com/Cyan4973/lz4
|
||||
|
||||
.SH AUTHOR
|
||||
\fB\-\-\fR
|
||||
Treat all subsequent arguments as files
|
||||
.
|
||||
.SS "Benchmark mode"
|
||||
.
|
||||
.TP
|
||||
\fB\-b#\fR
|
||||
Benchmark file(s), using # compression level
|
||||
.
|
||||
.TP
|
||||
\fB\-e#\fR
|
||||
Benchmark multiple compression levels, from b# to e# (included)
|
||||
.
|
||||
.TP
|
||||
\fB\-i#\fR
|
||||
Minimum evaluation in seconds [1\-9] (default : 3)
|
||||
.
|
||||
.SH "BUGS"
|
||||
Report bugs at: https://github\.com/lz4/lz4/issues
|
||||
.
|
||||
.SH "AUTHOR"
|
||||
Yann Collet
|
||||
|
||||
218
programs/lz4.1.md
Normal file
218
programs/lz4.1.md
Normal file
@@ -0,0 +1,218 @@
|
||||
lz4(1) -- lz4, unlz4, lz4cat - Compress or decompress .lz4 files
|
||||
================================================================
|
||||
|
||||
SYNOPSIS
|
||||
--------
|
||||
|
||||
`lz4` [*OPTIONS*] [-|INPUT-FILE] <OUTPUT-FILE>
|
||||
|
||||
`unlz4` is equivalent to `lz4 -d`
|
||||
|
||||
`lz4cat` is equivalent to `lz4 -dcfm`
|
||||
|
||||
When writing scripts that need to decompress files,
|
||||
it is recommended to always use the name `lz4` with appropriate arguments
|
||||
(`lz4 -d` or `lz4 -dc`) instead of the names `unlz4` and `lz4cat`.
|
||||
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
`lz4` is an extremely fast lossless compression algorithm,
|
||||
based on **byte-aligned LZ77** family of compression scheme.
|
||||
`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with
|
||||
multi-core CPUs.
|
||||
It features an extremely fast decoder, with speed in multiple GB/s per core,
|
||||
typically reaching RAM speed limit on multi-core systems.
|
||||
The native file format is the `.lz4` format.
|
||||
|
||||
### Difference between lz4 and gzip
|
||||
|
||||
`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`.
|
||||
Differences are :
|
||||
|
||||
* `lz4` preserves original files
|
||||
* `lz4` compresses a single file by default (see `-m` for multiple files)
|
||||
* `lz4 file1 file2` means : compress file1 _into_ file2
|
||||
* `lz4 file.lz4` will default to decompression (use `-z` to force compression)
|
||||
* `lz4` shows real-time notification statistics
|
||||
during compression or decompression of a single file
|
||||
(use `-q` to silent them)
|
||||
* If no destination name is provided, result is sent to `stdout`
|
||||
_except if stdout is the console_.
|
||||
* If no destination name is provided, __and__ if `stdout` is the console,
|
||||
`file` is compressed into `file.lz4`.
|
||||
* As a consequence of previous rules, note the following example :
|
||||
`lz4 file | consumer` sends compressed data to `consumer` through `stdout`,
|
||||
hence it does _not_ create `file.lz4`.
|
||||
|
||||
Default behaviors can be modified by opt-in commands, detailed below.
|
||||
|
||||
* `lz4 -m` makes it possible to provide multiple input filenames,
|
||||
which will be compressed into files using suffix `.lz4`.
|
||||
Progress notifications are also disabled by default (use `-v` to enable them).
|
||||
This mode has a behavior which more closely mimics `gzip` command line,
|
||||
with the main remaining difference being that source files are preserved by default.
|
||||
* Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files.
|
||||
* It's possible to opt-in to erase source files
|
||||
on successful compression or decompression, using `--rm` command.
|
||||
* Consequently, `lz4 -m --rm` behaves the same as `gzip`.
|
||||
|
||||
### Concatenation of .lz4 files
|
||||
|
||||
It is possible to concatenate `.lz4` files as is.
|
||||
`lz4` will decompress such files as if they were a single `.lz4` file.
|
||||
For example:
|
||||
|
||||
lz4 file1 > foo.lz4
|
||||
lz4 file2 >> foo.lz4
|
||||
|
||||
Then `lz4cat foo.lz4` is equivalent to `cat file1 file2`.
|
||||
|
||||
OPTIONS
|
||||
-------
|
||||
|
||||
### Short commands concatenation
|
||||
|
||||
In some cases, some options can be expressed using short command `-x`
|
||||
or long command `--long-word`.
|
||||
Short commands can be concatenated together.
|
||||
For example, `-d -c` is equivalent to `-dc`.
|
||||
Long commands cannot be concatenated.
|
||||
They must be clearly separated by a space.
|
||||
|
||||
### Multiple commands
|
||||
|
||||
When multiple contradictory commands are issued on a same command line,
|
||||
only the latest one will be applied.
|
||||
|
||||
### Operation mode
|
||||
|
||||
* `-z` `--compress`:
|
||||
Compress.
|
||||
This is the default operation mode when no operation mode option is
|
||||
specified, no other operation mode is implied from the command name
|
||||
(for example, `unlz4` implies `--decompress`),
|
||||
nor from the input file name
|
||||
(for example, a file extension `.lz4` implies `--decompress` by default).
|
||||
`-z` can also be used to force compression of an already compressed
|
||||
`.lz4` file.
|
||||
|
||||
* `-d` `--decompress` `--uncompress`:
|
||||
Decompress.
|
||||
`--decompress` is also the default operation when the input filename has an
|
||||
`.lz4` extension.
|
||||
|
||||
* `-t` `--test`:
|
||||
Test the integrity of compressed `.lz4` files.
|
||||
The decompressed data is discarded.
|
||||
No files are created nor removed.
|
||||
|
||||
* `-b#`:
|
||||
Benchmark mode, using `#` compression level.
|
||||
|
||||
### Operation modifiers
|
||||
|
||||
* `-#`:
|
||||
Compression level, with # being any value from 1 to 16.
|
||||
Higher values trade compression speed for compression ratio.
|
||||
Values above 16 are considered the same as 16.
|
||||
Recommended values are 1 for fast compression (default),
|
||||
and 9 for high compression.
|
||||
Speed/compression trade-off will vary depending on data to compress.
|
||||
Decompression speed remains fast at all settings.
|
||||
|
||||
* `-f` `--[no-]force`:
|
||||
This option has several effects:
|
||||
|
||||
If the target file already exists, overwrite it without prompting.
|
||||
|
||||
When used with `--decompress` and `lz4` cannot recognize the type of
|
||||
the source file, copy the source file as is to standard output.
|
||||
This allows `lz4cat --force` to be used like `cat (1)` for files
|
||||
that have not been compressed with `lz4`.
|
||||
|
||||
* `-c` `--stdout` `--to-stdout`:
|
||||
Force write to standard output, even if it is the console.
|
||||
|
||||
* `-m` `--multiple`:
|
||||
Multiple input files.
|
||||
Compressed file names will be appended a `.lz4` suffix.
|
||||
This mode also reduces notification level.
|
||||
`lz4 -m` has a behavior equivalent to `gzip -k`
|
||||
(it preserves source files by default).
|
||||
|
||||
* `-r` :
|
||||
operate recursively on directories.
|
||||
This mode also sets `-m` (multiple input files).
|
||||
|
||||
* `-B#`:
|
||||
Block size \[4-7\](default : 7)<br/>
|
||||
`-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB
|
||||
|
||||
* `-BD`:
|
||||
Block Dependency (improves compression ratio on small blocks)
|
||||
|
||||
* `--[no-]frame-crc`:
|
||||
Select frame checksum (default:enabled)
|
||||
|
||||
* `--[no-]content-size`:
|
||||
Header includes original size (default:not present)<br/>
|
||||
Note : this option can only be activated when the original size can be
|
||||
determined, hence for a file. It won't work with unknown source size,
|
||||
such as stdin or pipe.
|
||||
|
||||
* `--[no-]sparse`:
|
||||
Sparse mode support (default:enabled on file, disabled on stdout)
|
||||
|
||||
* `-l`:
|
||||
Use Legacy format (typically for Linux Kernel compression)<br/>
|
||||
Note : `-l` is not compatible with `-m` (`--multiple`) nor `-r`
|
||||
|
||||
### Other options
|
||||
|
||||
* `-v` `--verbose`:
|
||||
Verbose mode
|
||||
|
||||
* `-q` `--quiet`:
|
||||
Suppress warnings and real-time statistics;
|
||||
specify twice to suppress errors too
|
||||
|
||||
* `-h` `-H` `--help`:
|
||||
Display help/long help and exit
|
||||
|
||||
* `-V` `--version`:
|
||||
Display Version number and exit
|
||||
|
||||
* `-k` `--keep`:
|
||||
Preserve source files (default behavior)
|
||||
|
||||
* `--rm` :
|
||||
Delete source files on successful compression or decompression
|
||||
|
||||
* `--` :
|
||||
Treat all subsequent arguments as files
|
||||
|
||||
|
||||
### Benchmark mode
|
||||
|
||||
* `-b#`:
|
||||
Benchmark file(s), using # compression level
|
||||
|
||||
* `-e#`:
|
||||
Benchmark multiple compression levels, from b# to e# (included)
|
||||
|
||||
* `-i#`:
|
||||
Minimum evaluation in seconds \[1-9\] (default : 3)
|
||||
|
||||
|
||||
BUGS
|
||||
----
|
||||
|
||||
Report bugs at: https://github.com/lz4/lz4/issues
|
||||
|
||||
|
||||
AUTHOR
|
||||
------
|
||||
|
||||
Yann Collet
|
||||
@@ -1,6 +1,6 @@
|
||||
/*
|
||||
LZ4cli - LZ4 Command Line Interface
|
||||
Copyright (C) Yann Collet 2011-2015
|
||||
Copyright (C) Yann Collet 2011-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : https://github.com/Cyan4973/lz4
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
/*
|
||||
@@ -30,65 +30,32 @@
|
||||
The license of this compression CLI program is GPLv2.
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Tuning parameters
|
||||
***************************************/
|
||||
/* ENABLE_LZ4C_LEGACY_OPTIONS :
|
||||
Control the availability of -c0, -c1 and -hc legacy arguments
|
||||
Default : Legacy options are disabled */
|
||||
/* #define ENABLE_LZ4C_LEGACY_OPTIONS */
|
||||
|
||||
|
||||
/**************************************
|
||||
* Compiler Options
|
||||
***************************************/
|
||||
/* Disable some Visual warning messages */
|
||||
#ifdef _MSC_VER
|
||||
# define _CRT_SECURE_NO_WARNINGS
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
#define _POSIX_SOURCE 1 /* for fileno() within <stdio.h> on unix */
|
||||
|
||||
|
||||
/****************************
|
||||
* Includes
|
||||
*****************************/
|
||||
#include "platform.h" /* Compiler options, IS_CONSOLE */
|
||||
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
|
||||
#include <stdio.h> /* fprintf, getchar */
|
||||
#include <stdlib.h> /* exit, calloc, free */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include "bench.h" /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */
|
||||
#include "lz4io.h" /* LZ4IO_compressFilename, LZ4IO_decompressFilename, LZ4IO_compressMultipleFilenames */
|
||||
|
||||
|
||||
/****************************
|
||||
* OS-specific Includes
|
||||
*****************************/
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
|
||||
# include <io.h> /* _isatty */
|
||||
# ifdef __MINGW32__
|
||||
int _fileno(FILE *stream); /* MINGW somehow forgets to include this prototype into <stdio.h> */
|
||||
# endif
|
||||
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
||||
#else
|
||||
# include <unistd.h> /* isatty */
|
||||
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
|
||||
#endif
|
||||
#include "lz4hc.h" /* LZ4HC_CLEVEL_MAX */
|
||||
#include "lz4.h" /* LZ4_VERSION_STRING */
|
||||
|
||||
|
||||
/*****************************
|
||||
* Constants
|
||||
******************************/
|
||||
#define COMPRESSOR_NAME "LZ4 command line interface"
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION "r128"
|
||||
#endif
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
|
||||
#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_versionString(), AUTHOR
|
||||
#define LZ4_EXTENSION ".lz4"
|
||||
#define LZ4CAT "lz4cat"
|
||||
#define UNLZ4 "unlz4"
|
||||
#define LZ4_LEGACY "lz4c"
|
||||
static int g_lz4c_legacy_commands = 0;
|
||||
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
@@ -97,7 +64,7 @@
|
||||
#define LZ4_BLOCKSIZEID_DEFAULT 7
|
||||
|
||||
|
||||
/**************************************
|
||||
/*-************************************
|
||||
* Macros
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
@@ -105,13 +72,7 @@
|
||||
static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */
|
||||
|
||||
|
||||
/**************************************
|
||||
* Local Variables
|
||||
***************************************/
|
||||
static char* programName;
|
||||
|
||||
|
||||
/**************************************
|
||||
/*-************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#define DEBUG 0
|
||||
@@ -126,7 +87,7 @@ static char* programName;
|
||||
}
|
||||
|
||||
|
||||
/**************************************
|
||||
/*-************************************
|
||||
* Version modifiers
|
||||
***************************************/
|
||||
#define EXTENDED_ARGUMENTS
|
||||
@@ -137,113 +98,132 @@ static char* programName;
|
||||
int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */
|
||||
|
||||
|
||||
/*****************************
|
||||
/*-***************************
|
||||
* Functions
|
||||
*****************************/
|
||||
static int usage(void)
|
||||
static int usage(const char* exeName)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [arg] [input] [output]\n", programName);
|
||||
DISPLAY( "Usage : \n");
|
||||
DISPLAY( " %s [arg] [input] [output] \n", exeName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "input : a filename\n");
|
||||
DISPLAY( "input : a filename \n");
|
||||
DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark);
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( "Arguments : \n");
|
||||
DISPLAY( " -1 : Fast compression (default) \n");
|
||||
DISPLAY( " -9 : High compression \n");
|
||||
DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION);
|
||||
DISPLAY( " -z : force compression\n");
|
||||
DISPLAY( " -z : force compression \n");
|
||||
DISPLAY( " -f : overwrite output without prompting \n");
|
||||
DISPLAY( " -h/-H : display help/long help and exit\n");
|
||||
DISPLAY( " -k : preserve source files(s) (default) \n");
|
||||
DISPLAY( "--rm : remove source file(s) after successful de/compression \n");
|
||||
DISPLAY( " -h/-H : display help/long help and exit \n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int usage_advanced(void)
|
||||
static int usage_advanced(const char* exeName)
|
||||
{
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
usage();
|
||||
usage(exeName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Advanced arguments :\n");
|
||||
DISPLAY( " -V : display Version number and exit\n");
|
||||
DISPLAY( " -v : verbose mode\n");
|
||||
DISPLAY( " -V : display Version number and exit \n");
|
||||
DISPLAY( " -v : verbose mode \n");
|
||||
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
|
||||
DISPLAY( " -c : force write to standard output, even if it is the console\n");
|
||||
DISPLAY( " -t : test compressed file integrity\n");
|
||||
DISPLAY( " -m : multiple input files (implies automatic output filenames)\n");
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
DISPLAY( " -r : operate recursively on directories (sets also -m) \n");
|
||||
#endif
|
||||
DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n");
|
||||
DISPLAY( " -B# : Block size [4-7](default : 7)\n");
|
||||
DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
|
||||
/* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */
|
||||
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n");
|
||||
DISPLAY( " -B# : Block size [4-7] (default : 7) \n");
|
||||
DISPLAY( " -BD : Block dependency (improve compression ratio) \n");
|
||||
DISPLAY( " -BX : enable block checksum (default:disabled) \n");
|
||||
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n");
|
||||
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
|
||||
DISPLAY( "--[no-]sparse : sparse file support (default:enabled)\n");
|
||||
DISPLAY( "Benchmark arguments :\n");
|
||||
DISPLAY( " -b : benchmark file(s)\n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");
|
||||
#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
|
||||
DISPLAY( "Legacy arguments :\n");
|
||||
DISPLAY( " -c0 : fast compression\n");
|
||||
DISPLAY( " -c1 : high compression\n");
|
||||
DISPLAY( " -hc : high compression\n");
|
||||
DISPLAY( " -y : overwrite output without prompting \n");
|
||||
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
|
||||
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
|
||||
DISPLAY( "Benchmark arguments : \n");
|
||||
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
|
||||
DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n");
|
||||
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n");
|
||||
DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n");
|
||||
DISPLAY( " or predefined block size [4-7] (default: 7) \n");
|
||||
if (g_lz4c_legacy_commands) {
|
||||
DISPLAY( "Legacy arguments : \n");
|
||||
DISPLAY( " -c0 : fast compression \n");
|
||||
DISPLAY( " -c1 : high compression \n");
|
||||
DISPLAY( " -hc : high compression \n");
|
||||
DISPLAY( " -y : overwrite output without prompting \n");
|
||||
}
|
||||
EXTENDED_HELP;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int usage_longhelp(void)
|
||||
static int usage_longhelp(const char* exeName)
|
||||
{
|
||||
usage_advanced(exeName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Which values can get [output] ? \n");
|
||||
DISPLAY( "[output] : a filename\n");
|
||||
DISPLAY( "****************************\n");
|
||||
DISPLAY( "***** Advanced comment *****\n");
|
||||
DISPLAY( "****************************\n");
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Which values can [output] have ? \n");
|
||||
DISPLAY( "---------------------------------\n");
|
||||
DISPLAY( "[output] : a filename \n");
|
||||
DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark);
|
||||
DISPLAY( " '%s' to discard output (test mode)\n", NULL_OUTPUT);
|
||||
DISPLAY( "[output] can be left empty. In this case, it receives the following value : \n");
|
||||
DISPLAY( " '%s' to discard output (test mode) \n", NULL_OUTPUT);
|
||||
DISPLAY( "[output] can be left empty. In this case, it receives the following value :\n");
|
||||
DISPLAY( " - if stdout is not the console, then [output] = stdout \n");
|
||||
DISPLAY( " - if stdout is console : \n");
|
||||
DISPLAY( " + if compression selected, output to filename%s \n", LZ4_EXTENSION);
|
||||
DISPLAY( " + if decompression selected, output to filename without '%s'\n", LZ4_EXTENSION);
|
||||
DISPLAY( " > if input filename has no '%s' extension : error\n", LZ4_EXTENSION);
|
||||
DISPLAY( " + for compression, output to filename%s \n", LZ4_EXTENSION);
|
||||
DISPLAY( " + for decompression, output to filename without '%s'\n", LZ4_EXTENSION);
|
||||
DISPLAY( " > if input filename has no '%s' extension : error \n", LZ4_EXTENSION);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Compression levels : \n");
|
||||
DISPLAY( "There are technically 2 accessible compression levels.\n");
|
||||
DISPLAY( "-0 ... -2 => Fast compression\n");
|
||||
DISPLAY( "-3 ... -9 => High compression\n");
|
||||
DISPLAY( "---------------------\n");
|
||||
DISPLAY( "-0 ... -2 => Fast compression, all identicals\n");
|
||||
DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "stdin, stdout and the console : \n");
|
||||
DISPLAY( "--------------------------------\n");
|
||||
DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n");
|
||||
DISPLAY( "%s will refuse to read from console, or write to console \n", programName);
|
||||
DISPLAY( "%s will refuse to read from console, or write to console \n", exeName);
|
||||
DISPLAY( "except if '-c' command is specified, to force output to console \n");
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Simple example :\n");
|
||||
DISPLAY( "----------------\n");
|
||||
DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n");
|
||||
DISPLAY( " %s filename\n", programName);
|
||||
DISPLAY( " %s filename\n", exeName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments can be appended together, or provided independently. For example :\n");
|
||||
DISPLAY( "Short arguments can be aggregated. For example :\n");
|
||||
DISPLAY( "----------------------------------\n");
|
||||
DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n");
|
||||
DISPLAY( " %s -f9 filename \n", programName);
|
||||
DISPLAY( " %s -9 -f filename \n", exeName);
|
||||
DISPLAY( " is equivalent to :\n");
|
||||
DISPLAY( " %s -f -9 filename \n", programName);
|
||||
DISPLAY( " %s -9f filename \n", exeName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName);
|
||||
DISPLAY( "%s can be used in 'pure pipe mode'. For example :\n", exeName);
|
||||
DISPLAY( "-------------------------------------\n");
|
||||
DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n");
|
||||
DISPLAY( " generator | %s | consumer \n", programName);
|
||||
#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Warning :\n");
|
||||
DISPLAY( "Legacy arguments take precedence. Therefore : \n");
|
||||
DISPLAY( " %s -hc filename\n", programName);
|
||||
DISPLAY( "means 'compress filename in high compression mode'\n");
|
||||
DISPLAY( "It is not equivalent to :\n");
|
||||
DISPLAY( " %s -h -c filename\n", programName);
|
||||
DISPLAY( "which would display help text and exit\n");
|
||||
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
|
||||
DISPLAY( " generator | %s | consumer \n", exeName);
|
||||
if (g_lz4c_legacy_commands) {
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "***** Warning *****\n");
|
||||
DISPLAY( "Legacy arguments take precedence. Therefore : \n");
|
||||
DISPLAY( "---------------------------------\n");
|
||||
DISPLAY( " %s -hc filename\n", exeName);
|
||||
DISPLAY( "means 'compress filename in high compression mode'\n");
|
||||
DISPLAY( "It is not equivalent to :\n");
|
||||
DISPLAY( " %s -h -c filename\n", exeName);
|
||||
DISPLAY( "which would display help text and exit\n");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int badusage(void)
|
||||
static int badusage(const char* exeName)
|
||||
{
|
||||
DISPLAYLEVEL(1, "Incorrect parameters\n");
|
||||
if (displayLevel >= 1) usage();
|
||||
if (displayLevel >= 1) usage(exeName);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
@@ -254,195 +234,262 @@ static void waitEnter(void)
|
||||
(void)getchar();
|
||||
}
|
||||
|
||||
static const char* lastNameFromPath(const char* path)
|
||||
{
|
||||
const char* name = path;
|
||||
if (strrchr(name, '/')) name = strrchr(name, '/') + 1;
|
||||
if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */
|
||||
return name;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
/*! exeNameMatch() :
|
||||
@return : a non-zero value if exeName matches test, excluding the extension
|
||||
*/
|
||||
static int exeNameMatch(const char* exeName, const char* test)
|
||||
{
|
||||
return !strncmp(exeName, test, strlen(test)) &&
|
||||
(exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.');
|
||||
}
|
||||
|
||||
/*! readU32FromChar() :
|
||||
@return : unsigned integer value read from input in `char` format
|
||||
allows and interprets K, KB, KiB, M, MB and MiB suffix.
|
||||
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
|
||||
Note : function result can overflow if digit string > MAX_UINT */
|
||||
static unsigned readU32FromChar(const char** stringPtr)
|
||||
{
|
||||
unsigned result = 0;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9'))
|
||||
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||
if ((**stringPtr=='K') || (**stringPtr=='M')) {
|
||||
result <<= 10;
|
||||
if (**stringPtr=='M') result <<= 10;
|
||||
(*stringPtr)++ ;
|
||||
if (**stringPtr=='i') (*stringPtr)++;
|
||||
if (**stringPtr=='B') (*stringPtr)++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e;
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
int i,
|
||||
cLevel=0,
|
||||
decode=0,
|
||||
bench=0,
|
||||
cLevel=1,
|
||||
cLevelLast=1,
|
||||
legacy_format=0,
|
||||
forceStdout=0,
|
||||
forceCompress=0,
|
||||
main_pause=0,
|
||||
multiple_inputs=0,
|
||||
all_arguments_are_files=0,
|
||||
operationResult=0;
|
||||
const char* input_filename=0;
|
||||
const char* output_filename=0;
|
||||
char* dynNameSpace=0;
|
||||
const char** inFileNames = NULL;
|
||||
operationMode_e mode = om_auto;
|
||||
const char* input_filename = NULL;
|
||||
const char* output_filename= NULL;
|
||||
char* dynNameSpace = NULL;
|
||||
const char** inFileNames = (const char**) calloc(argc, sizeof(char*));
|
||||
unsigned ifnIdx=0;
|
||||
char nullOutput[] = NULL_OUTPUT;
|
||||
char extension[] = LZ4_EXTENSION;
|
||||
int blockSize;
|
||||
const char nullOutput[] = NULL_OUTPUT;
|
||||
const char extension[] = LZ4_EXTENSION;
|
||||
size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
|
||||
const char* const exeName = lastNameFromPath(argv[0]);
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
const char** extendedFileList = NULL;
|
||||
char* fileNamesBuf = NULL;
|
||||
unsigned fileNamesNb, recursive=0;
|
||||
#endif
|
||||
|
||||
/* Init */
|
||||
programName = argv[0];
|
||||
if (inFileNames==NULL) {
|
||||
DISPLAY("Allocation error : not enough memory \n");
|
||||
return 1;
|
||||
}
|
||||
inFileNames[0] = stdinmark;
|
||||
LZ4IO_setOverwrite(0);
|
||||
blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
|
||||
|
||||
/* lz4cat predefined behavior */
|
||||
if (!strcmp(programName, LZ4CAT)) { decode=1; forceStdout=1; output_filename=stdoutmark; displayLevel=1; }
|
||||
if (!strcmp(programName, UNLZ4)) { decode=1; }
|
||||
/* predefined behaviors, based on binary/link name */
|
||||
if (exeNameMatch(exeName, LZ4CAT)) {
|
||||
mode = om_decompress;
|
||||
LZ4IO_setOverwrite(1);
|
||||
LZ4IO_setRemoveSrcFile(0);
|
||||
forceStdout=1;
|
||||
output_filename=stdoutmark;
|
||||
displayLevel=1;
|
||||
multiple_inputs=1;
|
||||
}
|
||||
if (exeNameMatch(exeName, UNLZ4)) { mode = om_decompress; }
|
||||
if (exeNameMatch(exeName, LZ4_LEGACY)) { g_lz4c_legacy_commands=1; }
|
||||
|
||||
/* command switches */
|
||||
for(i=1; i<argc; i++)
|
||||
{
|
||||
char* argument = argv[i];
|
||||
for(i=1; i<argc; i++) {
|
||||
const char* argument = argv[i];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* long commands (--long-word) */
|
||||
if (!strcmp(argument, "--compress")) { forceCompress = 1; continue; }
|
||||
if ((!strcmp(argument, "--decompress"))
|
||||
|| (!strcmp(argument, "--uncompress"))) { decode = 1; continue; }
|
||||
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; if (inFileNames==NULL) inFileNames = (const char**)malloc(argc * sizeof(char*)); continue; }
|
||||
if (!strcmp(argument, "--test")) { decode = 1; LZ4IO_setOverwrite(1); output_filename=nulmark; continue; }
|
||||
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
|
||||
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; }
|
||||
if ((!strcmp(argument, "--stdout"))
|
||||
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; }
|
||||
if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; }
|
||||
if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
|
||||
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
|
||||
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
|
||||
if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; }
|
||||
if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
|
||||
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
|
||||
if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; }
|
||||
if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
|
||||
if (!strcmp(argument, "--keep")) { continue; } /* keep source file (default anyway; just for xz/lzma compatibility) */
|
||||
|
||||
|
||||
/* Short commands (note : aggregated short commands are allowed) */
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
if (!all_arguments_are_files && argument[0]=='-') {
|
||||
/* '-' means stdin/stdout */
|
||||
if (argument[1]==0)
|
||||
{
|
||||
if (argument[1]==0) {
|
||||
if (!input_filename) input_filename=stdinmark;
|
||||
else output_filename=stdoutmark;
|
||||
continue;
|
||||
}
|
||||
|
||||
while (argument[1]!=0)
|
||||
{
|
||||
/* long commands (--long-word) */
|
||||
if (argument[1]=='-') {
|
||||
if (!strcmp(argument, "--")) { all_arguments_are_files = 1; continue; }
|
||||
if (!strcmp(argument, "--compress")) { mode = om_compress; continue; }
|
||||
if ((!strcmp(argument, "--decompress"))
|
||||
|| (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
|
||||
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; }
|
||||
if (!strcmp(argument, "--test")) { mode = om_test; continue; }
|
||||
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
|
||||
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; }
|
||||
if ((!strcmp(argument, "--stdout"))
|
||||
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; }
|
||||
if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; }
|
||||
if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
|
||||
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
|
||||
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
|
||||
if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; }
|
||||
if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
|
||||
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
|
||||
if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; }
|
||||
if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
|
||||
if (!strcmp(argument, "--help")) { usage_advanced(exeName); goto _cleanup; }
|
||||
if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; } /* keep source file (default) */
|
||||
if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(1); continue; }
|
||||
}
|
||||
|
||||
while (argument[1]!=0) {
|
||||
argument ++;
|
||||
|
||||
#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
|
||||
/* Legacy arguments (-c0, -c1, -hc, -y, -s) */
|
||||
if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */
|
||||
if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } /* -c1 (high compression) */
|
||||
if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } /* -hc (high compression) */
|
||||
if (*argument=='y') { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */
|
||||
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
|
||||
if (g_lz4c_legacy_commands) {
|
||||
/* Legacy commands (-c0, -c1, -hc, -y) */
|
||||
if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */
|
||||
if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } /* -c1 (high compression) */
|
||||
if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } /* -hc (high compression) */
|
||||
if (argument[0]=='y') { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */
|
||||
}
|
||||
|
||||
if ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
cLevel = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9'))
|
||||
{
|
||||
cLevel *= 10;
|
||||
cLevel += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
cLevel = readU32FromChar(&argument);
|
||||
argument--;
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
switch(argument[0])
|
||||
{
|
||||
/* Display help */
|
||||
case 'V': DISPLAY(WELCOME_MESSAGE); return 0; /* Version */
|
||||
case 'h': usage_advanced(); return 0;
|
||||
case 'H': usage_advanced(); usage_longhelp(); return 0;
|
||||
case 'V': DISPLAY(WELCOME_MESSAGE); goto _cleanup; /* Version */
|
||||
case 'h': usage_advanced(exeName); goto _cleanup;
|
||||
case 'H': usage_longhelp(exeName); goto _cleanup;
|
||||
|
||||
case 'e':
|
||||
argument++;
|
||||
cLevelLast = readU32FromChar(&argument);
|
||||
argument--;
|
||||
break;
|
||||
|
||||
/* Compression (default) */
|
||||
case 'z': forceCompress = 1; break;
|
||||
case 'z': mode = om_compress; break;
|
||||
|
||||
/* Use Legacy format (ex : Linux kernel compression) */
|
||||
case 'l': legacy_format = 1; blockSize = 8 MB; break;
|
||||
|
||||
/* Decoding */
|
||||
case 'd': decode=1; break;
|
||||
case 'd': mode = om_decompress; break;
|
||||
|
||||
/* Force stdout, even if stdout==console */
|
||||
case 'c': forceStdout=1; output_filename=stdoutmark; displayLevel=1; break;
|
||||
case 'c': forceStdout=1; output_filename=stdoutmark; break;
|
||||
|
||||
/* Test integrity */
|
||||
case 't': decode=1; LZ4IO_setOverwrite(1); output_filename=nulmark; break;
|
||||
case 't': mode = om_test; break;
|
||||
|
||||
/* Overwrite */
|
||||
case 'f': LZ4IO_setOverwrite(1); break;
|
||||
|
||||
/* Verbose mode */
|
||||
case 'v': displayLevel=4; break;
|
||||
case 'v': displayLevel++; break;
|
||||
|
||||
/* Quiet mode */
|
||||
case 'q': if (displayLevel) displayLevel--; break;
|
||||
|
||||
/* keep source file (default anyway, so useless) (for xz/lzma compatibility) */
|
||||
case 'k': break;
|
||||
case 'k': LZ4IO_setRemoveSrcFile(0); break;
|
||||
|
||||
/* Modify Block Properties */
|
||||
case 'B':
|
||||
while (argument[1]!=0)
|
||||
{
|
||||
while (argument[1]!=0) {
|
||||
int exitBlockProperties=0;
|
||||
switch(argument[1])
|
||||
{
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
{
|
||||
int B = argument[1] - '0';
|
||||
blockSize = LZ4IO_setBlockSizeID(B);
|
||||
BMK_setBlocksize(blockSize);
|
||||
argument++;
|
||||
break;
|
||||
}
|
||||
case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break;
|
||||
case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* currently disabled */
|
||||
default : exitBlockProperties=1;
|
||||
case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* disabled by default */
|
||||
default :
|
||||
if (argument[1] < '0' || argument[1] > '9') {
|
||||
exitBlockProperties=1;
|
||||
break;
|
||||
} else {
|
||||
unsigned B;
|
||||
argument++;
|
||||
B = readU32FromChar(&argument);
|
||||
argument--;
|
||||
if (B < 4) badusage(exeName);
|
||||
if (B <= 7) {
|
||||
blockSize = LZ4IO_setBlockSizeID(B);
|
||||
BMK_SetBlockSize(blockSize);
|
||||
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
|
||||
} else {
|
||||
if (B < 32) badusage(exeName);
|
||||
BMK_SetBlockSize(B);
|
||||
if (B >= 1024) {
|
||||
DISPLAYLEVEL(2, "bench: using blocks of size %u KB \n", (U32)(B>>10));
|
||||
} else {
|
||||
DISPLAYLEVEL(2, "bench: using blocks of size %u bytes \n", (U32)(B));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (exitBlockProperties) break;
|
||||
}
|
||||
break;
|
||||
|
||||
/* Benchmark */
|
||||
case 'b': bench=1; multiple_inputs=1;
|
||||
if (inFileNames == NULL)
|
||||
inFileNames = (const char**) malloc(argc * sizeof(char*));
|
||||
case 'b': mode = om_bench; multiple_inputs=1;
|
||||
break;
|
||||
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
/* recursive */
|
||||
case 'r': recursive=1;
|
||||
#endif
|
||||
/* fall-through */
|
||||
/* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */
|
||||
case 'm': multiple_inputs=1;
|
||||
if (inFileNames == NULL)
|
||||
inFileNames = (const char**) malloc(argc * sizeof(char*));
|
||||
break;
|
||||
|
||||
/* Modify Nb Iterations (benchmark only) */
|
||||
/* Modify Nb Seconds (benchmark only) */
|
||||
case 'i':
|
||||
{
|
||||
unsigned iters = 0;
|
||||
while ((argument[1] >='0') && (argument[1] <='9'))
|
||||
{
|
||||
iters *= 10;
|
||||
iters += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
BMK_setNbIterations(iters);
|
||||
{ unsigned iters;
|
||||
argument++;
|
||||
iters = readU32FromChar(&argument);
|
||||
argument--;
|
||||
BMK_setNotificationLevel(displayLevel);
|
||||
BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */
|
||||
}
|
||||
break;
|
||||
|
||||
/* Pause at the end (hidden option) */
|
||||
case 'p': main_pause=1; BMK_setPause(); break;
|
||||
case 'p': main_pause=1; break;
|
||||
|
||||
/* Specific commands for customized versions */
|
||||
EXTENDED_ARGUMENTS;
|
||||
|
||||
/* Unrecognised command */
|
||||
default : badusage();
|
||||
default : badusage(exeName);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
@@ -455,8 +502,7 @@ int main(int argc, char** argv)
|
||||
if (!input_filename) { input_filename=argument; continue; }
|
||||
|
||||
/* Second non-option arg in output_filename to preserve original cli logic. */
|
||||
if (!output_filename)
|
||||
{
|
||||
if (!output_filename) {
|
||||
output_filename=argument;
|
||||
if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;
|
||||
continue;
|
||||
@@ -467,94 +513,135 @@ int main(int argc, char** argv)
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, WELCOME_MESSAGE);
|
||||
if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
|
||||
#ifdef _POSIX_C_SOURCE
|
||||
DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
|
||||
#endif
|
||||
#ifdef _POSIX_VERSION
|
||||
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION);
|
||||
#endif
|
||||
#ifdef PLATFORM_POSIX_VERSION
|
||||
DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
|
||||
#endif
|
||||
#ifdef _FILE_OFFSET_BITS
|
||||
DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS);
|
||||
#endif
|
||||
if ((mode == om_compress) || (mode == om_bench))
|
||||
DISPLAYLEVEL(4, "Blocks size : %u KB\n", (U32)(blockSize>>10));
|
||||
|
||||
/* No input filename ==> use stdin */
|
||||
if (multiple_inputs) input_filename = inFileNames[0], output_filename = (const char*)(inFileNames[0]);
|
||||
if(!input_filename) { input_filename=stdinmark; }
|
||||
|
||||
/* Check if input is defined as console; trigger an error in this case */
|
||||
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
|
||||
|
||||
/* Check if benchmark is selected */
|
||||
if (bench)
|
||||
{
|
||||
int bmkResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel);
|
||||
free((void*)inFileNames);
|
||||
return bmkResult;
|
||||
if (multiple_inputs) {
|
||||
input_filename = inFileNames[0];
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
|
||||
extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb);
|
||||
if (extendedFileList) {
|
||||
unsigned u;
|
||||
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
|
||||
free((void*)inFileNames);
|
||||
inFileNames = extendedFileList;
|
||||
ifnIdx = fileNamesNb;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/* benchmark and test modes */
|
||||
if (mode == om_bench) {
|
||||
BMK_setNotificationLevel(displayLevel);
|
||||
operationResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel, cLevelLast);
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
if (mode == om_test) {
|
||||
LZ4IO_setTestMode(1);
|
||||
output_filename = nulmark;
|
||||
mode = om_decompress; /* defer to decompress */
|
||||
}
|
||||
|
||||
/* compress or decompress */
|
||||
if (!input_filename) input_filename = stdinmark;
|
||||
/* Check if input is defined as console; trigger an error in this case */
|
||||
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) {
|
||||
DISPLAYLEVEL(1, "refusing to read from a console\n");
|
||||
exit(1);
|
||||
}
|
||||
/* if input==stdin and no output defined, stdout becomes default output */
|
||||
if (!strcmp(input_filename, stdinmark) && !output_filename)
|
||||
output_filename = stdoutmark;
|
||||
|
||||
/* No output filename ==> try to select one automatically (when possible) */
|
||||
while (!output_filename)
|
||||
{
|
||||
while ((!output_filename) && (multiple_inputs==0)) {
|
||||
if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */
|
||||
if ((!decode) && !(forceCompress)) /* auto-determine compression or decompression, based on file extension */
|
||||
{
|
||||
size_t l = strlen(input_filename);
|
||||
if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1;
|
||||
if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */
|
||||
size_t const inSize = strlen(input_filename);
|
||||
size_t const extSize = strlen(LZ4_EXTENSION);
|
||||
size_t const extStart= (inSize > extSize) ? inSize-extSize : 0;
|
||||
if (!strcmp(input_filename+extStart, LZ4_EXTENSION)) mode = om_decompress;
|
||||
else mode = om_compress;
|
||||
}
|
||||
if (!decode) /* compression to file */
|
||||
{
|
||||
size_t l = strlen(input_filename);
|
||||
if (mode == om_compress) { /* compression to file */
|
||||
size_t const l = strlen(input_filename);
|
||||
dynNameSpace = (char*)calloc(1,l+5);
|
||||
if (dynNameSpace==NULL) { perror(exeName); exit(1); }
|
||||
strcpy(dynNameSpace, input_filename);
|
||||
strcat(dynNameSpace, LZ4_EXTENSION);
|
||||
output_filename = dynNameSpace;
|
||||
DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
|
||||
break;
|
||||
}
|
||||
/* decompression to file (automatic name will work only if input filename has correct format extension) */
|
||||
{
|
||||
if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */
|
||||
size_t outl;
|
||||
size_t inl = strlen(input_filename);
|
||||
size_t const inl = strlen(input_filename);
|
||||
dynNameSpace = (char*)calloc(1,inl+1);
|
||||
if (dynNameSpace==NULL) { perror(exeName); exit(1); }
|
||||
strcpy(dynNameSpace, input_filename);
|
||||
outl = inl;
|
||||
if (inl>4)
|
||||
while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0;
|
||||
if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); }
|
||||
if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); }
|
||||
output_filename = dynNameSpace;
|
||||
DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
/* Check if output is defined as console; trigger an error in this case */
|
||||
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage();
|
||||
|
||||
/* Downgrade notification level in pure pipe mode (stdin + stdout) and multiple file mode */
|
||||
if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
|
||||
if (!output_filename) output_filename = "*\\dummy^!//";
|
||||
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) {
|
||||
DISPLAYLEVEL(1, "refusing to write to console without -c\n");
|
||||
exit(1);
|
||||
}
|
||||
/* Downgrade notification level in stdout and multiple file mode */
|
||||
if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
|
||||
if ((multiple_inputs) && (displayLevel==2)) displayLevel=1;
|
||||
|
||||
|
||||
/* IO Stream/File */
|
||||
LZ4IO_setNotificationLevel(displayLevel);
|
||||
if (decode)
|
||||
{
|
||||
if (multiple_inputs)
|
||||
operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION);
|
||||
else
|
||||
DEFAULT_DECOMPRESSOR(input_filename, output_filename);
|
||||
}
|
||||
else
|
||||
{
|
||||
/* compression is default action */
|
||||
if (legacy_format)
|
||||
{
|
||||
DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
|
||||
LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (ifnIdx == 0) multiple_inputs = 0;
|
||||
if (mode == om_decompress) {
|
||||
if (multiple_inputs)
|
||||
operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel);
|
||||
operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION);
|
||||
else
|
||||
DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
|
||||
}
|
||||
operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename);
|
||||
} else { /* compression is default action */
|
||||
if (legacy_format) {
|
||||
DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
|
||||
LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
|
||||
} else {
|
||||
if (multiple_inputs)
|
||||
operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel);
|
||||
else
|
||||
operationResult = DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
|
||||
}
|
||||
}
|
||||
|
||||
_cleanup:
|
||||
if (main_pause) waitEnter();
|
||||
free(dynNameSpace);
|
||||
free((void*)inFileNames);
|
||||
if (operationResult != 0) return operationResult;
|
||||
return 0;
|
||||
if (dynNameSpace) free(dynNameSpace);
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
if (extendedFileList)
|
||||
UTIL_freeFileList(extendedFileList, fileNamesBuf);
|
||||
else
|
||||
#endif
|
||||
free((void*)inFileNames);
|
||||
return operationResult;
|
||||
}
|
||||
|
||||
727
programs/lz4io.c
727
programs/lz4io.c
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
/*
|
||||
LZ4io.h - LZ4 File/Stream Interface
|
||||
Copyright (C) Yann Collet 2011-2015
|
||||
Copyright (C) Yann Collet 2011-2016
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
@@ -18,7 +18,7 @@
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : https://github.com/Cyan4973/lz4
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
/*
|
||||
@@ -29,18 +29,23 @@
|
||||
- The license of this source file is GPLv2.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#ifndef LZ4IO_H_237902873
|
||||
#define LZ4IO_H_237902873
|
||||
|
||||
/*--- Dependency ---*/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* ************************************************** */
|
||||
/* Special input/output values */
|
||||
/* ************************************************** */
|
||||
#define NULL_OUTPUT "null"
|
||||
static char const stdinmark[] = "stdin";
|
||||
static char const stdoutmark[] = "stdout";
|
||||
static const char stdinmark[] = "stdin";
|
||||
static const char stdoutmark[] = "stdout";
|
||||
#ifdef _WIN32
|
||||
static char const nulmark[] = "nul";
|
||||
static const char nulmark[] = "nul";
|
||||
#else
|
||||
static char const nulmark[] = "/dev/null";
|
||||
static const char nulmark[] = "/dev/null";
|
||||
#endif
|
||||
|
||||
|
||||
@@ -51,10 +56,10 @@ static char const nulmark[] = "/dev/null";
|
||||
int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel);
|
||||
int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename);
|
||||
|
||||
|
||||
int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel);
|
||||
int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix);
|
||||
|
||||
|
||||
/* ************************************************** */
|
||||
/* ****************** Parameters ******************** */
|
||||
/* ************************************************** */
|
||||
@@ -63,9 +68,13 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz
|
||||
return : overwrite mode (0/1) */
|
||||
int LZ4IO_setOverwrite(int yes);
|
||||
|
||||
/* Default setting : testMode = 0;
|
||||
return : testMode (0/1) */
|
||||
int LZ4IO_setTestMode(int yes);
|
||||
|
||||
/* blockSizeID : valid values : 4-5-6-7
|
||||
return : -1 if error, blockSize if OK */
|
||||
int LZ4IO_setBlockSizeID(int blockSizeID);
|
||||
return : 0 if error, blockSize if OK */
|
||||
size_t LZ4IO_setBlockSizeID(unsigned blockSizeID);
|
||||
|
||||
/* Default setting : independent blocks */
|
||||
typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t;
|
||||
@@ -85,3 +94,8 @@ int LZ4IO_setSparseFile(int enable);
|
||||
|
||||
/* Default setting : 0 (disabled) */
|
||||
int LZ4IO_setContentSize(int enable);
|
||||
|
||||
void LZ4IO_setRemoveSrcFile(unsigned flag);
|
||||
|
||||
|
||||
#endif /* LZ4IO_H_237902873 */
|
||||
|
||||
154
programs/platform.h
Normal file
154
programs/platform.h
Normal file
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
platform.h - compiler and OS detection
|
||||
Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*/
|
||||
|
||||
#ifndef PLATFORM_H_MODULE
|
||||
#define PLATFORM_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Compiler Options
|
||||
****************************************/
|
||||
#if defined(_MSC_VER)
|
||||
# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
|
||||
# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */
|
||||
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Detect 64-bit OS
|
||||
* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
|
||||
****************************************/
|
||||
#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \
|
||||
|| defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \
|
||||
|| (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \
|
||||
|| defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \
|
||||
|| defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \
|
||||
|| (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \
|
||||
|| defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \
|
||||
|| (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */
|
||||
# if !defined(__64BIT__)
|
||||
# define __64BIT__ 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* *********************************************************
|
||||
* Turn on Large Files support (>4GB) for 32-bit Linux/Unix
|
||||
***********************************************************/
|
||||
#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */
|
||||
# if !defined(_FILE_OFFSET_BITS)
|
||||
# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
|
||||
# endif
|
||||
# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */
|
||||
# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */
|
||||
# endif
|
||||
# if defined(_AIX) || defined(__hpux)
|
||||
# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* ************************************************************
|
||||
* Detect POSIX version
|
||||
* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows
|
||||
* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX
|
||||
* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION
|
||||
***************************************************************/
|
||||
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
|
||||
|| defined(__midipix__) || defined(__VMS))
|
||||
# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
|
||||
|| defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */
|
||||
# define PLATFORM_POSIX_VERSION 200112L
|
||||
# else
|
||||
# if defined(__linux__) || defined(__linux)
|
||||
# ifndef _POSIX_C_SOURCE
|
||||
# define _POSIX_C_SOURCE 200112L /* use feature test macro */
|
||||
# endif
|
||||
# endif
|
||||
# include <unistd.h> /* declares _POSIX_VERSION */
|
||||
# if defined(_POSIX_VERSION) /* POSIX compliant */
|
||||
# define PLATFORM_POSIX_VERSION _POSIX_VERSION
|
||||
# else
|
||||
# define PLATFORM_POSIX_VERSION 0
|
||||
# endif
|
||||
# endif
|
||||
#endif
|
||||
#if !defined(PLATFORM_POSIX_VERSION)
|
||||
# define PLATFORM_POSIX_VERSION -1
|
||||
#endif
|
||||
|
||||
|
||||
/*-*********************************************
|
||||
* Detect if isatty() and fileno() are available
|
||||
************************************************/
|
||||
#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__)
|
||||
# include <unistd.h> /* isatty */
|
||||
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
|
||||
#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__)
|
||||
# include <io.h> /* _isatty */
|
||||
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
||||
#elif defined(WIN32) || defined(_WIN32)
|
||||
# include <io.h> /* _isatty */
|
||||
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
|
||||
# include <stdio.h> /* FILE */
|
||||
static __inline int IS_CONSOLE(FILE* stdStream)
|
||||
{
|
||||
DWORD dummy;
|
||||
return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
|
||||
}
|
||||
#else
|
||||
# define IS_CONSOLE(stdStream) 0
|
||||
#endif
|
||||
|
||||
|
||||
/******************************
|
||||
* OS-specific Includes
|
||||
******************************/
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
# include <io.h> /* _setmode, _fileno, _get_osfhandle */
|
||||
# if !defined(__DJGPP__)
|
||||
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
|
||||
# include <winioctl.h> /* FSCTL_SET_SPARSE */
|
||||
# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
|
||||
# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
|
||||
# else
|
||||
# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
|
||||
# define SET_SPARSE_FILE_MODE(file)
|
||||
# endif
|
||||
#else
|
||||
# define SET_BINARY_MODE(file)
|
||||
# define SET_SPARSE_FILE_MODE(file)
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* PLATFORM_H_MODULE */
|
||||
494
programs/util.h
Normal file
494
programs/util.h
Normal file
@@ -0,0 +1,494 @@
|
||||
/*
|
||||
util.h - utility functions
|
||||
Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*/
|
||||
|
||||
#ifndef UTIL_H_MODULE
|
||||
#define UTIL_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include "platform.h" /* PLATFORM_POSIX_VERSION */
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <sys/types.h> /* stat, utime */
|
||||
#include <sys/stat.h> /* stat */
|
||||
#if defined(_MSC_VER)
|
||||
# include <sys/utime.h> /* utime */
|
||||
# include <io.h> /* _chmod */
|
||||
#else
|
||||
# include <unistd.h> /* chown, stat */
|
||||
# include <utime.h> /* utime */
|
||||
#endif
|
||||
#include <time.h> /* time */
|
||||
#include <errno.h>
|
||||
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Basic Types
|
||||
*****************************************************************/
|
||||
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef int16_t S16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
typedef int64_t S64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef signed short S16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
typedef signed long long S64;
|
||||
#endif
|
||||
|
||||
|
||||
/* ************************************************************
|
||||
* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
|
||||
***************************************************************/
|
||||
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
|
||||
# define UTIL_fseek _fseeki64
|
||||
#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
|
||||
# define UTIL_fseek fseeko
|
||||
#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS)
|
||||
# define UTIL_fseek fseeko64
|
||||
#else
|
||||
# define UTIL_fseek fseek
|
||||
#endif
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Sleep functions: Windows - Posix - others
|
||||
******************************************/
|
||||
#if defined(_WIN32)
|
||||
# include <windows.h>
|
||||
# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
|
||||
# define UTIL_sleep(s) Sleep(1000*s)
|
||||
# define UTIL_sleepMilli(milli) Sleep(milli)
|
||||
#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
|
||||
# include <unistd.h>
|
||||
# include <sys/resource.h> /* setpriority */
|
||||
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
|
||||
# if defined(PRIO_PROCESS)
|
||||
# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
# else
|
||||
# define SET_REALTIME_PRIORITY /* disabled */
|
||||
# endif
|
||||
# define UTIL_sleep(s) sleep(s)
|
||||
# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */
|
||||
# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
|
||||
# else
|
||||
# define UTIL_sleepMilli(milli) /* disabled */
|
||||
# endif
|
||||
#else
|
||||
# define SET_REALTIME_PRIORITY /* disabled */
|
||||
# define UTIL_sleep(s) /* disabled */
|
||||
# define UTIL_sleepMilli(milli) /* disabled */
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define LIST_SIZE_INCREASE (8*1024)
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Compiler specifics
|
||||
******************************************/
|
||||
#if defined(__INTEL_COMPILER)
|
||||
# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */
|
||||
#endif
|
||||
#if defined(__GNUC__)
|
||||
# define UTIL_STATIC static __attribute__((unused))
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define UTIL_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define UTIL_STATIC static __inline
|
||||
#else
|
||||
# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Time functions
|
||||
******************************************/
|
||||
#if (PLATFORM_POSIX_VERSION >= 1)
|
||||
#include <unistd.h>
|
||||
#include <sys/times.h> /* times */
|
||||
typedef U64 UTIL_time_t;
|
||||
UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); }
|
||||
UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
|
||||
#elif defined(_WIN32) /* Windows */
|
||||
typedef LARGE_INTEGER UTIL_time_t;
|
||||
UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
|
||||
UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
|
||||
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
|
||||
typedef clock_t UTIL_time_t;
|
||||
UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; }
|
||||
UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
|
||||
#endif
|
||||
|
||||
|
||||
/* returns time span in microseconds */
|
||||
UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond )
|
||||
{
|
||||
UTIL_time_t clockEnd;
|
||||
UTIL_getTime(&clockEnd);
|
||||
return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd);
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
|
||||
{
|
||||
UTIL_time_t clockStart, clockEnd;
|
||||
UTIL_getTime(&clockStart);
|
||||
do {
|
||||
UTIL_getTime(&clockEnd);
|
||||
} while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* File functions
|
||||
******************************************/
|
||||
#if defined(_MSC_VER)
|
||||
#define chmod _chmod
|
||||
typedef struct __stat64 stat_t;
|
||||
#else
|
||||
typedef struct stat stat_t;
|
||||
#endif
|
||||
|
||||
|
||||
UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
|
||||
{
|
||||
int res = 0;
|
||||
struct utimbuf timebuf;
|
||||
|
||||
timebuf.actime = time(NULL);
|
||||
timebuf.modtime = statbuf->st_mtime;
|
||||
res += utime(filename, &timebuf); /* set access and modification times */
|
||||
|
||||
#if !defined(_WIN32)
|
||||
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
|
||||
#endif
|
||||
|
||||
res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
|
||||
|
||||
errno = 0;
|
||||
return -res; /* number of errors is returned */
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
r = _stat64(infilename, statbuf);
|
||||
if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
|
||||
#else
|
||||
r = stat(infilename, statbuf);
|
||||
if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
|
||||
#endif
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC int UTIL_isRegFile(const char* infilename)
|
||||
{
|
||||
stat_t statbuf;
|
||||
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
stat_t statbuf;
|
||||
#if defined(_MSC_VER)
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
|
||||
#else
|
||||
r = stat(infilename, &statbuf);
|
||||
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct __stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
|
||||
#elif defined(__MINGW32__) && defined (__MSVCRT__)
|
||||
struct _stati64 statbuf;
|
||||
r = _stati64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
|
||||
#endif
|
||||
return (U64)statbuf.st_size;
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
U64 total = 0;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++)
|
||||
total += UTIL_getFileSize(fileNamesTable[n]);
|
||||
return total;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* A modified version of realloc().
|
||||
* If UTIL_realloc() fails the original block is freed.
|
||||
*/
|
||||
UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
|
||||
{
|
||||
void *newptr = realloc(ptr, size);
|
||||
if (newptr) return newptr;
|
||||
free(ptr);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
#ifdef _WIN32
|
||||
# define UTIL_HAS_CREATEFILELIST
|
||||
|
||||
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
|
||||
{
|
||||
char* path;
|
||||
int dirLength, fnameLength, pathLength, nbFiles = 0;
|
||||
WIN32_FIND_DATAA cFile;
|
||||
HANDLE hFile;
|
||||
|
||||
dirLength = (int)strlen(dirName);
|
||||
path = (char*) malloc(dirLength + 3);
|
||||
if (!path) return 0;
|
||||
|
||||
memcpy(path, dirName, dirLength);
|
||||
path[dirLength] = '\\';
|
||||
path[dirLength+1] = '*';
|
||||
path[dirLength+2] = 0;
|
||||
|
||||
hFile=FindFirstFileA(path, &cFile);
|
||||
if (hFile == INVALID_HANDLE_VALUE) {
|
||||
fprintf(stderr, "Cannot open directory '%s'\n", dirName);
|
||||
return 0;
|
||||
}
|
||||
free(path);
|
||||
|
||||
do {
|
||||
fnameLength = (int)strlen(cFile.cFileName);
|
||||
path = (char*) malloc(dirLength + fnameLength + 2);
|
||||
if (!path) { FindClose(hFile); return 0; }
|
||||
memcpy(path, dirName, dirLength);
|
||||
path[dirLength] = '\\';
|
||||
memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
|
||||
pathLength = dirLength+1+fnameLength;
|
||||
path[pathLength] = 0;
|
||||
if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
|
||||
if (strcmp (cFile.cFileName, "..") == 0 ||
|
||||
strcmp (cFile.cFileName, ".") == 0) continue;
|
||||
|
||||
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */
|
||||
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
|
||||
}
|
||||
else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) {
|
||||
if (*bufStart + *pos + pathLength >= *bufEnd) {
|
||||
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
|
||||
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
|
||||
*bufEnd = *bufStart + newListSize;
|
||||
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
|
||||
}
|
||||
if (*bufStart + *pos + pathLength < *bufEnd) {
|
||||
strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
|
||||
*pos += pathLength + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
}
|
||||
free(path);
|
||||
} while (FindNextFileA(hFile, &cFile));
|
||||
|
||||
FindClose(hFile);
|
||||
return nbFiles;
|
||||
}
|
||||
|
||||
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
|
||||
# define UTIL_HAS_CREATEFILELIST
|
||||
# include <dirent.h> /* opendir, readdir */
|
||||
# include <string.h> /* strerror, memcpy */
|
||||
|
||||
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
|
||||
{
|
||||
DIR *dir;
|
||||
struct dirent *entry;
|
||||
char* path;
|
||||
int dirLength, fnameLength, pathLength, nbFiles = 0;
|
||||
|
||||
if (!(dir = opendir(dirName))) {
|
||||
fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
|
||||
return 0;
|
||||
}
|
||||
|
||||
dirLength = (int)strlen(dirName);
|
||||
errno = 0;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp (entry->d_name, "..") == 0 ||
|
||||
strcmp (entry->d_name, ".") == 0) continue;
|
||||
fnameLength = (int)strlen(entry->d_name);
|
||||
path = (char*) malloc(dirLength + fnameLength + 2);
|
||||
if (!path) { closedir(dir); return 0; }
|
||||
memcpy(path, dirName, dirLength);
|
||||
path[dirLength] = '/';
|
||||
memcpy(path+dirLength+1, entry->d_name, fnameLength);
|
||||
pathLength = dirLength+1+fnameLength;
|
||||
path[pathLength] = 0;
|
||||
|
||||
if (UTIL_isDirectory(path)) {
|
||||
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */
|
||||
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
|
||||
} else {
|
||||
if (*bufStart + *pos + pathLength >= *bufEnd) {
|
||||
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
|
||||
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
|
||||
*bufEnd = *bufStart + newListSize;
|
||||
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
|
||||
}
|
||||
if (*bufStart + *pos + pathLength < *bufEnd) {
|
||||
strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
|
||||
*pos += pathLength + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
}
|
||||
free(path);
|
||||
errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
|
||||
}
|
||||
|
||||
if (errno != 0) {
|
||||
fprintf(stderr, "readdir(%s) error: %s\n", dirName, strerror(errno));
|
||||
free(*bufStart);
|
||||
*bufStart = NULL;
|
||||
}
|
||||
closedir(dir);
|
||||
return nbFiles;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
|
||||
{
|
||||
(void)bufStart; (void)bufEnd; (void)pos;
|
||||
fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif /* #ifdef _WIN32 */
|
||||
|
||||
/*
|
||||
* UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
|
||||
* and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
|
||||
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
|
||||
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
|
||||
*/
|
||||
UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb)
|
||||
{
|
||||
size_t pos;
|
||||
unsigned i, nbFiles;
|
||||
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
|
||||
char* bufend = buf + LIST_SIZE_INCREASE;
|
||||
const char** fileTable;
|
||||
|
||||
if (!buf) return NULL;
|
||||
|
||||
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
|
||||
if (!UTIL_isDirectory(inputNames[i])) {
|
||||
size_t const len = strlen(inputNames[i]);
|
||||
if (buf + pos + len >= bufend) {
|
||||
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
|
||||
buf = (char*)UTIL_realloc(buf, newListSize);
|
||||
bufend = buf + newListSize;
|
||||
if (!buf) return NULL;
|
||||
}
|
||||
if (buf + pos + len < bufend) {
|
||||
strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
|
||||
pos += len + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
} else {
|
||||
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
|
||||
if (buf == NULL) return NULL;
|
||||
} }
|
||||
|
||||
if (nbFiles == 0) { free(buf); return NULL; }
|
||||
|
||||
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
|
||||
if (!fileTable) { free(buf); return NULL; }
|
||||
|
||||
for (i=0, pos=0; i<nbFiles; i++) {
|
||||
fileTable[i] = buf + pos;
|
||||
pos += strlen(fileTable[i]) + 1;
|
||||
}
|
||||
|
||||
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
|
||||
|
||||
*allocatedBuffer = buf;
|
||||
*allocatedNamesNb = nbFiles;
|
||||
|
||||
return fileTable;
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
|
||||
{
|
||||
if (allocatedBuffer) free(allocatedBuffer);
|
||||
if (filenameTable) free((void*)filenameTable);
|
||||
}
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* UTIL_H_MODULE */
|
||||
Reference in New Issue
Block a user