mirror of
https://github.com/Xahau/xahaud.git
synced 2025-12-06 17:27:52 +00:00
Merge commit '1784f24c5f81e864bf0ad8dcfdf4266ca1108290' as 'src/lz4'
This commit is contained in:
Vinnie Falco
339
src/lz4/programs/COPYING
Normal file
339
src/lz4/programs/COPYING
Normal file
@@ -0,0 +1,339 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 2, June 1991
|
||||
|
||||
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The licenses for most software are designed to take away your
|
||||
freedom to share and change it. By contrast, the GNU General Public
|
||||
License is intended to guarantee your freedom to share and change free
|
||||
software--to make sure the software is free for all its users. This
|
||||
General Public License applies to most of the Free Software
|
||||
Foundation's software and to any other program whose authors commit to
|
||||
using it. (Some other Free Software Foundation software is covered by
|
||||
the GNU Lesser General Public License instead.) You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
this service if you wish), that you receive source code or can get it
|
||||
if you want it, that you can change the software or use pieces of it
|
||||
in new free programs; and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to make restrictions that forbid
|
||||
anyone to deny you these rights or to ask you to surrender the rights.
|
||||
These restrictions translate to certain responsibilities for you if you
|
||||
distribute copies of the software, or if you modify it.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must give the recipients all the rights that
|
||||
you have. You must make sure that they, too, receive or can get the
|
||||
source code. And you must show them these terms so they know their
|
||||
rights.
|
||||
|
||||
We protect your rights with two steps: (1) copyright the software, and
|
||||
(2) offer you this license which gives you legal permission to copy,
|
||||
distribute and/or modify the software.
|
||||
|
||||
Also, for each author's protection and ours, we want to make certain
|
||||
that everyone understands that there is no warranty for this free
|
||||
software. If the software is modified by someone else and passed on, we
|
||||
want its recipients to know that what they have is not the original, so
|
||||
that any problems introduced by others will not reflect on the original
|
||||
authors' reputations.
|
||||
|
||||
Finally, any free program is threatened constantly by software
|
||||
patents. We wish to avoid the danger that redistributors of a free
|
||||
program will individually obtain patent licenses, in effect making the
|
||||
program proprietary. To prevent this, we have made it clear that any
|
||||
patent must be licensed for everyone's free use or not licensed at all.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||
|
||||
0. This License applies to any program or other work which contains
|
||||
a notice placed by the copyright holder saying it may be distributed
|
||||
under the terms of this General Public License. The "Program", below,
|
||||
refers to any such program or work, and a "work based on the Program"
|
||||
means either the Program or any derivative work under copyright law:
|
||||
that is to say, a work containing the Program or a portion of it,
|
||||
either verbatim or with modifications and/or translated into another
|
||||
language. (Hereinafter, translation is included without limitation in
|
||||
the term "modification".) Each licensee is addressed as "you".
|
||||
|
||||
Activities other than copying, distribution and modification are not
|
||||
covered by this License; they are outside its scope. The act of
|
||||
running the Program is not restricted, and the output from the Program
|
||||
is covered only if its contents constitute a work based on the
|
||||
Program (independent of having been made by running the Program).
|
||||
Whether that is true depends on what the Program does.
|
||||
|
||||
1. You may copy and distribute verbatim copies of the Program's
|
||||
source code as you receive it, in any medium, provided that you
|
||||
conspicuously and appropriately publish on each copy an appropriate
|
||||
copyright notice and disclaimer of warranty; keep intact all the
|
||||
notices that refer to this License and to the absence of any warranty;
|
||||
and give any other recipients of the Program a copy of this License
|
||||
along with the Program.
|
||||
|
||||
You may charge a fee for the physical act of transferring a copy, and
|
||||
you may at your option offer warranty protection in exchange for a fee.
|
||||
|
||||
2. You may modify your copy or copies of the Program or any portion
|
||||
of it, thus forming a work based on the Program, and copy and
|
||||
distribute such modifications or work under the terms of Section 1
|
||||
above, provided that you also meet all of these conditions:
|
||||
|
||||
a) You must cause the modified files to carry prominent notices
|
||||
stating that you changed the files and the date of any change.
|
||||
|
||||
b) You must cause any work that you distribute or publish, that in
|
||||
whole or in part contains or is derived from the Program or any
|
||||
part thereof, to be licensed as a whole at no charge to all third
|
||||
parties under the terms of this License.
|
||||
|
||||
c) If the modified program normally reads commands interactively
|
||||
when run, you must cause it, when started running for such
|
||||
interactive use in the most ordinary way, to print or display an
|
||||
announcement including an appropriate copyright notice and a
|
||||
notice that there is no warranty (or else, saying that you provide
|
||||
a warranty) and that users may redistribute the program under
|
||||
these conditions, and telling the user how to view a copy of this
|
||||
License. (Exception: if the Program itself is interactive but
|
||||
does not normally print such an announcement, your work based on
|
||||
the Program is not required to print an announcement.)
|
||||
|
||||
These requirements apply to the modified work as a whole. If
|
||||
identifiable sections of that work are not derived from the Program,
|
||||
and can be reasonably considered independent and separate works in
|
||||
themselves, then this License, and its terms, do not apply to those
|
||||
sections when you distribute them as separate works. But when you
|
||||
distribute the same sections as part of a whole which is a work based
|
||||
on the Program, the distribution of the whole must be on the terms of
|
||||
this License, whose permissions for other licensees extend to the
|
||||
entire whole, and thus to each and every part regardless of who wrote it.
|
||||
|
||||
Thus, it is not the intent of this section to claim rights or contest
|
||||
your rights to work written entirely by you; rather, the intent is to
|
||||
exercise the right to control the distribution of derivative or
|
||||
collective works based on the Program.
|
||||
|
||||
In addition, mere aggregation of another work not based on the Program
|
||||
with the Program (or with a work based on the Program) on a volume of
|
||||
a storage or distribution medium does not bring the other work under
|
||||
the scope of this License.
|
||||
|
||||
3. You may copy and distribute the Program (or a work based on it,
|
||||
under Section 2) in object code or executable form under the terms of
|
||||
Sections 1 and 2 above provided that you also do one of the following:
|
||||
|
||||
a) Accompany it with the complete corresponding machine-readable
|
||||
source code, which must be distributed under the terms of Sections
|
||||
1 and 2 above on a medium customarily used for software interchange; or,
|
||||
|
||||
b) Accompany it with a written offer, valid for at least three
|
||||
years, to give any third party, for a charge no more than your
|
||||
cost of physically performing source distribution, a complete
|
||||
machine-readable copy of the corresponding source code, to be
|
||||
distributed under the terms of Sections 1 and 2 above on a medium
|
||||
customarily used for software interchange; or,
|
||||
|
||||
c) Accompany it with the information you received as to the offer
|
||||
to distribute corresponding source code. (This alternative is
|
||||
allowed only for noncommercial distribution and only if you
|
||||
received the program in object code or executable form with such
|
||||
an offer, in accord with Subsection b above.)
|
||||
|
||||
The source code for a work means the preferred form of the work for
|
||||
making modifications to it. For an executable work, complete source
|
||||
code means all the source code for all modules it contains, plus any
|
||||
associated interface definition files, plus the scripts used to
|
||||
control compilation and installation of the executable. However, as a
|
||||
special exception, the source code distributed need not include
|
||||
anything that is normally distributed (in either source or binary
|
||||
form) with the major components (compiler, kernel, and so on) of the
|
||||
operating system on which the executable runs, unless that component
|
||||
itself accompanies the executable.
|
||||
|
||||
If distribution of executable or object code is made by offering
|
||||
access to copy from a designated place, then offering equivalent
|
||||
access to copy the source code from the same place counts as
|
||||
distribution of the source code, even though third parties are not
|
||||
compelled to copy the source along with the object code.
|
||||
|
||||
4. You may not copy, modify, sublicense, or distribute the Program
|
||||
except as expressly provided under this License. Any attempt
|
||||
otherwise to copy, modify, sublicense or distribute the Program is
|
||||
void, and will automatically terminate your rights under this License.
|
||||
However, parties who have received copies, or rights, from you under
|
||||
this License will not have their licenses terminated so long as such
|
||||
parties remain in full compliance.
|
||||
|
||||
5. You are not required to accept this License, since you have not
|
||||
signed it. However, nothing else grants you permission to modify or
|
||||
distribute the Program or its derivative works. These actions are
|
||||
prohibited by law if you do not accept this License. Therefore, by
|
||||
modifying or distributing the Program (or any work based on the
|
||||
Program), you indicate your acceptance of this License to do so, and
|
||||
all its terms and conditions for copying, distributing or modifying
|
||||
the Program or works based on it.
|
||||
|
||||
6. Each time you redistribute the Program (or any work based on the
|
||||
Program), the recipient automatically receives a license from the
|
||||
original licensor to copy, distribute or modify the Program subject to
|
||||
these terms and conditions. You may not impose any further
|
||||
restrictions on the recipients' exercise of the rights granted herein.
|
||||
You are not responsible for enforcing compliance by third parties to
|
||||
this License.
|
||||
|
||||
7. If, as a consequence of a court judgment or allegation of patent
|
||||
infringement or for any other reason (not limited to patent issues),
|
||||
conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot
|
||||
distribute so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you
|
||||
may not distribute the Program at all. For example, if a patent
|
||||
license would not permit royalty-free redistribution of the Program by
|
||||
all those who receive copies directly or indirectly through you, then
|
||||
the only way you could satisfy both it and this License would be to
|
||||
refrain entirely from distribution of the Program.
|
||||
|
||||
If any portion of this section is held invalid or unenforceable under
|
||||
any particular circumstance, the balance of the section is intended to
|
||||
apply and the section as a whole is intended to apply in other
|
||||
circumstances.
|
||||
|
||||
It is not the purpose of this section to induce you to infringe any
|
||||
patents or other property right claims or to contest validity of any
|
||||
such claims; this section has the sole purpose of protecting the
|
||||
integrity of the free software distribution system, which is
|
||||
implemented by public license practices. Many people have made
|
||||
generous contributions to the wide range of software distributed
|
||||
through that system in reliance on consistent application of that
|
||||
system; it is up to the author/donor to decide if he or she is willing
|
||||
to distribute software through any other system and a licensee cannot
|
||||
impose that choice.
|
||||
|
||||
This section is intended to make thoroughly clear what is believed to
|
||||
be a consequence of the rest of this License.
|
||||
|
||||
8. If the distribution and/or use of the Program is restricted in
|
||||
certain countries either by patents or by copyrighted interfaces, the
|
||||
original copyright holder who places the Program under this License
|
||||
may add an explicit geographical distribution limitation excluding
|
||||
those countries, so that distribution is permitted only in or among
|
||||
countries not thus excluded. In such case, this License incorporates
|
||||
the limitation as if written in the body of this License.
|
||||
|
||||
9. The Free Software Foundation may publish revised and/or new versions
|
||||
of the General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program
|
||||
specifies a version number of this License which applies to it and "any
|
||||
later version", you have the option of following the terms and conditions
|
||||
either of that version or of any later version published by the Free
|
||||
Software Foundation. If the Program does not specify a version number of
|
||||
this License, you may choose any version ever published by the Free Software
|
||||
Foundation.
|
||||
|
||||
10. If you wish to incorporate parts of the Program into other free
|
||||
programs whose distribution conditions are different, write to the author
|
||||
to ask for permission. For software which is copyrighted by the Free
|
||||
Software Foundation, write to the Free Software Foundation; we sometimes
|
||||
make exceptions for this. Our decision will be guided by the two goals
|
||||
of preserving the free status of all derivatives of our free software and
|
||||
of promoting the sharing and reuse of software generally.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
|
||||
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
|
||||
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
|
||||
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
|
||||
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
|
||||
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
|
||||
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
|
||||
REPAIR OR CORRECTION.
|
||||
|
||||
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
|
||||
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
|
||||
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
|
||||
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
|
||||
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
|
||||
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
convey the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program is interactive, make it output a short notice like this
|
||||
when it starts in an interactive mode:
|
||||
|
||||
Gnomovision version 69, Copyright (C) year name of author
|
||||
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, the commands you use may
|
||||
be called something other than `show w' and `show c'; they could even be
|
||||
mouse-clicks or menu items--whatever suits your program.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or your
|
||||
school, if any, to sign a "copyright disclaimer" for the program, if
|
||||
necessary. Here is a sample; alter the names:
|
||||
|
||||
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
|
||||
`Gnomovision' (which makes passes at compilers) written by James Hacker.
|
||||
|
||||
<signature of Ty Coon>, 1 April 1989
|
||||
Ty Coon, President of Vice
|
||||
|
||||
This General Public License does not permit incorporating your program into
|
||||
proprietary programs. If your program is a subroutine library, you may
|
||||
consider it more useful to permit linking proprietary applications with the
|
||||
library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License.
|
||||
206
src/lz4/programs/Makefile
Normal file
206
src/lz4/programs/Makefile
Normal file
@@ -0,0 +1,206 @@
|
||||
# ##########################################################################
|
||||
# LZ4 programs - Makefile
|
||||
# Copyright (C) Yann Collet 2011-2015
|
||||
#
|
||||
# GPL v2 License
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
#
|
||||
# You can contact the author at :
|
||||
# - LZ4 source repository : http://code.google.com/p/lz4/
|
||||
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
|
||||
# ##########################################################################
|
||||
# lz4 : Command Line Utility, supporting gzip-like arguments
|
||||
# lz4c : CLU, supporting also legacy lz4demo arguments
|
||||
# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
|
||||
# fuzzer : Test tool, to check lz4 integrity on target platform
|
||||
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
|
||||
# fullbench : Precisely measure speed for each LZ4 function variant
|
||||
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
|
||||
# ##########################################################################
|
||||
|
||||
RELEASE?= r126
|
||||
|
||||
DESTDIR?=
|
||||
PREFIX ?= /usr
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
|
||||
FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
BINDIR=$(PREFIX)/bin
|
||||
MANDIR=$(PREFIX)/share/man/man1
|
||||
LZ4DIR=../lib
|
||||
|
||||
TEST_FILES = COPYING
|
||||
TEST_TARGETS=test-native
|
||||
|
||||
|
||||
# Define *.exe as extension for Windows systems
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
EXT =.exe
|
||||
VOID = nul
|
||||
else
|
||||
EXT =
|
||||
VOID = /dev/null
|
||||
endif
|
||||
|
||||
|
||||
# Select test target for Travis CI's Build Matrix
|
||||
TRAVIS_TARGET=$(LZ4_TRAVIS_CI_ENV)
|
||||
|
||||
|
||||
default: lz4 lz4c
|
||||
|
||||
all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 frametest frametest32 datagen
|
||||
|
||||
lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c
|
||||
$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
|
||||
|
||||
lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c
|
||||
$(CC) -m32 $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
|
||||
|
||||
fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
datagen : datagen.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
|
||||
clean:
|
||||
@rm -f core *.o *.test \
|
||||
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) \
|
||||
frametest$(EXT) frametest32$(EXT) \
|
||||
datagen$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
|
||||
|
||||
install: lz4 lz4c
|
||||
@echo Installing binaries
|
||||
@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
|
||||
@install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT)
|
||||
@ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat
|
||||
@install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT)
|
||||
@echo Installing man pages
|
||||
@install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
|
||||
@install -m 644 lz4c.1 $(DESTDIR)$(MANDIR)/lz4c.1
|
||||
@install -m 644 lz4cat.1 $(DESTDIR)$(MANDIR)/lz4cat.1
|
||||
@echo lz4 installation completed
|
||||
|
||||
uninstall:
|
||||
rm -f $(DESTDIR)$(BINDIR)/lz4cat
|
||||
[ -x $(DESTDIR)$(BINDIR)/lz4$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4$(EXT)
|
||||
[ -x $(DESTDIR)$(BINDIR)/lz4c$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4c$(EXT)
|
||||
[ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1
|
||||
[ -f $(DESTDIR)$(MANDIR)/lz4c.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4c.1
|
||||
[ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
|
||||
@echo lz4 programs successfully uninstalled
|
||||
|
||||
test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-mem
|
||||
|
||||
test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-mem32
|
||||
|
||||
test-all: test test32
|
||||
|
||||
test-travis: $(TRAVIS_TARGET)
|
||||
|
||||
test-lz4: lz4 datagen
|
||||
./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID)
|
||||
./datagen | ./lz4 | ./lz4 -vdq > $(VOID)
|
||||
./datagen -g6M -p100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID)
|
||||
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID)
|
||||
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID)
|
||||
# test frame concatenation with null-length frame
|
||||
@echo -n > empty.test
|
||||
@echo hi > nonempty.test
|
||||
cat nonempty.test empty.test nonempty.test > orig.test
|
||||
@./lz4 -zq empty.test > empty.lz4.test
|
||||
@./lz4 -zq nonempty.test > nonempty.lz4.test
|
||||
cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test
|
||||
./lz4 -d concat.lz4.test > result.test
|
||||
sdiff orig.test result.test
|
||||
@rm *.test
|
||||
@echo frame concatenation test completed
|
||||
# test frame concatenation with null-length frame
|
||||
|
||||
|
||||
test-lz4c: lz4c datagen
|
||||
./datagen -g256MB | ./lz4c -l -v | ./lz4c -vdq > $(VOID)
|
||||
|
||||
test-lz4c32: lz4 lz4c32 lz4 datagen
|
||||
./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -vdq > $(VOID)
|
||||
./datagen -g16KB | ./lz4c32 -9 | ./lz4 -vdq > $(VOID)
|
||||
./datagen | ./lz4c32 | ./lz4c32 -vdq > $(VOID)
|
||||
./datagen | ./lz4c32 | ./lz4 -vdq > $(VOID)
|
||||
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -vdq > $(VOID)
|
||||
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -vdq > $(VOID)
|
||||
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID)
|
||||
|
||||
test-fullbench: fullbench
|
||||
./fullbench --no-prompt $(TEST_FILES)
|
||||
|
||||
test-fullbench32: fullbench32
|
||||
./fullbench32 --no-prompt $(TEST_FILES)
|
||||
|
||||
test-fuzzer: fuzzer
|
||||
./fuzzer
|
||||
|
||||
test-fuzzer32: fuzzer32
|
||||
./fuzzer32
|
||||
|
||||
test-frametest: frametest
|
||||
./frametest
|
||||
|
||||
test-frametest32: frametest32
|
||||
./frametest32
|
||||
|
||||
test-mem: lz4 datagen fuzzer frametest
|
||||
./datagen -g16KB > tmp
|
||||
valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null
|
||||
./datagen -g16MB > tmp
|
||||
valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp /dev/null
|
||||
./datagen -g256MB > tmp
|
||||
valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp /dev/null
|
||||
rm tmp
|
||||
valgrind --leak-check=yes ./fuzzer -i64 -t1
|
||||
valgrind --leak-check=yes ./frametest -i256
|
||||
|
||||
test-mem32: lz4c32 datagen
|
||||
# unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...
|
||||
|
||||
endif
|
||||
435
src/lz4/programs/bench.c
Normal file
435
src/lz4/programs/bench.c
Normal file
@@ -0,0 +1,435 @@
|
||||
/*
|
||||
bench.c - Demo program to benchmark open-source compression algorithm
|
||||
Copyright (C) Yann Collet 2012-2014
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Compiler Options
|
||||
***************************************/
|
||||
/* Disable some Visual warning messages */
|
||||
#define _CRT_SECURE_NO_WARNINGS
|
||||
#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
|
||||
|
||||
/* Unix Large Files support (>4GB) */
|
||||
#define _FILE_OFFSET_BITS 64
|
||||
#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
|
||||
# define _LARGEFILE_SOURCE
|
||||
#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
|
||||
# define _LARGEFILE64_SOURCE
|
||||
#endif
|
||||
|
||||
/* S_ISREG & gettimeofday() are not supported by MSVC */
|
||||
#if defined(_MSC_VER) || defined(_WIN32)
|
||||
# define BMK_LEGACY_TIMER 1
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <sys/types.h> /* stat64 */
|
||||
#include <sys/stat.h> /* stat64 */
|
||||
|
||||
/* Use ftime() if gettimeofday() is not available on your target */
|
||||
#if defined(BMK_LEGACY_TIMER)
|
||||
# include <sys/timeb.h> /* timeb, ftime */
|
||||
#else
|
||||
# include <sys/time.h> /* gettimeofday */
|
||||
#endif
|
||||
|
||||
#include "lz4.h"
|
||||
#define COMPRESSOR0 LZ4_compress_local
|
||||
static int LZ4_compress_local(const char* src, char* dst, int size, int clevel) { (void)clevel; return LZ4_compress(src, dst, size); }
|
||||
#include "lz4hc.h"
|
||||
#define COMPRESSOR1 LZ4_compressHC2
|
||||
#define DEFAULTCOMPRESSOR COMPRESSOR0
|
||||
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
/**************************************
|
||||
* Compiler specifics
|
||||
***************************************/
|
||||
#if !defined(S_ISREG)
|
||||
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Basic Types
|
||||
***************************************/
|
||||
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define NBLOOPS 3
|
||||
#define TIMELOOP 2000
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define MAX_MEM (2 GB - 64 MB)
|
||||
#define DEFAULT_CHUNKSIZE (4 MB)
|
||||
|
||||
|
||||
/**************************************
|
||||
* Local structures
|
||||
***************************************/
|
||||
struct chunkParameters
|
||||
{
|
||||
U32 id;
|
||||
char* origBuffer;
|
||||
char* compressedBuffer;
|
||||
int origSize;
|
||||
int compressedSize;
|
||||
};
|
||||
|
||||
struct compressionParameters
|
||||
{
|
||||
int (*compressionFunction)(const char*, char*, int, int);
|
||||
int (*decompressionFunction)(const char*, char*, int);
|
||||
};
|
||||
|
||||
|
||||
/**************************************
|
||||
* MACRO
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
|
||||
|
||||
/**************************************
|
||||
* Benchmark Parameters
|
||||
***************************************/
|
||||
static int chunkSize = DEFAULT_CHUNKSIZE;
|
||||
static int nbIterations = NBLOOPS;
|
||||
static int BMK_pause = 0;
|
||||
|
||||
void BMK_SetBlocksize(int bsize) { chunkSize = bsize; }
|
||||
|
||||
void BMK_SetNbIterations(int nbLoops)
|
||||
{
|
||||
nbIterations = nbLoops;
|
||||
DISPLAY("- %i iterations -\n", nbIterations);
|
||||
}
|
||||
|
||||
void BMK_SetPause(void) { BMK_pause = 1; }
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Private functions
|
||||
**********************************************************/
|
||||
|
||||
#if defined(BMK_LEGACY_TIMER)
|
||||
|
||||
static int BMK_GetMilliStart(void)
|
||||
{
|
||||
/* Based on Legacy ftime()
|
||||
Rolls over every ~ 12.1 days (0x100000/24/60/60)
|
||||
Use GetMilliSpan to correct for rollover */
|
||||
struct timeb tb;
|
||||
int nCount;
|
||||
ftime( &tb );
|
||||
nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static int BMK_GetMilliStart(void)
|
||||
{
|
||||
/* Based on newer gettimeofday()
|
||||
Use GetMilliSpan to correct for rollover */
|
||||
struct timeval tv;
|
||||
int nCount;
|
||||
gettimeofday(&tv, NULL);
|
||||
nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
static int BMK_GetMilliSpan( int nTimeStart )
|
||||
{
|
||||
int nSpan = BMK_GetMilliStart() - nTimeStart;
|
||||
if ( nSpan < 0 )
|
||||
nSpan += 0x100000 * 1000;
|
||||
return nSpan;
|
||||
}
|
||||
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
size_t step = 64 MB;
|
||||
BYTE* testmem=NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||
requiredMem += 2*step;
|
||||
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
|
||||
|
||||
while (!testmem)
|
||||
{
|
||||
requiredMem -= step;
|
||||
testmem = (BYTE*) malloc ((size_t)requiredMem);
|
||||
}
|
||||
|
||||
free (testmem);
|
||||
return (size_t) (requiredMem - step);
|
||||
}
|
||||
|
||||
|
||||
static U64 BMK_GetFileSize(char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
#endif
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
|
||||
return (U64)statbuf.st_size;
|
||||
}
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Public function
|
||||
**********************************************************/
|
||||
|
||||
int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
|
||||
{
|
||||
int fileIdx=0;
|
||||
char* orig_buff;
|
||||
struct compressionParameters compP;
|
||||
int cfunctionId;
|
||||
|
||||
U64 totals = 0;
|
||||
U64 totalz = 0;
|
||||
double totalc = 0.;
|
||||
double totald = 0.;
|
||||
|
||||
|
||||
/* Init */
|
||||
if (cLevel <= 3) cfunctionId = 0; else cfunctionId = 1;
|
||||
switch (cfunctionId)
|
||||
{
|
||||
#ifdef COMPRESSOR0
|
||||
case 0 : compP.compressionFunction = COMPRESSOR0; break;
|
||||
#endif
|
||||
#ifdef COMPRESSOR1
|
||||
case 1 : compP.compressionFunction = COMPRESSOR1; break;
|
||||
#endif
|
||||
default : compP.compressionFunction = DEFAULTCOMPRESSOR;
|
||||
}
|
||||
compP.decompressionFunction = LZ4_decompress_fast;
|
||||
|
||||
/* Loop for each file */
|
||||
while (fileIdx<nbFiles)
|
||||
{
|
||||
FILE* inFile;
|
||||
char* inFileName;
|
||||
U64 inFileSize;
|
||||
size_t benchedSize;
|
||||
int nbChunks;
|
||||
int maxCompressedChunkSize;
|
||||
size_t readSize;
|
||||
char* compressedBuffer; int compressedBuffSize;
|
||||
struct chunkParameters* chunkP;
|
||||
U32 crcOrig;
|
||||
|
||||
/* Check file existence */
|
||||
inFileName = fileNamesTable[fileIdx++];
|
||||
inFile = fopen( inFileName, "rb" );
|
||||
if (inFile==NULL)
|
||||
{
|
||||
DISPLAY( "Pb opening %s\n", inFileName);
|
||||
return 11;
|
||||
}
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
inFileSize = BMK_GetFileSize(inFileName);
|
||||
benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2;
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize)
|
||||
{
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
|
||||
}
|
||||
|
||||
/* Alloc */
|
||||
chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters));
|
||||
orig_buff = (char*)malloc((size_t )benchedSize);
|
||||
nbChunks = (int) ((int)benchedSize / chunkSize) + 1;
|
||||
maxCompressedChunkSize = LZ4_compressBound(chunkSize);
|
||||
compressedBuffSize = nbChunks * maxCompressedChunkSize;
|
||||
compressedBuffer = (char*)malloc((size_t )compressedBuffSize);
|
||||
|
||||
|
||||
if (!orig_buff || !compressedBuffer)
|
||||
{
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
free(orig_buff);
|
||||
free(compressedBuffer);
|
||||
free(chunkP);
|
||||
fclose(inFile);
|
||||
return 12;
|
||||
}
|
||||
|
||||
/* Init chunks data */
|
||||
{
|
||||
int i;
|
||||
size_t remaining = benchedSize;
|
||||
char* in = orig_buff;
|
||||
char* out = compressedBuffer;
|
||||
for (i=0; i<nbChunks; i++)
|
||||
{
|
||||
chunkP[i].id = i;
|
||||
chunkP[i].origBuffer = in; in += chunkSize;
|
||||
if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
|
||||
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
|
||||
chunkP[i].compressedSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Fill input buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
readSize = fread(orig_buff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
|
||||
if (readSize != benchedSize)
|
||||
{
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(orig_buff);
|
||||
free(compressedBuffer);
|
||||
free(chunkP);
|
||||
return 13;
|
||||
}
|
||||
|
||||
/* Calculating input Checksum */
|
||||
crcOrig = XXH32(orig_buff, (unsigned int)benchedSize,0);
|
||||
|
||||
|
||||
/* Bench */
|
||||
{
|
||||
int loopNb, chunkNb;
|
||||
size_t cSize=0;
|
||||
double fastestC = 100000000., fastestD = 100000000.;
|
||||
double ratio=0.;
|
||||
U32 crcCheck=0;
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
|
||||
{
|
||||
int nbLoops;
|
||||
int milliTime;
|
||||
|
||||
/* Compression */
|
||||
DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, inFileName, (int)benchedSize);
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) compressedBuffer[i]=(char)i; } /* warmimg up memory */
|
||||
|
||||
nbLoops = 0;
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliStart() == milliTime);
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
|
||||
{
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
|
||||
chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, cLevel);
|
||||
nbLoops++;
|
||||
}
|
||||
milliTime = BMK_GetMilliSpan(milliTime);
|
||||
|
||||
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops;
|
||||
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
|
||||
ratio = (double)cSize/(double)benchedSize*100.;
|
||||
|
||||
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000.);
|
||||
|
||||
/* Decompression */
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing area, for CRC checking */
|
||||
|
||||
nbLoops = 0;
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliStart() == milliTime);
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
|
||||
{
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
|
||||
chunkP[chunkNb].compressedSize = LZ4_decompress_fast(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].origSize);
|
||||
nbLoops++;
|
||||
}
|
||||
milliTime = BMK_GetMilliSpan(milliTime);
|
||||
|
||||
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops;
|
||||
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
|
||||
|
||||
/* CRC Checking */
|
||||
crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0);
|
||||
if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; }
|
||||
}
|
||||
|
||||
if (crcOrig==crcCheck)
|
||||
{
|
||||
if (ratio<100.)
|
||||
DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
|
||||
else
|
||||
DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
|
||||
}
|
||||
totals += benchedSize;
|
||||
totalz += cSize;
|
||||
totalc += fastestC;
|
||||
totald += fastestD;
|
||||
}
|
||||
|
||||
free(orig_buff);
|
||||
free(compressedBuffer);
|
||||
free(chunkP);
|
||||
}
|
||||
|
||||
if (nbFiles > 1)
|
||||
DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.);
|
||||
|
||||
if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
42
src/lz4/programs/bench.h
Normal file
42
src/lz4/programs/bench.h
Normal file
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
bench.h - Demo program to benchmark open-source compression algorithm
|
||||
Copyright (C) Yann Collet 2012-2014
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
- LZ4 public forum : https://group.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* Main function */
|
||||
int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel);
|
||||
|
||||
/* Set Parameters */
|
||||
void BMK_SetBlocksize(int bsize);
|
||||
void BMK_SetNbIterations(int nbLoops);
|
||||
void BMK_SetPause(void);
|
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
286
src/lz4/programs/datagen.c
Normal file
286
src/lz4/programs/datagen.c
Normal file
@@ -0,0 +1,286 @@
|
||||
/*
|
||||
datagen.c - compressible data generator test tool
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : http://code.google.com/p/lz4
|
||||
- LZ4 source mirror : https://github.com/Cyan4973/lz4
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
Remove Visual warning messages
|
||||
**************************************/
|
||||
#define _CRT_SECURE_NO_WARNINGS // fgets
|
||||
|
||||
|
||||
/**************************************
|
||||
Includes
|
||||
**************************************/
|
||||
#include <stdio.h> // fgets, sscanf
|
||||
#include <string.h> // strcmp
|
||||
|
||||
|
||||
/**************************************
|
||||
Basic Types
|
||||
**************************************/
|
||||
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
Constants
|
||||
**************************************/
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION "r125"
|
||||
#endif
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define CDG_SIZE_DEFAULT (64 KB)
|
||||
#define CDG_SEED_DEFAULT 0
|
||||
#define CDG_COMPRESSIBILITY_DEFAULT 50
|
||||
#define PRIME1 2654435761U
|
||||
#define PRIME2 2246822519U
|
||||
|
||||
|
||||
/**************************************
|
||||
Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
|
||||
/**************************************
|
||||
Local Parameters
|
||||
**************************************/
|
||||
static unsigned no_prompt = 0;
|
||||
static char* programName;
|
||||
static unsigned displayLevel = 2;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
functions
|
||||
*********************************************************/
|
||||
|
||||
#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static unsigned int CDG_rand(U32* src)
|
||||
{
|
||||
U32 rand32 = *src;
|
||||
rand32 *= PRIME1;
|
||||
rand32 += PRIME2;
|
||||
rand32 = CDG_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32;
|
||||
}
|
||||
|
||||
|
||||
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
|
||||
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15)
|
||||
#define CDG_RANDCHAR (((CDG_rand(seed) >> 9) & 63) + '0')
|
||||
static void CDG_generate(U64 size, U32* seed, double proba)
|
||||
{
|
||||
BYTE fullbuff[32 KB + 128 KB + 1];
|
||||
BYTE* buff = fullbuff + 32 KB;
|
||||
U64 total=0;
|
||||
U32 P32 = (U32)(32768 * proba);
|
||||
U32 pos=1;
|
||||
U32 genBlockSize = 128 KB;
|
||||
|
||||
// Build initial prefix
|
||||
fullbuff[0] = CDG_RANDCHAR;
|
||||
while (pos<32 KB)
|
||||
{
|
||||
// Select : Literal (char) or Match (within 32K)
|
||||
if (CDG_RAND15BITS < P32)
|
||||
{
|
||||
// Copy (within 64K)
|
||||
U32 d;
|
||||
int ref;
|
||||
int length = CDG_RANDLENGTH + 4;
|
||||
U32 offset = CDG_RAND15BITS + 1;
|
||||
if (offset > pos) offset = pos;
|
||||
ref = pos - offset;
|
||||
d = pos + length;
|
||||
while (pos < d) fullbuff[pos++] = fullbuff[ref++];
|
||||
}
|
||||
else
|
||||
{
|
||||
// Literal (noise)
|
||||
U32 d = pos + CDG_RANDLENGTH;
|
||||
while (pos < d) fullbuff[pos++] = CDG_RANDCHAR;
|
||||
}
|
||||
}
|
||||
|
||||
// Generate compressible data
|
||||
pos = 0;
|
||||
while (total < size)
|
||||
{
|
||||
if (size-total < 128 KB) genBlockSize = (U32)(size-total);
|
||||
total += genBlockSize;
|
||||
buff[genBlockSize] = 0;
|
||||
pos = 0;
|
||||
while (pos<genBlockSize)
|
||||
{
|
||||
// Select : Literal (char) or Match (within 32K)
|
||||
if (CDG_RAND15BITS < P32)
|
||||
{
|
||||
// Copy (within 64K)
|
||||
int ref;
|
||||
U32 d;
|
||||
int length = CDG_RANDLENGTH + 4;
|
||||
U32 offset = CDG_RAND15BITS + 1;
|
||||
if (pos + length > genBlockSize ) length = genBlockSize - pos;
|
||||
ref = pos - offset;
|
||||
d = pos + length;
|
||||
while (pos < d) buff[pos++] = buff[ref++];
|
||||
}
|
||||
else
|
||||
{
|
||||
// Literal (noise)
|
||||
U32 d;
|
||||
int length = CDG_RANDLENGTH;
|
||||
if (pos + length > genBlockSize) length = genBlockSize - pos;
|
||||
d = pos + length;
|
||||
while (pos < d) buff[pos++] = CDG_RANDCHAR;
|
||||
}
|
||||
}
|
||||
// output datagen
|
||||
pos=0;
|
||||
for (;pos+512<=genBlockSize;pos+=512)
|
||||
printf("%512.512s", buff+pos);
|
||||
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
|
||||
// Regenerate prefix
|
||||
memcpy(fullbuff, buff + 96 KB, 32 KB);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int CDG_usage(void)
|
||||
{
|
||||
DISPLAY( "Compressible data generator\n");
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [size] [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -g# : generate # data (default:%i)\n", CDG_SIZE_DEFAULT);
|
||||
DISPLAY( " -s# : Select seed (default:%i)\n", CDG_SEED_DEFAULT);
|
||||
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", CDG_COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int argNb;
|
||||
int proba = CDG_COMPRESSIBILITY_DEFAULT;
|
||||
U64 size = CDG_SIZE_DEFAULT;
|
||||
U32 seed = CDG_SEED_DEFAULT;
|
||||
|
||||
// Check command line
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; // Protection if argument empty
|
||||
|
||||
// Decode command (note : aggregated commands are allowed)
|
||||
if (*argument=='-')
|
||||
{
|
||||
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; }
|
||||
|
||||
argument++;
|
||||
while (*argument!=0)
|
||||
{
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return CDG_usage();
|
||||
case 'g':
|
||||
argument++;
|
||||
size=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
size *= 10;
|
||||
size += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (*argument=='K') { size <<= 10; argument++; }
|
||||
if (*argument=='M') { size <<= 20; argument++; }
|
||||
if (*argument=='G') { size <<= 30; argument++; }
|
||||
if (*argument=='B') { argument++; }
|
||||
break;
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 'p':
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba<0) proba=0;
|
||||
if (proba>100) proba=100;
|
||||
break;
|
||||
case 'v':
|
||||
displayLevel = 4;
|
||||
argument++;
|
||||
break;
|
||||
default: ;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Get Seed
|
||||
DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
|
||||
|
||||
CDG_generate(size, &seed, ((double)proba) / 100);
|
||||
|
||||
return 0;
|
||||
}
|
||||
669
src/lz4/programs/frametest.c
Normal file
669
src/lz4/programs/frametest.c
Normal file
@@ -0,0 +1,669 @@
|
||||
/*
|
||||
frameTest - test tool for lz4frame
|
||||
Copyright (C) Yann Collet 2014
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
Compiler specific
|
||||
**************************************/
|
||||
#define _CRT_SECURE_NO_WARNINGS // fgets
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
|
||||
#endif
|
||||
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
#ifdef __GNUC__
|
||||
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
|
||||
# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> // free
|
||||
#include <stdio.h> // fgets, sscanf
|
||||
#include <sys/timeb.h> // timeb
|
||||
#include <string.h> // strcmp
|
||||
#include "lz4frame_static.h"
|
||||
#include "xxhash.h" // XXH64
|
||||
|
||||
|
||||
/**************************************
|
||||
Basic Types
|
||||
**************************************/
|
||||
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
Constants
|
||||
**************************************/
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION ""
|
||||
#endif
|
||||
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const U32 nbTestsDefault = 256 KB;
|
||||
#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
|
||||
|
||||
|
||||
/**************************************
|
||||
Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
|
||||
if ((FUZ_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=4)) \
|
||||
{ g_time = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel>=4) fflush(stdout); } }
|
||||
static const U32 refreshRate = 150;
|
||||
static U32 g_time = 0;
|
||||
|
||||
|
||||
/*****************************************
|
||||
Local Parameters
|
||||
*****************************************/
|
||||
static U32 no_prompt = 0;
|
||||
static char* programName;
|
||||
static U32 displayLevel = 2;
|
||||
static U32 pause = 0;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
Fuzzer functions
|
||||
*********************************************************/
|
||||
static U32 FUZ_GetMilliStart(void)
|
||||
{
|
||||
struct timeb tb;
|
||||
U32 nCount;
|
||||
ftime( &tb );
|
||||
nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
|
||||
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
||||
{
|
||||
U32 nCurrent = FUZ_GetMilliStart();
|
||||
U32 nSpan = nCurrent - nTimeStart;
|
||||
if (nTimeStart > nCurrent)
|
||||
nSpan += 0x100000 * 1000;
|
||||
return nSpan;
|
||||
}
|
||||
|
||||
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* src)
|
||||
{
|
||||
U32 rand32 = *src;
|
||||
rand32 *= prime1;
|
||||
rand32 += prime2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
|
||||
#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF)
|
||||
#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
|
||||
static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, double proba, U32* seed)
|
||||
{
|
||||
BYTE* BBuffer = (BYTE*)buffer;
|
||||
unsigned pos = 0;
|
||||
U32 P32 = (U32)(32768 * proba);
|
||||
|
||||
// First Byte
|
||||
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
|
||||
|
||||
while (pos < bufferSize)
|
||||
{
|
||||
// Select : Literal (noise) or copy (within 64K)
|
||||
if (FUZ_RAND15BITS < P32)
|
||||
{
|
||||
// Copy (within 64K)
|
||||
unsigned match, end;
|
||||
unsigned length = FUZ_RANDLENGTH + 4;
|
||||
unsigned offset = FUZ_RAND15BITS + 1;
|
||||
if (offset > pos) offset = pos;
|
||||
if (pos + length > bufferSize) length = bufferSize - pos;
|
||||
match = pos - offset;
|
||||
end = pos + length;
|
||||
while (pos < end) BBuffer[pos++] = BBuffer[match++];
|
||||
}
|
||||
else
|
||||
{
|
||||
// Literal (noise)
|
||||
unsigned end;
|
||||
unsigned length = FUZ_RANDLENGTH;
|
||||
if (pos + length > bufferSize) length = bufferSize - pos;
|
||||
end = pos + length;
|
||||
while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static unsigned FUZ_highbit(U32 v32)
|
||||
{
|
||||
unsigned nbBits = 0;
|
||||
if (v32==0) return 0;
|
||||
while (v32)
|
||||
{
|
||||
v32 >>= 1;
|
||||
nbBits ++;
|
||||
}
|
||||
return nbBits;
|
||||
}
|
||||
|
||||
|
||||
int basicTests(U32 seed, double compressibility)
|
||||
{
|
||||
int testResult = 0;
|
||||
void* CNBuffer;
|
||||
void* compressedBuffer;
|
||||
void* decodedBuffer;
|
||||
U32 randState = seed;
|
||||
size_t cSize, testSize;
|
||||
LZ4F_preferences_t prefs = { 0 };
|
||||
LZ4F_decompressionContext_t dCtx;
|
||||
U64 crcOrig;
|
||||
|
||||
// Create compressible test buffer
|
||||
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
|
||||
compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL));
|
||||
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
|
||||
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
|
||||
crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
|
||||
// Trivial tests : one-step frame
|
||||
testSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
DISPLAYLEVEL(3, "Using NULL preferences : \n");
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Decompression test : \n");
|
||||
{
|
||||
size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
size_t compressedBufferSize = cSize;
|
||||
BYTE* op = (BYTE*)decodedBuffer;
|
||||
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
|
||||
BYTE* ip = (BYTE*)compressedBuffer;
|
||||
BYTE* const iend = (BYTE*)compressedBuffer + cSize;
|
||||
U64 crcDest;
|
||||
|
||||
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
|
||||
DISPLAYLEVEL(3, "Single Block : \n");
|
||||
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
|
||||
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Byte after byte : \n");
|
||||
while (ip < iend)
|
||||
{
|
||||
size_t oSize = oend-op;
|
||||
size_t iSize = 1;
|
||||
//DISPLAY("%7i \n", (int)(ip-(BYTE*)compressedBuffer));
|
||||
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
}
|
||||
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
|
||||
|
||||
errorCode = LZ4F_freeDecompressionContext(dCtx);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "Using 64 KB block : \n");
|
||||
prefs.frameInfo.blockSizeID = max64KB;
|
||||
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "without checksum : \n");
|
||||
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Using 256 KB block : \n");
|
||||
prefs.frameInfo.blockSizeID = max256KB;
|
||||
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Decompression test : \n");
|
||||
{
|
||||
size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
|
||||
BYTE* op = (BYTE*)decodedBuffer;
|
||||
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
|
||||
BYTE* ip = (BYTE*)compressedBuffer;
|
||||
BYTE* const iend = (BYTE*)compressedBuffer + cSize;
|
||||
U64 crcDest;
|
||||
|
||||
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
|
||||
DISPLAYLEVEL(3, "random segment sizes : \n");
|
||||
while (ip < iend)
|
||||
{
|
||||
unsigned nbBits = FUZ_rand(&randState) % maxBits;
|
||||
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
|
||||
size_t oSize = oend-op;
|
||||
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
|
||||
//DISPLAY("%7i : + %6i\n", (int)(ip-(BYTE*)compressedBuffer), (int)iSize);
|
||||
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
}
|
||||
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
|
||||
|
||||
errorCode = LZ4F_freeDecompressionContext(dCtx);
|
||||
if (LZ4F_isError(errorCode)) goto _output_error;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "without checksum : \n");
|
||||
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Using 1 MB block : \n");
|
||||
prefs.frameInfo.blockSizeID = max1MB;
|
||||
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "without checksum : \n");
|
||||
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "Using 4 MB block : \n");
|
||||
prefs.frameInfo.blockSizeID = max4MB;
|
||||
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAYLEVEL(3, "without checksum : \n");
|
||||
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
|
||||
if (LZ4F_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
|
||||
|
||||
DISPLAY("Basic tests completed \n");
|
||||
_end:
|
||||
free(CNBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
return testResult;
|
||||
|
||||
_output_error:
|
||||
testResult = 1;
|
||||
DISPLAY("Error detected ! \n");
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
||||
static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
|
||||
{
|
||||
int p=0;
|
||||
BYTE* b1=(BYTE*)buff1;
|
||||
BYTE* b2=(BYTE*)buff2;
|
||||
if (nonContiguous)
|
||||
{
|
||||
DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
|
||||
return;
|
||||
}
|
||||
while (b1[p]==b2[p]) p++;
|
||||
DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]);
|
||||
}
|
||||
|
||||
|
||||
static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
|
||||
|
||||
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility)
|
||||
{
|
||||
unsigned testResult = 0;
|
||||
unsigned testNb = 0;
|
||||
void* srcBuffer = NULL;
|
||||
void* compressedBuffer = NULL;
|
||||
void* decodedBuffer = NULL;
|
||||
U32 coreRand = seed;
|
||||
LZ4F_decompressionContext_t dCtx = NULL;
|
||||
LZ4F_compressionContext_t cCtx = NULL;
|
||||
size_t result;
|
||||
XXH64_state_t xxh64;
|
||||
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
// Create buffers
|
||||
result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
|
||||
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
|
||||
result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
|
||||
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
|
||||
srcBuffer = malloc(srcDataLength);
|
||||
CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
|
||||
compressedBuffer = malloc(LZ4F_compressFrameBound(srcDataLength, NULL));
|
||||
CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
|
||||
decodedBuffer = malloc(srcDataLength);
|
||||
CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
|
||||
FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
|
||||
|
||||
// jump to requested testNb
|
||||
for (testNb =0; testNb < startTest; testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
|
||||
|
||||
// main fuzzer loop
|
||||
for ( ; testNb < nbTests; testNb++)
|
||||
{
|
||||
U32 randState = coreRand ^ prime1;
|
||||
unsigned BSId = 4 + (FUZ_rand(&randState) & 3);
|
||||
unsigned BMId = FUZ_rand(&randState) & 1;
|
||||
unsigned CCflag = FUZ_rand(&randState) & 1;
|
||||
unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
|
||||
LZ4F_preferences_t prefs = { 0 };
|
||||
LZ4F_compressOptions_t cOptions = { 0 };
|
||||
LZ4F_decompressOptions_t dOptions = { 0 };
|
||||
unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
|
||||
size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
|
||||
size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
|
||||
size_t cSize;
|
||||
U64 crcOrig, crcDecoded;
|
||||
LZ4F_preferences_t* prefsPtr = &prefs;
|
||||
|
||||
(void)FUZ_rand(&coreRand); // update rand seed
|
||||
prefs.frameInfo.blockMode = (blockMode_t)BMId;
|
||||
prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId;
|
||||
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag;
|
||||
prefs.autoFlush = autoflush;
|
||||
prefs.compressionLevel = FUZ_rand(&randState) % 5;
|
||||
if ((FUZ_rand(&randState)&0xF) == 1) prefsPtr = NULL;
|
||||
|
||||
DISPLAYUPDATE(2, "\r%5u ", testNb);
|
||||
crcOrig = XXH64((BYTE*)srcBuffer+srcStart, (U32)srcSize, 1);
|
||||
|
||||
if ((FUZ_rand(&randState)&0xF) == 2)
|
||||
{
|
||||
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), (char*)srcBuffer + srcStart, srcSize, prefsPtr);
|
||||
CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
|
||||
}
|
||||
else
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)srcBuffer + srcStart;
|
||||
const BYTE* const iend = ip + srcSize;
|
||||
BYTE* op = (BYTE*)compressedBuffer;
|
||||
BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL);
|
||||
unsigned maxBits = FUZ_highbit((U32)srcSize);
|
||||
result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr);
|
||||
CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result);
|
||||
op += result;
|
||||
while (ip < iend)
|
||||
{
|
||||
unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits;
|
||||
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
|
||||
size_t oSize = LZ4F_compressBound(iSize, prefsPtr);
|
||||
unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
|
||||
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
|
||||
cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
|
||||
|
||||
result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
|
||||
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
|
||||
op += result;
|
||||
ip += iSize;
|
||||
|
||||
if (forceFlush)
|
||||
{
|
||||
result = LZ4F_flush(cCtx, op, oend-op, &cOptions);
|
||||
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
|
||||
op += result;
|
||||
}
|
||||
}
|
||||
result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions);
|
||||
CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result);
|
||||
op += result;
|
||||
cSize = op-(BYTE*)compressedBuffer;
|
||||
}
|
||||
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)compressedBuffer;
|
||||
const BYTE* const iend = ip + cSize;
|
||||
BYTE* op = (BYTE*)decodedBuffer;
|
||||
BYTE* const oend = op + srcDataLength;
|
||||
unsigned maxBits = FUZ_highbit((U32)cSize);
|
||||
unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1;
|
||||
nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */
|
||||
XXH64_reset(&xxh64, 1);
|
||||
while (ip < iend)
|
||||
{
|
||||
unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
|
||||
unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1;
|
||||
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1;
|
||||
size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
|
||||
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
|
||||
if (oSize > (size_t)(oend-op)) oSize = oend-op;
|
||||
dOptions.stableDst = FUZ_rand(&randState) & 1;
|
||||
if (nonContiguousDst==2) dOptions.stableDst = 0;
|
||||
//if (ip == compressedBuffer+62073) DISPLAY("oSize : %i : pos %i \n", (int)oSize, (int)(op-(BYTE*)decodedBuffer));
|
||||
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
|
||||
//if (op+oSize >= (BYTE*)decodedBuffer+94727) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
|
||||
//if ((int)result<0) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
|
||||
if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
|
||||
CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result));
|
||||
XXH64_update(&xxh64, op, (U32)oSize);
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
op += nonContiguousDst;
|
||||
if (nonContiguousDst==2) op = decodedBuffer; // overwritten destination
|
||||
}
|
||||
CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
|
||||
crcDecoded = XXH64_digest(&xxh64);
|
||||
if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
|
||||
CHECK(crcDecoded != crcOrig, "Decompression corruption");
|
||||
}
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "\rAll tests completed \n");
|
||||
|
||||
_end:
|
||||
LZ4F_freeDecompressionContext(dCtx);
|
||||
LZ4F_freeCompressionContext(cCtx);
|
||||
free(srcBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
|
||||
if (pause)
|
||||
{
|
||||
DISPLAY("press enter to finish \n");
|
||||
getchar();
|
||||
}
|
||||
return testResult;
|
||||
|
||||
_output_error:
|
||||
testResult = 1;
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
||||
int FUZ_usage(void)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
|
||||
DISPLAY( " -s# : Select seed (default:prompt user)\n");
|
||||
DISPLAY( " -t# : Select starting test number (default:0)\n");
|
||||
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -v : verbose\n");
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
U32 seed=0;
|
||||
int seedset=0;
|
||||
int argNb;
|
||||
int nbTests = nbTestsDefault;
|
||||
int testNb = 0;
|
||||
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
|
||||
int result=0;
|
||||
|
||||
// Check command line
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; // Protection if argument empty
|
||||
|
||||
// Decode command (note : aggregated commands are allowed)
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
if (!strcmp(argument, "--no-prompt"))
|
||||
{
|
||||
no_prompt=1;
|
||||
seedset=1;
|
||||
displayLevel=1;
|
||||
continue;
|
||||
}
|
||||
argument++;
|
||||
|
||||
while (*argument!=0)
|
||||
{
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return FUZ_usage();
|
||||
case 'v':
|
||||
argument++;
|
||||
displayLevel=4;
|
||||
break;
|
||||
case 'q':
|
||||
argument++;
|
||||
displayLevel--;
|
||||
break;
|
||||
case 'p': /* pause at the end */
|
||||
argument++;
|
||||
pause = 1;
|
||||
break;
|
||||
|
||||
case 'i':
|
||||
argument++;
|
||||
nbTests=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
nbTests *= 10;
|
||||
nbTests += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
seedset=1;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 't':
|
||||
argument++;
|
||||
testNb=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
testNb *= 10;
|
||||
testNb += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 'P': /* compressibility % */
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba<0) proba=0;
|
||||
if (proba>100) proba=100;
|
||||
break;
|
||||
default:
|
||||
;
|
||||
return FUZ_usage();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get Seed
|
||||
printf("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);
|
||||
|
||||
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
|
||||
printf("Seed = %u\n", seed);
|
||||
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
|
||||
|
||||
if (nbTests<=0) nbTests=1;
|
||||
|
||||
if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
|
||||
if (result) return 1;
|
||||
return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
||||
}
|
||||
842
src/lz4/programs/fullbench.c
Normal file
842
src/lz4/programs/fullbench.c
Normal file
@@ -0,0 +1,842 @@
|
||||
/*
|
||||
bench.c - Demo program to benchmark open-source compression algorithm
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : http://code.google.com/p/lz4
|
||||
- LZ4 source mirror : https://github.com/Cyan4973/lz4
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
//**************************************
|
||||
// Compiler Options
|
||||
//**************************************
|
||||
// Disable some Visual warning messages
|
||||
#define _CRT_SECURE_NO_WARNINGS
|
||||
#define _CRT_SECURE_NO_DEPRECATE // VS2005
|
||||
|
||||
// Unix Large Files support (>4GB)
|
||||
#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
|
||||
# define _LARGEFILE_SOURCE
|
||||
# define _FILE_OFFSET_BITS 64
|
||||
#elif ! defined(__LP64__) // No point defining Large file for 64 bit
|
||||
# define _LARGEFILE64_SOURCE
|
||||
#endif
|
||||
|
||||
// S_ISREG & gettimeofday() are not supported by MSVC
|
||||
#if defined(_MSC_VER) || defined(_WIN32)
|
||||
# define BMK_LEGACY_TIMER 1
|
||||
#endif
|
||||
|
||||
|
||||
//**************************************
|
||||
// Includes
|
||||
//**************************************
|
||||
#include <stdlib.h> // malloc
|
||||
#include <stdio.h> // fprintf, fopen, ftello64
|
||||
#include <sys/types.h> // stat64
|
||||
#include <sys/stat.h> // stat64
|
||||
#include <string.h> // strcmp
|
||||
|
||||
// Use ftime() if gettimeofday() is not available on your target
|
||||
#if defined(BMK_LEGACY_TIMER)
|
||||
# include <sys/timeb.h> // timeb, ftime
|
||||
#else
|
||||
# include <sys/time.h> // gettimeofday
|
||||
#endif
|
||||
|
||||
#include "lz4.h"
|
||||
#include "lz4hc.h"
|
||||
#include "lz4frame.h"
|
||||
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
//**************************************
|
||||
// Compiler Options
|
||||
//**************************************
|
||||
// S_ISREG & gettimeofday() are not supported by MSVC
|
||||
#if !defined(S_ISREG)
|
||||
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
|
||||
#endif
|
||||
|
||||
// GCC does not support _rotl outside of Windows
|
||||
#if !defined(_WIN32)
|
||||
# define _rotl(x,r) ((x << r) | (x >> (32 - r)))
|
||||
#endif
|
||||
|
||||
|
||||
//**************************************
|
||||
// Basic Types
|
||||
//**************************************
|
||||
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
//****************************
|
||||
// Constants
|
||||
//****************************
|
||||
#define PROGRAM_DESCRIPTION "LZ4 speed analyzer"
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION ""
|
||||
#endif
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__
|
||||
|
||||
#define NBLOOPS 6
|
||||
#define TIMELOOP 2500
|
||||
|
||||
#define KNUTH 2654435761U
|
||||
#define MAX_MEM (1984<<20)
|
||||
#define DEFAULT_CHUNKSIZE (4<<20)
|
||||
|
||||
#define ALL_COMPRESSORS 0
|
||||
#define ALL_DECOMPRESSORS 0
|
||||
|
||||
|
||||
//**************************************
|
||||
// Local structures
|
||||
//**************************************
|
||||
struct chunkParameters
|
||||
{
|
||||
U32 id;
|
||||
char* origBuffer;
|
||||
char* compressedBuffer;
|
||||
int origSize;
|
||||
int compressedSize;
|
||||
};
|
||||
|
||||
|
||||
//**************************************
|
||||
// MACRO
|
||||
//**************************************
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__)
|
||||
|
||||
|
||||
|
||||
//**************************************
|
||||
// Benchmark Parameters
|
||||
//**************************************
|
||||
static int chunkSize = DEFAULT_CHUNKSIZE;
|
||||
static int nbIterations = NBLOOPS;
|
||||
static int BMK_pause = 0;
|
||||
static int compressionTest = 1;
|
||||
static int decompressionTest = 1;
|
||||
static int compressionAlgo = ALL_COMPRESSORS;
|
||||
static int decompressionAlgo = ALL_DECOMPRESSORS;
|
||||
static int no_prompt = 0;
|
||||
|
||||
void BMK_SetBlocksize(int bsize)
|
||||
{
|
||||
chunkSize = bsize;
|
||||
DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10);
|
||||
}
|
||||
|
||||
void BMK_SetNbIterations(int nbLoops)
|
||||
{
|
||||
nbIterations = nbLoops;
|
||||
DISPLAY("- %i iterations -\n", nbIterations);
|
||||
}
|
||||
|
||||
void BMK_SetPause(void)
|
||||
{
|
||||
BMK_pause = 1;
|
||||
}
|
||||
|
||||
//*********************************************************
|
||||
// Private functions
|
||||
//*********************************************************
|
||||
|
||||
#if defined(BMK_LEGACY_TIMER)
|
||||
|
||||
static int BMK_GetMilliStart(void)
|
||||
{
|
||||
// Based on Legacy ftime()
|
||||
// Rolls over every ~ 12.1 days (0x100000/24/60/60)
|
||||
// Use GetMilliSpan to correct for rollover
|
||||
struct timeb tb;
|
||||
int nCount;
|
||||
ftime( &tb );
|
||||
nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static int BMK_GetMilliStart(void)
|
||||
{
|
||||
// Based on newer gettimeofday()
|
||||
// Use GetMilliSpan to correct for rollover
|
||||
struct timeval tv;
|
||||
int nCount;
|
||||
gettimeofday(&tv, NULL);
|
||||
nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
static int BMK_GetMilliSpan( int nTimeStart )
|
||||
{
|
||||
int nSpan = BMK_GetMilliStart() - nTimeStart;
|
||||
if ( nSpan < 0 )
|
||||
nSpan += 0x100000 * 1000;
|
||||
return nSpan;
|
||||
}
|
||||
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
size_t step = (64U<<20); // 64 MB
|
||||
BYTE* testmem=NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 25) + 1) << 26);
|
||||
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
|
||||
|
||||
requiredMem += 2*step;
|
||||
while (!testmem)
|
||||
{
|
||||
requiredMem -= step;
|
||||
testmem = (BYTE*) malloc ((size_t)requiredMem);
|
||||
}
|
||||
|
||||
free (testmem);
|
||||
return (size_t) (requiredMem - step);
|
||||
}
|
||||
|
||||
|
||||
static U64 BMK_GetFileSize(char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
#endif
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
|
||||
return (U64)statbuf.st_size;
|
||||
}
|
||||
|
||||
|
||||
/*********************************************************
|
||||
Benchmark function
|
||||
*********************************************************/
|
||||
|
||||
static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static void* stateLZ4;
|
||||
static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_withState(stateLZ4, in, out, inSize);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static LZ4_stream_t* ctx;
|
||||
static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_continue(ctx, in, out, inSize);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
|
||||
LZ4_stream_t LZ4_dict;
|
||||
static void* local_LZ4_resetDictT(const char* fake)
|
||||
{
|
||||
(void)fake;
|
||||
memset(&LZ4_dict, 0, sizeof(LZ4_stream_t));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize);
|
||||
static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_forceExtDict(&LZ4_dict, in, out, inSize);
|
||||
}
|
||||
|
||||
|
||||
static void* stateLZ4HC;
|
||||
static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize);
|
||||
}
|
||||
|
||||
static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_continue((LZ4_streamHC_t*)ctx, in, out, inSize);
|
||||
}
|
||||
|
||||
static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)
|
||||
{
|
||||
return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL);
|
||||
}
|
||||
|
||||
static int local_LZ4_saveDict(const char* in, char* out, int inSize)
|
||||
{
|
||||
(void)in;
|
||||
return LZ4_saveDict(&LZ4_dict, out, inSize);
|
||||
}
|
||||
|
||||
LZ4_streamHC_t LZ4_dictHC;
|
||||
static int local_LZ4_saveDictHC(const char* in, char* out, int inSize)
|
||||
{
|
||||
(void)in;
|
||||
return LZ4_saveDictHC(&LZ4_dictHC, out, inSize);
|
||||
}
|
||||
|
||||
|
||||
static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_fast(in, out, outSize);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_fast_withPrefix64k(in, out, outSize);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65536);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_safe_usingDict(in, out, inSize, outSize, out - 65536, 65536);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const char* dict, int dictSize);
|
||||
|
||||
static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
|
||||
}
|
||||
|
||||
static LZ4F_decompressionContext_t g_dCtx;
|
||||
|
||||
static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
size_t srcSize = inSize;
|
||||
size_t dstSize = outSize;
|
||||
size_t result;
|
||||
result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL);
|
||||
if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame\n"); exit(8); }
|
||||
if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); }
|
||||
return (int)dstSize;
|
||||
}
|
||||
|
||||
|
||||
int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
{
|
||||
int fileIdx=0;
|
||||
char* orig_buff;
|
||||
# define NB_COMPRESSION_ALGORITHMS 16
|
||||
double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
|
||||
double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0};
|
||||
# define NB_DECOMPRESSION_ALGORITHMS 9
|
||||
double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0};
|
||||
size_t errorCode;
|
||||
|
||||
errorCode = LZ4F_createDecompressionContext(&g_dCtx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode))
|
||||
{
|
||||
DISPLAY("dctx allocation issue \n");
|
||||
return 10;
|
||||
}
|
||||
|
||||
// Loop for each file
|
||||
while (fileIdx<nbFiles)
|
||||
{
|
||||
FILE* inFile;
|
||||
char* inFileName;
|
||||
U64 inFileSize;
|
||||
size_t benchedSize;
|
||||
int nbChunks;
|
||||
int maxCompressedChunkSize;
|
||||
struct chunkParameters* chunkP;
|
||||
size_t readSize;
|
||||
char* compressed_buff; int compressedBuffSize;
|
||||
U32 crcOriginal;
|
||||
|
||||
|
||||
// Init
|
||||
stateLZ4 = LZ4_createStream();
|
||||
stateLZ4HC = LZ4_createStreamHC();
|
||||
|
||||
// Check file existence
|
||||
inFileName = fileNamesTable[fileIdx++];
|
||||
inFile = fopen( inFileName, "rb" );
|
||||
if (inFile==NULL)
|
||||
{
|
||||
DISPLAY( "Pb opening %s\n", inFileName);
|
||||
return 11;
|
||||
}
|
||||
|
||||
// Memory allocation & restrictions
|
||||
inFileSize = BMK_GetFileSize(inFileName);
|
||||
benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2;
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize)
|
||||
{
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
|
||||
}
|
||||
|
||||
// Alloc
|
||||
chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters));
|
||||
orig_buff = (char*) malloc((size_t)benchedSize);
|
||||
nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize);
|
||||
maxCompressedChunkSize = LZ4_compressBound(chunkSize);
|
||||
compressedBuffSize = nbChunks * maxCompressedChunkSize;
|
||||
compressed_buff = (char*)malloc((size_t)compressedBuffSize);
|
||||
|
||||
|
||||
if(!orig_buff || !compressed_buff)
|
||||
{
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
free(orig_buff);
|
||||
free(compressed_buff);
|
||||
free(chunkP);
|
||||
fclose(inFile);
|
||||
return 12;
|
||||
}
|
||||
|
||||
// Fill input buffer
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
readSize = fread(orig_buff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
|
||||
if(readSize != benchedSize)
|
||||
{
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(orig_buff);
|
||||
free(compressed_buff);
|
||||
free(chunkP);
|
||||
return 13;
|
||||
}
|
||||
|
||||
// Calculating input Checksum
|
||||
crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0);
|
||||
|
||||
|
||||
// Bench
|
||||
{
|
||||
int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb;
|
||||
size_t cSize=0;
|
||||
double ratio=0.;
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY(" %s : \n", inFileName);
|
||||
|
||||
// Compression Algorithms
|
||||
for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
|
||||
{
|
||||
const char* compressorName;
|
||||
int (*compressionFunction)(const char*, char*, int);
|
||||
void* (*initFunction)(const char*) = NULL;
|
||||
double bestTime = 100000000.;
|
||||
|
||||
// Init data chunks
|
||||
{
|
||||
int i;
|
||||
size_t remaining = benchedSize;
|
||||
char* in = orig_buff;
|
||||
char* out = compressed_buff;
|
||||
nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize);
|
||||
for (i=0; i<nbChunks; i++)
|
||||
{
|
||||
chunkP[i].id = i;
|
||||
chunkP[i].origBuffer = in; in += chunkSize;
|
||||
if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
|
||||
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
|
||||
chunkP[i].compressedSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;
|
||||
|
||||
switch(cAlgNb)
|
||||
{
|
||||
case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
|
||||
case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
|
||||
case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
|
||||
case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
|
||||
case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break;
|
||||
case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break;
|
||||
case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
|
||||
case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
|
||||
case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
|
||||
case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
|
||||
case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
|
||||
case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
|
||||
case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
|
||||
case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame";
|
||||
chunkP[0].origSize = (int)benchedSize; nbChunks=1;
|
||||
break;
|
||||
case 15: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict";
|
||||
LZ4_loadDict(&LZ4_dict, chunkP[0].origBuffer, chunkP[0].origSize);
|
||||
break;
|
||||
case 16: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC";
|
||||
LZ4_loadDictHC(&LZ4_dictHC, chunkP[0].origBuffer, chunkP[0].origSize);
|
||||
break;
|
||||
default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
|
||||
}
|
||||
|
||||
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
|
||||
{
|
||||
double averageTime;
|
||||
int milliTime;
|
||||
|
||||
PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize);
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warming up memory
|
||||
|
||||
nb_loops = 0;
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliStart() == milliTime);
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
|
||||
{
|
||||
if (initFunction!=NULL) ctx = initFunction(chunkP[0].origBuffer);
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
|
||||
{
|
||||
chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
|
||||
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
|
||||
}
|
||||
if (initFunction!=NULL) free(ctx);
|
||||
nb_loops++;
|
||||
}
|
||||
milliTime = BMK_GetMilliSpan(milliTime);
|
||||
|
||||
averageTime = (double)milliTime / nb_loops;
|
||||
if (averageTime < bestTime) bestTime = averageTime;
|
||||
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
|
||||
ratio = (double)cSize/(double)benchedSize*100.;
|
||||
PROGRESS("%1i- %-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
|
||||
}
|
||||
|
||||
if (ratio<100.)
|
||||
DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
|
||||
else
|
||||
DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
|
||||
|
||||
totalCTime[cAlgNb] += bestTime;
|
||||
totalCSize[cAlgNb] += cSize;
|
||||
}
|
||||
|
||||
// Prepare layout for decompression
|
||||
// Init data chunks
|
||||
{
|
||||
int i;
|
||||
size_t remaining = benchedSize;
|
||||
char* in = orig_buff;
|
||||
char* out = compressed_buff;
|
||||
nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize);
|
||||
for (i=0; i<nbChunks; i++)
|
||||
{
|
||||
chunkP[i].id = i;
|
||||
chunkP[i].origBuffer = in; in += chunkSize;
|
||||
if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
|
||||
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
|
||||
chunkP[i].compressedSize = 0;
|
||||
}
|
||||
}
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
|
||||
{
|
||||
chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
|
||||
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
|
||||
}
|
||||
|
||||
// Decompression Algorithms
|
||||
for (dAlgNb=1; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++)
|
||||
{
|
||||
//const char* dName = decompressionNames[dAlgNb];
|
||||
const char* dName;
|
||||
int (*decompressionFunction)(const char*, char*, int, int);
|
||||
double bestTime = 100000000.;
|
||||
|
||||
if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue;
|
||||
|
||||
switch(dAlgNb)
|
||||
{
|
||||
case 1: decompressionFunction = local_LZ4_decompress_fast; dName = "LZ4_decompress_fast"; break;
|
||||
case 2: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; dName = "LZ4_decompress_fast_withPrefix64k"; break;
|
||||
case 3: decompressionFunction = local_LZ4_decompress_fast_usingDict; dName = "LZ4_decompress_fast_usingDict"; break;
|
||||
case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break;
|
||||
case 5: decompressionFunction = LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break;
|
||||
case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break;
|
||||
case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; break;
|
||||
case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
|
||||
case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress";
|
||||
errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL);
|
||||
if (LZ4F_isError(errorCode)) { DISPLAY("Preparation error compressing frame\n"); return 1; }
|
||||
chunkP[0].origSize = (int)benchedSize;
|
||||
chunkP[0].compressedSize = (int)errorCode;
|
||||
nbChunks = 1;
|
||||
break;
|
||||
default : DISPLAY("ERROR ! Bad decompression algorithm Id !! \n"); free(chunkP); return 1;
|
||||
}
|
||||
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking
|
||||
|
||||
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
|
||||
{
|
||||
double averageTime;
|
||||
int milliTime;
|
||||
U32 crcDecoded;
|
||||
|
||||
PROGRESS("%1i- %-29.29s :%10i ->\r", loopNb, dName, (int)benchedSize);
|
||||
|
||||
nb_loops = 0;
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliStart() == milliTime);
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
|
||||
{
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
|
||||
{
|
||||
int decodedSize = decompressionFunction(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize);
|
||||
if (chunkP[chunkNb].origSize != decodedSize) DISPLAY("ERROR ! %s() == %i != %i !! \n", dName, decodedSize, chunkP[chunkNb].origSize), exit(1);
|
||||
}
|
||||
nb_loops++;
|
||||
}
|
||||
milliTime = BMK_GetMilliSpan(milliTime);
|
||||
|
||||
averageTime = (double)milliTime / nb_loops;
|
||||
if (averageTime < bestTime) bestTime = averageTime;
|
||||
|
||||
PROGRESS("%1i- %-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
|
||||
|
||||
// CRC Checking
|
||||
crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
|
||||
if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); }
|
||||
}
|
||||
|
||||
DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
|
||||
|
||||
totalDTime[dAlgNb] += bestTime;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
free(orig_buff);
|
||||
free(compressed_buff);
|
||||
free(chunkP);
|
||||
}
|
||||
|
||||
if (BMK_pause) { printf("press enter...\n"); getchar(); }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int usage(char* exename)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -c : compression tests only\n");
|
||||
DISPLAY( " -d : decompression tests only\n");
|
||||
DISPLAY( " -H/-h : Help (this text + advanced options)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int usage_advanced(void)
|
||||
{
|
||||
DISPLAY( "\nAdvanced options :\n");
|
||||
DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS);
|
||||
DISPLAY( " -d# : test only decompression function # [1-%i]\n", NB_DECOMPRESSION_ALGORITHMS);
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
|
||||
DISPLAY( " -B# : Block size [4-7](default : 7)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int badusage(char* exename)
|
||||
{
|
||||
DISPLAY("Wrong parameters\n");
|
||||
usage(exename);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int i,
|
||||
filenamesStart=2;
|
||||
char* exename=argv[0];
|
||||
char* input_filename=0;
|
||||
|
||||
// Welcome message
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
|
||||
if (argc<2) { badusage(exename); return 1; }
|
||||
|
||||
for(i=1; i<argc; i++)
|
||||
{
|
||||
char* argument = argv[i];
|
||||
|
||||
if(!argument) continue; // Protection if argument empty
|
||||
if (!strcmp(argument, "--no-prompt"))
|
||||
{
|
||||
no_prompt = 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Decode command (note : aggregated commands are allowed)
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
while (argument[1]!=0)
|
||||
{
|
||||
argument ++;
|
||||
|
||||
switch(argument[0])
|
||||
{
|
||||
// Select compression algorithm only
|
||||
case 'c':
|
||||
decompressionTest = 0;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9'))
|
||||
{
|
||||
compressionAlgo *= 10;
|
||||
compressionAlgo += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
// Select decompression algorithm only
|
||||
case 'd':
|
||||
compressionTest = 0;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9'))
|
||||
{
|
||||
decompressionAlgo *= 10;
|
||||
decompressionAlgo += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
// Display help on usage
|
||||
case 'h' :
|
||||
case 'H': usage(exename); usage_advanced(); return 0;
|
||||
|
||||
// Modify Block Properties
|
||||
case 'B':
|
||||
while (argument[1]!=0)
|
||||
switch(argument[1])
|
||||
{
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
{
|
||||
int B = argument[1] - '0';
|
||||
int S = 1 << (8 + 2*B);
|
||||
BMK_SetBlocksize(S);
|
||||
argument++;
|
||||
break;
|
||||
}
|
||||
case 'D': argument++; break;
|
||||
default : goto _exit_blockProperties;
|
||||
}
|
||||
_exit_blockProperties:
|
||||
break;
|
||||
|
||||
// Modify Nb Iterations
|
||||
case 'i':
|
||||
if ((argument[1] >='1') && (argument[1] <='9'))
|
||||
{
|
||||
int iters = argument[1] - '0';
|
||||
BMK_SetNbIterations(iters);
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
// Pause at the end (hidden option)
|
||||
case 'p': BMK_SetPause(); break;
|
||||
|
||||
// Unknown command
|
||||
default : badusage(exename); return 1;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// first provided filename is input
|
||||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
||||
|
||||
}
|
||||
|
||||
// No input filename ==> Error
|
||||
if(!input_filename) { badusage(exename); return 1; }
|
||||
|
||||
return fullSpeedBench(argv+filenamesStart, argc-filenamesStart);
|
||||
|
||||
}
|
||||
|
||||
1167
src/lz4/programs/fuzzer.c
Normal file
1167
src/lz4/programs/fuzzer.c
Normal file
File diff suppressed because it is too large
Load Diff
88
src/lz4/programs/lz4.1
Normal file
88
src/lz4/programs/lz4.1
Normal file
@@ -0,0 +1,88 @@
|
||||
\"
|
||||
\" lz4.1: This is a manual page for 'lz4' program. This file is part of the
|
||||
\" lz4 <https://code.google.com/p/lz4/> project.
|
||||
\"
|
||||
|
||||
\" No hyphenation
|
||||
.hy 0
|
||||
.nr HY 0
|
||||
|
||||
.TH lz4 "1" "2014-02-27" "lz4" "User Commands"
|
||||
.SH NAME
|
||||
\fBlz4\fR - Extremely fast compression algorithm
|
||||
|
||||
.SH SYNOPSIS
|
||||
.TP 5
|
||||
\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
|
||||
|
||||
.SH DESCRIPTION
|
||||
.PP
|
||||
\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on
|
||||
the \fBLZ77\fR family of compression scheme. At the compression speed of 400
|
||||
MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features
|
||||
an extremely fast decoder, with speed in multiple GB/s per core, typically
|
||||
reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports
|
||||
following options
|
||||
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.B \-1
|
||||
fast compression (default)
|
||||
.TP
|
||||
.B \-9
|
||||
high compression
|
||||
.TP
|
||||
.B \-d
|
||||
decompression
|
||||
.TP
|
||||
.B \-f
|
||||
overwrite output without prompting
|
||||
.TP
|
||||
.B \-h/\-H
|
||||
display help/long help and exit
|
||||
.TP
|
||||
.B \-V
|
||||
display Version number and exit
|
||||
.TP
|
||||
.B \-v
|
||||
verbose mode
|
||||
.TP
|
||||
.B \-q
|
||||
suppress warnings; specify twice to suppress errors too
|
||||
.TP
|
||||
.B \-c
|
||||
force write to standard output, even if it is the console
|
||||
.TP
|
||||
.B \-t
|
||||
test compressed file integrity
|
||||
.TP
|
||||
.B \-z
|
||||
force compression
|
||||
.TP
|
||||
.B \-l
|
||||
use Legacy format (useful for Linux Kernel compression)
|
||||
.TP
|
||||
.B \-B#
|
||||
block size [4-7](default : 7)
|
||||
B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB
|
||||
.TP
|
||||
.B \-BD
|
||||
block dependency (improve compression ratio)
|
||||
.TP
|
||||
.B \-BX
|
||||
enable block checksum (default:disabled)
|
||||
.TP
|
||||
.B \-Sx
|
||||
disable stream checksum (default:enabled)
|
||||
.TP
|
||||
.B \-b
|
||||
benchmark file(s)
|
||||
.TP
|
||||
.B \-i#
|
||||
iteration loops [1-9](default : 3), benchmark mode only
|
||||
|
||||
.SH BUGS
|
||||
Report bugs at:- https://code.google.com/p/lz4/
|
||||
|
||||
.SH AUTHOR
|
||||
Yann Collet
|
||||
33
src/lz4/programs/lz4c.1
Normal file
33
src/lz4/programs/lz4c.1
Normal file
@@ -0,0 +1,33 @@
|
||||
\"
|
||||
\" lz4c.1: This is a manual page for 'lz4c' program. This file is part of the
|
||||
\" lz4 <https://code.google.com/p/lz4/> project.
|
||||
\"
|
||||
|
||||
\" No hyphenation
|
||||
.hy 0
|
||||
.nr HY 0
|
||||
|
||||
.TH lz4c "1" "2014-04-15" "lz4c" "User Commands"
|
||||
.SH NAME
|
||||
\fBlz4\fR - Extremely fast compression algorithm
|
||||
|
||||
.SH SYNOPSIS
|
||||
.TP 5
|
||||
\fBlz4c\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
|
||||
|
||||
.SH DESCRIPTION
|
||||
.PP
|
||||
\fBlz4c\fR is the legacy version of \fBlz4\fR.
|
||||
As such, it supports older supplementary legacy commands.
|
||||
\fBlz4c\fR is now deprecated.
|
||||
It is recommended to use \fBlz4\fR instead whenever possible.
|
||||
|
||||
To get a list of commands specific to lz4c, do :
|
||||
lz4c -h
|
||||
|
||||
|
||||
.SH BUGS
|
||||
Report bugs at:- https://code.google.com/p/lz4/
|
||||
|
||||
.SH AUTHOR
|
||||
Yann Collet
|
||||
32
src/lz4/programs/lz4cat.1
Normal file
32
src/lz4/programs/lz4cat.1
Normal file
@@ -0,0 +1,32 @@
|
||||
\"
|
||||
\" lz4cat.1: This is a manual page for 'lz4cat' program. This file is part of
|
||||
\" the lz4 <https://code.google.com/p/lz4/> project.
|
||||
\"
|
||||
|
||||
\" No hyphenation
|
||||
.hy 0
|
||||
.nr HY 0
|
||||
|
||||
.TH lz4cat "1" "2014-06-20" "lz4cat" "User Commands"
|
||||
.SH NAME
|
||||
\fBlz4cat\fR - Utility based on LZ4
|
||||
|
||||
.SH SYNOPSIS
|
||||
.TP 5
|
||||
\fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE]
|
||||
|
||||
.SH DESCRIPTION
|
||||
.PP
|
||||
\fBlz4cat\fR is an utility based on \fBlz4\fR, an extremely fast lossless compression algorithm.
|
||||
|
||||
\fBlz4cat\fR decompress input file or stream, redirecting its output to the console.
|
||||
It is equivalent to \fBlz4 -cd\fR,
|
||||
|
||||
Available options are the same as \fBlz4\fR ones (man lz4).
|
||||
|
||||
|
||||
.SH BUGS
|
||||
Report bugs at:- https://code.google.com/p/lz4/
|
||||
|
||||
.SH AUTHOR
|
||||
Yann Collet
|
||||
501
src/lz4/programs/lz4cli.c
Normal file
501
src/lz4/programs/lz4cli.c
Normal file
@@ -0,0 +1,501 @@
|
||||
/*
|
||||
LZ4cli - LZ4 Command Line Interface
|
||||
Copyright (C) Yann Collet 2011-2014
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
/*
|
||||
Note : this is stand-alone program.
|
||||
It is not part of LZ4 compression library, it is a user program of the LZ4 library.
|
||||
The license of LZ4 library is BSD.
|
||||
The license of xxHash library is BSD.
|
||||
The license of this compression CLI program is GPLv2.
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Tuning parameters
|
||||
***************************************/
|
||||
/* ENABLE_LZ4C_LEGACY_OPTIONS :
|
||||
Control the availability of -c0, -c1 and -hc legacy arguments
|
||||
Default : Legacy options are disabled */
|
||||
/* #define ENABLE_LZ4C_LEGACY_OPTIONS */
|
||||
|
||||
|
||||
/**************************************
|
||||
* Compiler Options
|
||||
***************************************/
|
||||
/* Disable some Visual warning messages */
|
||||
#ifdef _MSC_VER
|
||||
# define _CRT_SECURE_NO_WARNINGS
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
#define _POSIX_SOURCE 1 /* for fileno() within <stdio.h> on unix */
|
||||
|
||||
|
||||
/****************************
|
||||
* Includes
|
||||
*****************************/
|
||||
#include <stdio.h> /* fprintf, getchar */
|
||||
#include <stdlib.h> /* exit, calloc, free */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include "bench.h" /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */
|
||||
#include "lz4io.h"
|
||||
|
||||
|
||||
/****************************
|
||||
* OS-specific Includes
|
||||
*****************************/
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
# include <io.h> /* _setmode, _isatty */
|
||||
# ifdef __MINGW32__
|
||||
int _fileno(FILE *stream); /* MINGW somehow forgets to include this prototype into <stdio.h> */
|
||||
# endif
|
||||
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
|
||||
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
||||
#else
|
||||
# include <unistd.h> /* isatty */
|
||||
# define SET_BINARY_MODE(file)
|
||||
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
|
||||
#endif
|
||||
|
||||
|
||||
/*****************************
|
||||
* Constants
|
||||
******************************/
|
||||
#define COMPRESSOR_NAME "LZ4 command line interface"
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION "r126"
|
||||
#endif
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
|
||||
#define LZ4_EXTENSION ".lz4"
|
||||
#define LZ4_CAT "lz4cat"
|
||||
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define LZ4_BLOCKSIZEID_DEFAULT 7
|
||||
|
||||
|
||||
/**************************************
|
||||
* Macros
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static unsigned displayLevel = 2; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
|
||||
|
||||
|
||||
/**************************************
|
||||
* Local Variables
|
||||
***************************************/
|
||||
static char* programName;
|
||||
|
||||
|
||||
/**************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#define DEBUG 0
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, "\n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/**************************************
|
||||
* Version modifiers
|
||||
***************************************/
|
||||
#define EXTENDED_ARGUMENTS
|
||||
#define EXTENDED_HELP
|
||||
#define EXTENDED_FORMAT
|
||||
#define DEFAULT_COMPRESSOR LZ4IO_compressFilename
|
||||
#define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename
|
||||
int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel); /* hidden function */
|
||||
|
||||
|
||||
/****************************
|
||||
* Functions
|
||||
*****************************/
|
||||
static int usage(void)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [arg] [input] [output]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "input : a filename\n");
|
||||
DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark);
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -1 : Fast compression (default) \n");
|
||||
DISPLAY( " -9 : High compression \n");
|
||||
DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION);
|
||||
DISPLAY( " -z : force compression\n");
|
||||
DISPLAY( " -f : overwrite output without prompting \n");
|
||||
DISPLAY( " -h/-H : display help/long help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int usage_advanced(void)
|
||||
{
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
usage();
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Advanced arguments :\n");
|
||||
DISPLAY( " -V : display Version number and exit\n");
|
||||
DISPLAY( " -v : verbose mode\n");
|
||||
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
|
||||
DISPLAY( " -c : force write to standard output, even if it is the console\n");
|
||||
DISPLAY( " -t : test compressed file integrity\n");
|
||||
DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n");
|
||||
DISPLAY( " -B# : Block size [4-7](default : 7)\n");
|
||||
DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
|
||||
/* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */
|
||||
DISPLAY( " -Sx : disable stream checksum (default:enabled)\n");
|
||||
DISPLAY( "Benchmark arguments :\n");
|
||||
DISPLAY( " -b : benchmark file(s)\n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");
|
||||
#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
|
||||
DISPLAY( "Legacy arguments :\n");
|
||||
DISPLAY( " -c0 : fast compression\n");
|
||||
DISPLAY( " -c1 : high compression\n");
|
||||
DISPLAY( " -hc : high compression\n");
|
||||
DISPLAY( " -y : overwrite output without prompting \n");
|
||||
DISPLAY( " -s : suppress warnings \n");
|
||||
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
|
||||
EXTENDED_HELP;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int usage_longhelp(void)
|
||||
{
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Which values can get [output] ? \n");
|
||||
DISPLAY( "[output] : a filename\n");
|
||||
DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark);
|
||||
DISPLAY( " '%s' to discard output (test mode)\n", NULL_OUTPUT);
|
||||
DISPLAY( "[output] can be left empty. In this case, it receives the following value : \n");
|
||||
DISPLAY( " - if stdout is not the console, then [output] = stdout \n");
|
||||
DISPLAY( " - if stdout is console : \n");
|
||||
DISPLAY( " + if compression selected, output to filename%s \n", LZ4_EXTENSION);
|
||||
DISPLAY( " + if decompression selected, output to filename without '%s'\n", LZ4_EXTENSION);
|
||||
DISPLAY( " > if input filename has no '%s' extension : error\n", LZ4_EXTENSION);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Compression levels : \n");
|
||||
DISPLAY( "There are technically 2 accessible compression levels.\n");
|
||||
DISPLAY( "-0 ... -2 => Fast compression\n");
|
||||
DISPLAY( "-3 ... -9 => High compression\n");
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "stdin, stdout and the console : \n");
|
||||
DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n");
|
||||
DISPLAY( "%s will refuse to read from console, or write to console \n", programName);
|
||||
DISPLAY( "except if '-c' command is specified, to force output to console \n");
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Simple example :\n");
|
||||
DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n");
|
||||
DISPLAY( " %s filename\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments can be appended together, or provided independently. For example :\n");
|
||||
DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n");
|
||||
DISPLAY( " %s -f9 filename \n", programName);
|
||||
DISPLAY( " is equivalent to :\n");
|
||||
DISPLAY( " %s -f -9 filename \n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName);
|
||||
DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n");
|
||||
DISPLAY( " generator | %s | consumer \n", programName);
|
||||
#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Warning :\n");
|
||||
DISPLAY( "Legacy arguments take precedence. Therefore : \n");
|
||||
DISPLAY( " %s -hc filename\n", programName);
|
||||
DISPLAY( "means 'compress filename in high compression mode'\n");
|
||||
DISPLAY( "It is not equivalent to :\n");
|
||||
DISPLAY( " %s -h -c filename\n", programName);
|
||||
DISPLAY( "which would display help text and exit\n");
|
||||
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int badusage(void)
|
||||
{
|
||||
DISPLAYLEVEL(1, "Incorrect parameters\n");
|
||||
if (displayLevel >= 1) usage();
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
||||
static void waitEnter(void)
|
||||
{
|
||||
DISPLAY("Press enter to continue...\n");
|
||||
getchar();
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int i,
|
||||
cLevel=0,
|
||||
decode=0,
|
||||
bench=0,
|
||||
filenamesStart=2,
|
||||
legacy_format=0,
|
||||
forceStdout=0,
|
||||
forceCompress=0,
|
||||
main_pause=0;
|
||||
char* input_filename=0;
|
||||
char* output_filename=0;
|
||||
char* dynNameSpace=0;
|
||||
char nullOutput[] = NULL_OUTPUT;
|
||||
char extension[] = LZ4_EXTENSION;
|
||||
int blockSize;
|
||||
|
||||
/* Init */
|
||||
programName = argv[0];
|
||||
LZ4IO_setOverwrite(0);
|
||||
blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
|
||||
|
||||
/* lz4cat behavior */
|
||||
if (!strcmp(programName, LZ4_CAT)) { decode=1; forceStdout=1; output_filename=stdoutmark; displayLevel=1; }
|
||||
|
||||
/* command switches */
|
||||
for(i=1; i<argc; i++)
|
||||
{
|
||||
char* argument = argv[i];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Decode command (note : aggregated commands are allowed) */
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
/* '-' means stdin/stdout */
|
||||
if (argument[1]==0)
|
||||
{
|
||||
if (!input_filename) input_filename=stdinmark;
|
||||
else output_filename=stdoutmark;
|
||||
}
|
||||
|
||||
while (argument[1]!=0)
|
||||
{
|
||||
argument ++;
|
||||
|
||||
#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
|
||||
/* Legacy arguments (-c0, -c1, -hc, -y, -s) */
|
||||
if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */
|
||||
if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } /* -c1 (high compression) */
|
||||
if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } /* -hc (high compression) */
|
||||
if (*argument=='y') { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */
|
||||
if (*argument=='s') { displayLevel=1; continue; } /* -s (silent mode) */
|
||||
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
|
||||
|
||||
if ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
cLevel = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9'))
|
||||
{
|
||||
cLevel *= 10;
|
||||
cLevel += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
argument--;
|
||||
continue;
|
||||
}
|
||||
|
||||
switch(argument[0])
|
||||
{
|
||||
/* Display help */
|
||||
case 'V': DISPLAY(WELCOME_MESSAGE); return 0; /* Version */
|
||||
case 'h': usage_advanced(); return 0;
|
||||
case 'H': usage_advanced(); usage_longhelp(); return 0;
|
||||
|
||||
/* Compression (default) */
|
||||
case 'z': forceCompress = 1; break;
|
||||
|
||||
/* Use Legacy format (ex : Linux kernel compression) */
|
||||
case 'l': legacy_format = 1; blockSize = 8 MB; break;
|
||||
|
||||
/* Decoding */
|
||||
case 'd': decode=1; break;
|
||||
|
||||
/* Force stdout, even if stdout==console */
|
||||
case 'c': forceStdout=1; output_filename=stdoutmark; displayLevel=1; break;
|
||||
|
||||
/* Test integrity */
|
||||
case 't': decode=1; LZ4IO_setOverwrite(1); output_filename=nulmark; break;
|
||||
|
||||
/* Overwrite */
|
||||
case 'f': LZ4IO_setOverwrite(1); break;
|
||||
|
||||
/* Verbose mode */
|
||||
case 'v': displayLevel=4; break;
|
||||
|
||||
/* Quiet mode */
|
||||
case 'q': displayLevel--; break;
|
||||
|
||||
/* keep source file (default anyway, so useless) (for xz/lzma compatibility) */
|
||||
case 'k': break;
|
||||
|
||||
/* Modify Block Properties */
|
||||
case 'B':
|
||||
while (argument[1]!=0)
|
||||
{
|
||||
int exitBlockProperties=0;
|
||||
switch(argument[1])
|
||||
{
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
{
|
||||
int B = argument[1] - '0';
|
||||
blockSize = LZ4IO_setBlockSizeID(B);
|
||||
BMK_SetBlocksize(blockSize);
|
||||
argument++;
|
||||
break;
|
||||
}
|
||||
case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break;
|
||||
case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* currently disables */
|
||||
default : exitBlockProperties=1;
|
||||
}
|
||||
if (exitBlockProperties) break;
|
||||
}
|
||||
break;
|
||||
|
||||
/* Modify Stream properties */
|
||||
case 'S': if (argument[1]=='x') { LZ4IO_setStreamChecksumMode(0); argument++; break; } else { badusage(); }
|
||||
|
||||
/* Benchmark */
|
||||
case 'b': bench=1; break;
|
||||
|
||||
/* Modify Nb Iterations (benchmark only) */
|
||||
case 'i':
|
||||
if ((argument[1] >='1') && (argument[1] <='9'))
|
||||
{
|
||||
int iters = argument[1] - '0';
|
||||
BMK_SetNbIterations(iters);
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
/* Pause at the end (hidden option) */
|
||||
case 'p': main_pause=1; BMK_SetPause(); break;
|
||||
|
||||
/* Specific commands for customized versions */
|
||||
EXTENDED_ARGUMENTS;
|
||||
|
||||
/* Unrecognised command */
|
||||
default : badusage();
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
/* first provided filename is input */
|
||||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
||||
|
||||
/* second provided filename is output */
|
||||
if (!output_filename)
|
||||
{
|
||||
output_filename=argument;
|
||||
if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, WELCOME_MESSAGE);
|
||||
if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
|
||||
|
||||
/* No input filename ==> use stdin */
|
||||
if(!input_filename) { input_filename=stdinmark; }
|
||||
|
||||
/* Check if input or output are defined as console; trigger an error in this case */
|
||||
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
|
||||
|
||||
/* Check if benchmark is selected */
|
||||
if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel);
|
||||
|
||||
/* No output filename ==> try to select one automatically (when possible) */
|
||||
while (!output_filename)
|
||||
{
|
||||
if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */
|
||||
if ((!decode) && !(forceCompress)) /* auto-determine compression or decompression, based on file extension */
|
||||
{
|
||||
size_t l = strlen(input_filename);
|
||||
if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1;
|
||||
}
|
||||
if (!decode) /* compression to file */
|
||||
{
|
||||
size_t l = strlen(input_filename);
|
||||
dynNameSpace = (char*)calloc(1,l+5);
|
||||
output_filename = dynNameSpace;
|
||||
strcpy(output_filename, input_filename);
|
||||
strcpy(output_filename+l, LZ4_EXTENSION);
|
||||
DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
|
||||
break;
|
||||
}
|
||||
/* decompression to file (automatic name will work only if input filename has correct format extension) */
|
||||
{
|
||||
size_t outl;
|
||||
size_t inl = strlen(input_filename);
|
||||
dynNameSpace = (char*)calloc(1,inl+1);
|
||||
output_filename = dynNameSpace;
|
||||
strcpy(output_filename, input_filename);
|
||||
outl = inl;
|
||||
if (inl>4)
|
||||
while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0;
|
||||
if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); }
|
||||
DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
|
||||
}
|
||||
}
|
||||
|
||||
/* Check if output is defined as console; trigger an error in this case */
|
||||
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage();
|
||||
|
||||
/* No warning message in pure pipe mode (stdin + stdout) */
|
||||
if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
|
||||
|
||||
|
||||
/* IO Stream/File */
|
||||
LZ4IO_setNotificationLevel(displayLevel);
|
||||
if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename);
|
||||
else
|
||||
{
|
||||
/* compression is default action */
|
||||
if (legacy_format)
|
||||
{
|
||||
DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
|
||||
LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
|
||||
}
|
||||
else
|
||||
{
|
||||
DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
|
||||
}
|
||||
}
|
||||
|
||||
if (main_pause) waitEnter();
|
||||
free(dynNameSpace);
|
||||
return 0;
|
||||
}
|
||||
669
src/lz4/programs/lz4io.c
Normal file
669
src/lz4/programs/lz4io.c
Normal file
@@ -0,0 +1,669 @@
|
||||
/*
|
||||
LZ4io.c - LZ4 File/Stream Interface
|
||||
Copyright (C) Yann Collet 2011-2014
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
/*
|
||||
Note : this is stand-alone program.
|
||||
It is not part of LZ4 compression library, it is a user code of the LZ4 library.
|
||||
- The license of LZ4 library is BSD.
|
||||
- The license of xxHash library is BSD.
|
||||
- The license of this source file is GPLv2.
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Compiler Options
|
||||
***************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define _CRT_SECURE_NO_WARNINGS
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
#ifdef __GNUC__
|
||||
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
|
||||
# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
|
||||
#endif
|
||||
|
||||
#define _LARGE_FILES /* Large file support on 32-bits AIX */
|
||||
#define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */
|
||||
#define _POSIX_SOURCE 1 /* for fileno() within <stdio.h> on unix */
|
||||
|
||||
|
||||
/****************************
|
||||
* Includes
|
||||
*****************************/
|
||||
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include <time.h> /* clock */
|
||||
#include "lz4io.h"
|
||||
#include "lz4.h" /* still required for legacy format */
|
||||
#include "lz4hc.h" /* still required for legacy format */
|
||||
#include "lz4frame.h"
|
||||
|
||||
|
||||
/****************************
|
||||
* OS-specific Includes
|
||||
*****************************/
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
# include <io.h> /* _setmode, _isatty */
|
||||
# ifdef __MINGW32__
|
||||
int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
|
||||
# endif
|
||||
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
|
||||
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
||||
#else
|
||||
# include <unistd.h> /* isatty */
|
||||
# define SET_BINARY_MODE(file)
|
||||
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
|
||||
#endif
|
||||
|
||||
|
||||
/****************************
|
||||
* Constants
|
||||
*****************************/
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define _1BIT 0x01
|
||||
#define _2BITS 0x03
|
||||
#define _3BITS 0x07
|
||||
#define _4BITS 0x0F
|
||||
#define _8BITS 0xFF
|
||||
|
||||
#define MAGICNUMBER_SIZE 4
|
||||
#define LZ4S_MAGICNUMBER 0x184D2204
|
||||
#define LZ4S_SKIPPABLE0 0x184D2A50
|
||||
#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0
|
||||
#define LEGACY_MAGICNUMBER 0x184C2102
|
||||
|
||||
#define CACHELINE 64
|
||||
#define LEGACY_BLOCKSIZE (8 MB)
|
||||
#define MIN_STREAM_BUFSIZE (192 KB)
|
||||
#define LZ4S_BLOCKSIZEID_DEFAULT 7
|
||||
#define LZ4S_CHECKSUM_SEED 0
|
||||
#define LZ4S_EOS 0
|
||||
#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1)
|
||||
|
||||
|
||||
/**************************************
|
||||
* Macros
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
|
||||
if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=4)) \
|
||||
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel>=4) fflush(stdout); } }
|
||||
static const unsigned refreshRate = 150;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
|
||||
/**************************************
|
||||
* Local Parameters
|
||||
***************************************/
|
||||
static int displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
|
||||
static int overwrite = 1;
|
||||
static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
|
||||
static int blockChecksum = 0;
|
||||
static int streamChecksum = 1;
|
||||
static int blockIndependence = 1;
|
||||
|
||||
static const int minBlockSizeID = 4;
|
||||
static const int maxBlockSizeID = 7;
|
||||
|
||||
|
||||
/**************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#define DEBUG 0
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, "\n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/**************************************
|
||||
* Version modifiers
|
||||
***************************************/
|
||||
#define EXTENDED_ARGUMENTS
|
||||
#define EXTENDED_HELP
|
||||
#define EXTENDED_FORMAT
|
||||
#define DEFAULT_COMPRESSOR compress_file
|
||||
#define DEFAULT_DECOMPRESSOR decodeLZ4S
|
||||
|
||||
|
||||
/* ************************************************** */
|
||||
/* ****************** Parameters ******************** */
|
||||
/* ************************************************** */
|
||||
|
||||
/* Default setting : overwrite = 1; return : overwrite mode (0/1) */
|
||||
int LZ4IO_setOverwrite(int yes)
|
||||
{
|
||||
overwrite = (yes!=0);
|
||||
return overwrite;
|
||||
}
|
||||
|
||||
/* blockSizeID : valid values : 4-5-6-7 */
|
||||
int LZ4IO_setBlockSizeID(int bsid)
|
||||
{
|
||||
static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB };
|
||||
if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1;
|
||||
globalBlockSizeId = bsid;
|
||||
return blockSizeTable[globalBlockSizeId-minBlockSizeID];
|
||||
}
|
||||
|
||||
int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode)
|
||||
{
|
||||
blockIndependence = (blockMode == LZ4IO_blockIndependent);
|
||||
return blockIndependence;
|
||||
}
|
||||
|
||||
/* Default setting : no checksum */
|
||||
int LZ4IO_setBlockChecksumMode(int xxhash)
|
||||
{
|
||||
blockChecksum = (xxhash != 0);
|
||||
return blockChecksum;
|
||||
}
|
||||
|
||||
/* Default setting : checksum enabled */
|
||||
int LZ4IO_setStreamChecksumMode(int xxhash)
|
||||
{
|
||||
streamChecksum = (xxhash != 0);
|
||||
return streamChecksum;
|
||||
}
|
||||
|
||||
/* Default setting : 0 (no notification) */
|
||||
int LZ4IO_setNotificationLevel(int level)
|
||||
{
|
||||
displayLevel = level;
|
||||
return displayLevel;
|
||||
}
|
||||
|
||||
static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious)
|
||||
{
|
||||
clock_t nCurrent = clock();
|
||||
unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
|
||||
return nSpan;
|
||||
}
|
||||
|
||||
|
||||
/* ************************************************************************ */
|
||||
/* ********************** LZ4 File / Pipe compression ********************* */
|
||||
/* ************************************************************************ */
|
||||
|
||||
static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
|
||||
static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }
|
||||
|
||||
|
||||
static int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput)
|
||||
{
|
||||
|
||||
if (!strcmp (input_filename, stdinmark))
|
||||
{
|
||||
DISPLAYLEVEL(4,"Using stdin for input\n");
|
||||
*pfinput = stdin;
|
||||
SET_BINARY_MODE(stdin);
|
||||
}
|
||||
else
|
||||
{
|
||||
*pfinput = fopen(input_filename, "rb");
|
||||
}
|
||||
|
||||
if (!strcmp (output_filename, stdoutmark))
|
||||
{
|
||||
DISPLAYLEVEL(4,"Using stdout for output\n");
|
||||
*pfoutput = stdout;
|
||||
SET_BINARY_MODE(stdout);
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Check if destination file already exists */
|
||||
*pfoutput=0;
|
||||
if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" );
|
||||
if (*pfoutput!=0)
|
||||
{
|
||||
fclose(*pfoutput);
|
||||
if (!overwrite)
|
||||
{
|
||||
char ch;
|
||||
DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
|
||||
DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
|
||||
if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */
|
||||
ch = (char)getchar();
|
||||
if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
|
||||
}
|
||||
}
|
||||
*pfoutput = fopen( output_filename, "wb" );
|
||||
}
|
||||
|
||||
if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
|
||||
if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/***************************************
|
||||
* Legacy Compression
|
||||
* *************************************/
|
||||
|
||||
/* unoptimized version; solves endianess & alignment issues */
|
||||
static void LZ4IO_writeLE32 (void* p, unsigned value32)
|
||||
{
|
||||
unsigned char* dstPtr = p;
|
||||
dstPtr[0] = (unsigned char)value32;
|
||||
dstPtr[1] = (unsigned char)(value32 >> 8);
|
||||
dstPtr[2] = (unsigned char)(value32 >> 16);
|
||||
dstPtr[3] = (unsigned char)(value32 >> 24);
|
||||
}
|
||||
|
||||
/* LZ4IO_compressFilename_Legacy :
|
||||
* This function is intentionally "hidden" (not published in .h)
|
||||
* It generates compressed streams using the old 'legacy' format */
|
||||
int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel)
|
||||
{
|
||||
int (*compressionFunction)(const char*, char*, int);
|
||||
unsigned long long filesize = 0;
|
||||
unsigned long long compressedfilesize = MAGICNUMBER_SIZE;
|
||||
char* in_buff;
|
||||
char* out_buff;
|
||||
FILE* finput;
|
||||
FILE* foutput;
|
||||
clock_t start, end;
|
||||
size_t sizeCheck;
|
||||
|
||||
|
||||
/* Init */
|
||||
start = clock();
|
||||
if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;
|
||||
|
||||
get_fileHandle(input_filename, output_filename, &finput, &foutput);
|
||||
if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3;
|
||||
|
||||
/* Allocate Memory */
|
||||
in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
|
||||
out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
|
||||
if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
|
||||
|
||||
/* Write Archive Header */
|
||||
LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
|
||||
sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
|
||||
if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header");
|
||||
|
||||
/* Main Loop */
|
||||
while (1)
|
||||
{
|
||||
unsigned int outSize;
|
||||
/* Read Block */
|
||||
int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
|
||||
if( inSize<=0 ) break;
|
||||
filesize += inSize;
|
||||
|
||||
/* Compress Block */
|
||||
outSize = compressionFunction(in_buff, out_buff+4, inSize);
|
||||
compressedfilesize += outSize+4;
|
||||
DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
|
||||
|
||||
/* Write Block */
|
||||
LZ4IO_writeLE32(out_buff, outSize);
|
||||
sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);
|
||||
if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block");
|
||||
}
|
||||
|
||||
/* Status */
|
||||
end = clock();
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
|
||||
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
|
||||
{
|
||||
double seconds = (double)(end - start)/CLOCKS_PER_SEC;
|
||||
DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
|
||||
}
|
||||
|
||||
/* Close & Free */
|
||||
free(in_buff);
|
||||
free(out_buff);
|
||||
fclose(finput);
|
||||
fclose(foutput);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/***********************************************
|
||||
* Compression using Frame format
|
||||
* ********************************************/
|
||||
|
||||
int LZ4IO_compressFilename(char* input_filename, char* output_filename, int compressionLevel)
|
||||
{
|
||||
unsigned long long filesize = 0;
|
||||
unsigned long long compressedfilesize = 0;
|
||||
char* in_buff;
|
||||
char* out_buff;
|
||||
FILE* finput;
|
||||
FILE* foutput;
|
||||
clock_t start, end;
|
||||
int blockSize;
|
||||
size_t sizeCheck, headerSize, readSize, outBuffSize;
|
||||
LZ4F_compressionContext_t ctx;
|
||||
LZ4F_errorCode_t errorCode;
|
||||
LZ4F_preferences_t prefs = {0};
|
||||
|
||||
|
||||
/* Init */
|
||||
start = clock();
|
||||
if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3;
|
||||
errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
|
||||
get_fileHandle(input_filename, output_filename, &finput, &foutput);
|
||||
blockSize = LZ4S_GetBlockSize_FromBlockId (globalBlockSizeId);
|
||||
|
||||
/* Set compression parameters */
|
||||
prefs.autoFlush = 1;
|
||||
prefs.compressionLevel = compressionLevel;
|
||||
prefs.frameInfo.blockMode = blockIndependence;
|
||||
prefs.frameInfo.blockSizeID = globalBlockSizeId;
|
||||
prefs.frameInfo.contentChecksumFlag = streamChecksum;
|
||||
|
||||
/* Allocate Memory */
|
||||
in_buff = (char*)malloc(blockSize);
|
||||
outBuffSize = LZ4F_compressBound(blockSize, &prefs);
|
||||
out_buff = (char*)malloc(outBuffSize);
|
||||
if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
|
||||
|
||||
/* Write Archive Header */
|
||||
headerSize = LZ4F_compressBegin(ctx, out_buff, outBuffSize, &prefs);
|
||||
if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
|
||||
sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
|
||||
if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header");
|
||||
compressedfilesize += headerSize;
|
||||
|
||||
/* read first block */
|
||||
readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
|
||||
filesize += readSize;
|
||||
|
||||
/* Main Loop */
|
||||
while (readSize>0)
|
||||
{
|
||||
size_t outSize;
|
||||
|
||||
/* Compress Block */
|
||||
outSize = LZ4F_compressUpdate(ctx, out_buff, outBuffSize, in_buff, readSize, NULL);
|
||||
if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize));
|
||||
compressedfilesize += outSize;
|
||||
DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
|
||||
|
||||
/* Write Block */
|
||||
sizeCheck = fwrite(out_buff, 1, outSize, foutput);
|
||||
if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block");
|
||||
|
||||
/* Read next block */
|
||||
readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
|
||||
filesize += readSize;
|
||||
}
|
||||
|
||||
/* End of Stream mark */
|
||||
headerSize = LZ4F_compressEnd(ctx, out_buff, outBuffSize, NULL);
|
||||
if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
|
||||
|
||||
sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
|
||||
if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream");
|
||||
compressedfilesize += headerSize;
|
||||
|
||||
/* Close & Free */
|
||||
free(in_buff);
|
||||
free(out_buff);
|
||||
fclose(finput);
|
||||
fclose(foutput);
|
||||
errorCode = LZ4F_freeCompressionContext(ctx);
|
||||
if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
|
||||
|
||||
/* Final Status */
|
||||
end = clock();
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
|
||||
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
|
||||
{
|
||||
double seconds = (double)(end - start)/CLOCKS_PER_SEC;
|
||||
DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* ********************************************************************* */
|
||||
/* ********************** LZ4 File / Stream decoding ******************* */
|
||||
/* ********************************************************************* */
|
||||
|
||||
static unsigned LZ4IO_readLE32 (const void* s)
|
||||
{
|
||||
const unsigned char* srcPtr = s;
|
||||
unsigned value32 = srcPtr[0];
|
||||
value32 += (srcPtr[1]<<8);
|
||||
value32 += (srcPtr[2]<<16);
|
||||
value32 += (srcPtr[3]<<24);
|
||||
return value32;
|
||||
}
|
||||
|
||||
static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
|
||||
{
|
||||
unsigned long long filesize = 0;
|
||||
char* in_buff;
|
||||
char* out_buff;
|
||||
|
||||
/* Allocate Memory */
|
||||
in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
|
||||
out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
|
||||
if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
|
||||
|
||||
/* Main Loop */
|
||||
while (1)
|
||||
{
|
||||
int decodeSize;
|
||||
size_t sizeCheck;
|
||||
unsigned int blockSize;
|
||||
|
||||
/* Block Size */
|
||||
sizeCheck = fread(in_buff, 1, 4, finput);
|
||||
if (sizeCheck==0) break; /* Nothing to read : file read is completed */
|
||||
blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
|
||||
if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE))
|
||||
{ /* Cannot read next block : maybe new stream ? */
|
||||
fseek(finput, -4, SEEK_CUR);
|
||||
break;
|
||||
}
|
||||
|
||||
/* Read Block */
|
||||
sizeCheck = fread(in_buff, 1, blockSize, finput);
|
||||
|
||||
/* Decode Block */
|
||||
decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
|
||||
if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !");
|
||||
filesize += decodeSize;
|
||||
|
||||
/* Write Block */
|
||||
sizeCheck = fwrite(out_buff, 1, decodeSize, foutput);
|
||||
if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n");
|
||||
}
|
||||
|
||||
/* Free */
|
||||
free(in_buff);
|
||||
free(out_buff);
|
||||
|
||||
return filesize;
|
||||
}
|
||||
|
||||
|
||||
static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
|
||||
{
|
||||
unsigned long long filesize = 0;
|
||||
char* inBuff;
|
||||
char* outBuff;
|
||||
# define HEADERMAX 20
|
||||
char headerBuff[HEADERMAX];
|
||||
size_t sizeCheck, nextToRead, outBuffSize, inBuffSize;
|
||||
LZ4F_decompressionContext_t ctx;
|
||||
LZ4F_errorCode_t errorCode;
|
||||
LZ4F_frameInfo_t frameInfo;
|
||||
|
||||
/* init */
|
||||
errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode)) EXM_THROW(60, "Allocation error : can't create context : %s", LZ4F_getErrorName(errorCode));
|
||||
LZ4IO_writeLE32(headerBuff, LZ4S_MAGICNUMBER); /* regenerated here, as it was already read from finput */
|
||||
|
||||
/* Decode stream descriptor */
|
||||
outBuffSize = 0; inBuffSize = 0; sizeCheck = MAGICNUMBER_SIZE;
|
||||
nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL);
|
||||
if (LZ4F_isError(nextToRead)) EXM_THROW(61, "Decompression error : %s", LZ4F_getErrorName(nextToRead));
|
||||
if (nextToRead > HEADERMAX) EXM_THROW(62, "Header too large (%i>%i)", (int)nextToRead, HEADERMAX);
|
||||
sizeCheck = fread(headerBuff, 1, nextToRead, finput);
|
||||
if (sizeCheck!=nextToRead) EXM_THROW(63, "Read error ");
|
||||
nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL);
|
||||
errorCode = LZ4F_getFrameInfo(ctx, &frameInfo, NULL, &inBuffSize);
|
||||
if (LZ4F_isError(errorCode)) EXM_THROW(64, "can't decode frame header : %s", LZ4F_getErrorName(errorCode));
|
||||
|
||||
/* Allocate Memory */
|
||||
outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID);
|
||||
inBuffSize = outBuffSize + 4;
|
||||
inBuff = (char*)malloc(inBuffSize);
|
||||
outBuff = (char*)malloc(outBuffSize);
|
||||
if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory");
|
||||
|
||||
/* Main Loop */
|
||||
while (nextToRead != 0)
|
||||
{
|
||||
size_t decodedBytes = outBuffSize;
|
||||
|
||||
/* Read Block */
|
||||
sizeCheck = fread(inBuff, 1, nextToRead, finput);
|
||||
if (sizeCheck!=nextToRead) EXM_THROW(66, "Read error ");
|
||||
|
||||
/* Decode Block */
|
||||
errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, inBuff, &sizeCheck, NULL);
|
||||
if (LZ4F_isError(errorCode)) EXM_THROW(67, "Decompression error : %s", LZ4F_getErrorName(errorCode));
|
||||
if (sizeCheck!=nextToRead) EXM_THROW(67, "Synchronization error");
|
||||
nextToRead = errorCode;
|
||||
filesize += decodedBytes;
|
||||
|
||||
/* Write Block */
|
||||
sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
|
||||
if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n");
|
||||
}
|
||||
|
||||
/* Free */
|
||||
free(inBuff);
|
||||
free(outBuff);
|
||||
errorCode = LZ4F_freeDecompressionContext(ctx);
|
||||
if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
|
||||
|
||||
return filesize;
|
||||
}
|
||||
|
||||
|
||||
#define ENDOFSTREAM ((unsigned long long)-1)
|
||||
static unsigned long long selectDecoder( FILE* finput, FILE* foutput)
|
||||
{
|
||||
unsigned char U32store[MAGICNUMBER_SIZE];
|
||||
unsigned magicNumber, size;
|
||||
int errorNb;
|
||||
size_t nbReadBytes;
|
||||
|
||||
/* Check Archive Header */
|
||||
nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput);
|
||||
if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */
|
||||
if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
|
||||
magicNumber = LZ4IO_readLE32(U32store); /* Little Endian format */
|
||||
if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; /* fold skippable magic numbers */
|
||||
|
||||
switch(magicNumber)
|
||||
{
|
||||
case LZ4S_MAGICNUMBER:
|
||||
return DEFAULT_DECOMPRESSOR(finput, foutput);
|
||||
case LEGACY_MAGICNUMBER:
|
||||
DISPLAYLEVEL(4, "Detected : Legacy format \n");
|
||||
return decodeLegacyStream(finput, foutput);
|
||||
case LZ4S_SKIPPABLE0:
|
||||
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
|
||||
nbReadBytes = fread(U32store, 1, 4, finput);
|
||||
if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");
|
||||
size = LZ4IO_readLE32(U32store); /* Little Endian format */
|
||||
errorNb = fseek(finput, size, SEEK_CUR);
|
||||
if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");
|
||||
return selectDecoder(finput, foutput);
|
||||
EXTENDED_FORMAT;
|
||||
default:
|
||||
if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */
|
||||
DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");
|
||||
return ENDOFSTREAM;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int LZ4IO_decompressFilename(char* input_filename, char* output_filename)
|
||||
{
|
||||
unsigned long long filesize = 0, decodedSize=0;
|
||||
FILE* finput;
|
||||
FILE* foutput;
|
||||
clock_t start, end;
|
||||
|
||||
|
||||
/* Init */
|
||||
start = clock();
|
||||
get_fileHandle(input_filename, output_filename, &finput, &foutput);
|
||||
|
||||
/* Loop over multiple streams */
|
||||
do
|
||||
{
|
||||
decodedSize = selectDecoder(finput, foutput);
|
||||
if (decodedSize != ENDOFSTREAM)
|
||||
filesize += decodedSize;
|
||||
} while (decodedSize != ENDOFSTREAM);
|
||||
|
||||
/* Final Status */
|
||||
end = clock();
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
|
||||
{
|
||||
double seconds = (double)(end - start)/CLOCKS_PER_SEC;
|
||||
DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
|
||||
}
|
||||
|
||||
/* Close */
|
||||
fclose(finput);
|
||||
fclose(foutput);
|
||||
|
||||
/* Error status = OK */
|
||||
return 0;
|
||||
}
|
||||
|
||||
77
src/lz4/programs/lz4io.h
Normal file
77
src/lz4/programs/lz4io.h
Normal file
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
LZ4io.h - LZ4 File/Stream Interface
|
||||
Copyright (C) Yann Collet 2011-2014
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
/*
|
||||
Note : this is stand-alone program.
|
||||
It is not part of LZ4 compression library, it is a user code of the LZ4 library.
|
||||
- The license of LZ4 library is BSD.
|
||||
- The license of xxHash library is BSD.
|
||||
- The license of this source file is GPLv2.
|
||||
*/
|
||||
|
||||
|
||||
/* ************************************************** */
|
||||
/* Special input/output values */
|
||||
/* ************************************************** */
|
||||
#define NULL_OUTPUT "null"
|
||||
static char stdinmark[] = "stdin";
|
||||
static char stdoutmark[] = "stdout";
|
||||
#ifdef _WIN32
|
||||
static char nulmark[] = "nul";
|
||||
#else
|
||||
static char nulmark[] = "/dev/null";
|
||||
#endif
|
||||
|
||||
|
||||
/* ************************************************** */
|
||||
/* ****************** Functions ********************* */
|
||||
/* ************************************************** */
|
||||
|
||||
int LZ4IO_compressFilename (char* input_filename, char* output_filename, int compressionlevel);
|
||||
int LZ4IO_decompressFilename(char* input_filename, char* output_filename);
|
||||
|
||||
|
||||
/* ************************************************** */
|
||||
/* ****************** Parameters ******************** */
|
||||
/* ************************************************** */
|
||||
|
||||
/* Default setting : overwrite = 1;
|
||||
return : overwrite mode (0/1) */
|
||||
int LZ4IO_setOverwrite(int yes);
|
||||
|
||||
/* blockSizeID : valid values : 4-5-6-7
|
||||
return : -1 if error, blockSize if OK */
|
||||
int LZ4IO_setBlockSizeID(int blockSizeID);
|
||||
|
||||
/* Default setting : independent blocks */
|
||||
typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t;
|
||||
int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode);
|
||||
|
||||
/* Default setting : no checksum */
|
||||
int LZ4IO_setBlockChecksumMode(int xxhash);
|
||||
|
||||
/* Default setting : checksum enabled */
|
||||
int LZ4IO_setStreamChecksumMode(int xxhash);
|
||||
|
||||
/* Default setting : 0 (no notification) */
|
||||
int LZ4IO_setNotificationLevel(int level);
|
||||
Reference in New Issue
Block a user