Merge pull request #273 from Cyan4973/dev

v0.7.2
Cyan4973 · Oct 8, 2019 · e2f4695 · e2f4695
2 parents d7f47bc + 1ea98d6
commit e2f4695
Show file tree

Hide file tree

Showing 13 changed files with 1,200 additions and 1,022 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,3 @@
-# compilation chain
-.clang_complete
-
 # objects
 *.o
 *.s
@@ -18,6 +15,9 @@ xxhsum_privateXXH
 xxhsum_inlinedXXH
 xxhsum_inlinedXXH.exe
 
+# compilation chain
+.clang_complete
+
 # Mac OS-X artefacts
 *.dSYM
 .DS_Store
@@ -32,3 +32,10 @@ build*/
 
 # project managers artifacts
 .projectile
+
+# analyzer artifacts
+infer-out
+
+# test artifacts
+.test*
+tmp*
diff --git a/.travis.yml b/.travis.yml
@@ -40,36 +40,38 @@ matrix:
         # arm (32-bit)
         - CC=arm-linux-gnueabi-gcc CPPFLAGS=-DXXH_VECTOR=0 LDFLAGS=-static RUN_ENV=qemu-arm-static make check   # Scalar code path
         - make clean
-        # Note : the following test (ARM 32-bit + NEON) is disabled for the time being.
-        # I haven't yet found a way to make it link on Travis CI using gcc cross-compilation.
-        # NEON code path is fortunately validated through `aarch64` below.
-        # - CC=arm-linux-gnueabi-gcc CPPFLAGS=-DXXH_VECTOR=3 CFLAGS="-O3 -march=armv7-a -mfloat-abi=hard -mfpu=neon" LDFLAGS=-static RUN_ENV=qemu-arm-static make check   # NEON code path
+        # NEON (32-bit)
+        - CC=arm-linux-gnueabi-gcc CPPFLAGS=-DXXH_VECTOR=3 CFLAGS="-O3 -march=armv7-a -fPIC -mfloat-abi=softfp -mfpu=neon-vfpv4" LDFLAGS=-static RUN_ENV=qemu-arm-static make check   # NEON code path
         - make clean
         # aarch64
         - CC=aarch64-linux-gnu-gcc CPPFLAGS=-DXXH_VECTOR=0 LDFLAGS=-static RUN_ENV=qemu-aarch64-static make check   # Scalar code path
         - make clean
         - CC=aarch64-linux-gnu-gcc CPPFLAGS=-DXXH_VECTOR=3 LDFLAGS=-static RUN_ENV=qemu-aarch64-static make check   # NEON code path
         - make clean
-
-    - name: PowerPC + PPC64 compilation and consistency checks (Trusty)
-      dist: trusty
+    # We need Bionic here because the QEMU versions shipped in the older repos
+    # do not support POWER8 emulation, and compiling QEMU from source is a pain.
+    - name: PowerPC + PPC64 compilation and consistency checks (Bionic)
+      dist: bionic
       install:
-        - sudo apt-get install -qq qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu
+        - sudo apt-get install -qq
+            qemu-system-ppc
+            qemu-user-static
+            gcc-powerpc-linux-gnu
+            gcc-powerpc64-linux-gnu
+            libc6-dev-powerpc-cross
+            libc6-dev-ppc64-cross
+            gcc-powerpc64le-linux-gnu
+            libc6-dev-ppc64el-cross
       script:
-        - CC=powerpc-linux-gnu-gcc RUN_ENV=qemu-ppc-static CPPFLAGS=-m32 LDFLAGS=-static make check   # Scalar code path
+        - CC=powerpc-linux-gnu-gcc RUN_ENV=qemu-ppc-static LDFLAGS=-static make check   # Scalar code path
+        - make clean
+        - CC=powerpc64-linux-gnu-gcc RUN_ENV=qemu-ppc64-static CPPFLAGS=-DXXH_VECTOR=0 CFLAGS="-O3" LDFLAGS="-static -m64" make check   # Scalar code path
         - make clean
-        - CC=powerpc-linux-gnu-gcc RUN_ENV=qemu-ppc64-static CPPFLAGS=-DXXH_VECTOR=0 CFLAGS="-O3 -m64" LDFLAGS="-static -m64" make check   # Scalar code path
+        # VSX code
+        - CC=powerpc64-linux-gnu-gcc RUN_ENV="qemu-ppc64-static -cpu power8" CFLAGS="-O3 -maltivec -mvsx -mcpu=power8 -mpower8-vector" LDFLAGS="-static -m64" make check   # Auto code path
         - make clean
-        # Note : the following test will fail
-        # because it will trigger the VSX code path
-        # while missing the `vec_revb()` instruction, hence compilation will fail.
-        # to be fixed one day.
-        # - CC=powerpc-linux-gnu-gcc RUN_ENV=qemu-ppc64-static CFLAGS="-O3 -m64 -maltivec -mvsx" LDFLAGS="-static -m64" make check   # Auto code path
+        - CC=powerpc64le-linux-gnu-gcc RUN_ENV="qemu-ppc64le-static -cpu power8" CPPFLAGS=-DXXH_VECTOR=4 CFLAGS="-O3 -maltivec -mvsx -mpower8-vector -mcpu=power8" LDFLAGS="-static" make check   # VSX code path
         - make clean
-        # Note : the following test does not work on Ubuntu Trusty.
-        # It requires support for `power9` instruction due to `vec_revb()`
-        # which is missing from the powerpc-gcc compiler version shipped with Ubuntu Trusty (max `power8`)
-        # - CC=powerpc-linux-gnu-gcc RUN_ENV=qemu-ppc64-static CPPFLAGS=-DXXH_VECTOR=4 CFLAGS="-O3 -m64 -maltivec -mvsx -mcpu=power8" LDFLAGS="-static -m64" make check   # VSX code path
 
     - name: cmake
       script:

diff --git a/LICENSE b/LICENSE
@@ -1,7 +1,9 @@
 xxHash Library
-Copyright (c) 2012-2014, Yann Collet
+Copyright (c) 2012-present, Yann Collet
 All rights reserved.
 
+BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
@@ -22,3 +24,25 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------
+
+xxhsum command line interface
+Copyright (c) 2013-present, Yann Collet
+All rights reserved.
+
+GPL v2 License
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
diff --git a/Makefile b/Makefile
@@ -38,7 +38,7 @@ DEBUGFLAGS+=-Wall -Wextra -Wconversion -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
             -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls -Wstrict-overflow=5
+            -Wredundant-decls -Wstrict-overflow=2
 CFLAGS += $(DEBUGFLAGS)
 FLAGS   = $(CFLAGS) $(CPPFLAGS) $(MOREFLAGS)
 XXHSUM_VERSION = $(LIBVER)
@@ -160,14 +160,18 @@ check: xxhsum   ## basic tests for xxhsum CLI, set RUN_ENV for emulated environm
 	$(RUN_ENV) ./xxhsum -bi1
 	# file bench
 	$(RUN_ENV) ./xxhsum -bi1 xxhash.c
+	# 32-bit
+	$(RUN_ENV) ./xxhsum -H0 xxhash.c
+	# 128-bit
+	$(RUN_ENV) ./xxhsum -H2 xxhash.c
+	# request incorrect variant
+	$(RUN_ENV) ./xxhsum -H9 xxhash.c ; test $$? -eq 1
 
 
 .PHONY: test-mem
 VALGRIND = valgrind --leak-check=yes --error-exitcode=1
-test-mem: xxhsum  ## valgrind tests for xxhsum CLI, looking for memory leaks
-	$(VALGRIND) ./xxhsum -bi1 xxhash.c
-	$(VALGRIND) ./xxhsum -H0  xxhash.c
-	$(VALGRIND) ./xxhsum -H1  xxhash.c
+test-mem: RUN_ENV = $(VALGRIND)
+test-mem: xxhsum check
 
 .PHONY: test32
 test32: clean xxhsum32
@@ -177,13 +181,17 @@ test32: clean xxhsum32
 .PHONY: test-xxhsum-c
 test-xxhsum-c: xxhsum
 	# xxhsum to/from pipe
-	./xxhsum lib* | ./xxhsum -c -
-	./xxhsum -H0 lib* | ./xxhsum -c -
+	./xxhsum xxh* | ./xxhsum -c -
+	./xxhsum -H0 xxh* | ./xxhsum -c -
+	# xxhsum -q does not display "Loading" message into stderr (#251)
+	! ./xxhsum -q xxh* 2>&1 | grep Loading
 	# xxhsum to/from file, shell redirection
-	./xxhsum lib* > .test.xxh64
-	./xxhsum -H0 lib* > .test.xxh32
+	./xxhsum xxh* > .test.xxh64
+	./xxhsum -H0 xxh* > .test.xxh32
+	./xxhsum -H2 xxh* > .test.xxh128
 	./xxhsum -c .test.xxh64
 	./xxhsum -c .test.xxh32
+	./xxhsum -c .test.xxh128
 	./xxhsum -c < .test.xxh64
 	./xxhsum -c < .test.xxh32
 	# xxhsum -c warns improperly format lines.
@@ -195,7 +203,7 @@ test-xxhsum-c: xxhsum
 	# Expects "FAILED open or read"
 	echo "0000000000000000  test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1
 	echo "00000000  test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1
-	@$(RM) -f .test.xxh32 .test.xxh64
+	@$(RM) -f .test.xxh32 .test.xxh64 .test.xxh128
 
 .PHONY: armtest
 armtest: clean
@@ -248,8 +256,8 @@ namespaceTest:  ## ensure XXH_NAMESPACE redefines all public symbols
 
 MD2ROFF ?= ronn
 MD2ROFF_FLAGS ?= --roff --warnings --manual="User Commands" --organization="xxhsum $(XXHSUM_VERSION)"
-xxhsum.1: xxhsum.1.md
-	cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
+xxhsum.1: xxhsum.1.md xxhash.h
+	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
 
 .PHONY: man
 man: xxhsum.1  ## generate man page from markdown source
@@ -332,10 +340,12 @@ install: lib xxhsum  ## install libraries, CLI, links and man page
 	@$(INSTALL_PROGRAM) xxhsum $(DESTDIR)$(BINDIR)/xxhsum
 	@ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh32sum
 	@ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh64sum
+	@ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh128sum
 	@echo Installing man pages
 	@$(INSTALL_DATA) xxhsum.1 $(DESTDIR)$(MANDIR)/xxhsum.1
 	@ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh32sum.1
 	@ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh64sum.1
+	@ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh128sum.1
 	@echo xxhash installation completed
 
 .PHONY: uninstall
@@ -347,9 +357,11 @@ uninstall:  ## uninstall libraries, CLI, links and man page
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/xxhash.h
 	@$(RM) $(DESTDIR)$(BINDIR)/xxh32sum
 	@$(RM) $(DESTDIR)$(BINDIR)/xxh64sum
+	@$(RM) $(DESTDIR)$(BINDIR)/xxh128sum
 	@$(RM) $(DESTDIR)$(BINDIR)/xxhsum
 	@$(RM) $(DESTDIR)$(MANDIR)/xxh32sum.1
 	@$(RM) $(DESTDIR)$(MANDIR)/xxh64sum.1
+	@$(RM) $(DESTDIR)$(MANDIR)/xxh128sum.1
 	@$(RM) $(DESTDIR)$(MANDIR)/xxhsum.1
 	@echo xxhsum successfully uninstalled
 

diff --git a/README.md b/README.md
@@ -20,20 +20,22 @@ The benchmark uses SMHasher speed test, compiled with Visual 2010 on a Windows S
 The reference system uses a Core 2 Duo @3GHz
 
 
-| Name          |   Speed  | Quality | Author           |
-|---------------|----------|:-------:|------------------|
-| [xxHash]      | 5.4 GB/s |   10    | Y.C.             |
-| MurmurHash 3a | 2.7 GB/s |   10    | Austin Appleby   |
-| SBox          | 1.4 GB/s |    9    | Bret Mulvey      |
-| Lookup3       | 1.2 GB/s |    9    | Bob Jenkins      |
-| CityHash64    | 1.05 GB/s|   10    | Pike & Alakuijala|
-| FNV           | 0.55 GB/s|    5    | Fowler, Noll, Vo |
-| CRC32         | 0.43 GB/s|    9    |                  |
-| MD5-32        | 0.33 GB/s|   10    | Ronald L.Rivest  |
-| SHA1-32       | 0.28 GB/s|   10    |                  |
+| Name          |   Speed     | Quality | Author            |
+|---------------|-------------|:-------:|-------------------|
+| [xxHash]      | 5.4 GB/s    |   10    | Y.C.              |
+| MurmurHash 3a | 2.7 GB/s    |   10    | Austin Appleby    |
+| SBox          | 1.4 GB/s    |    9    | Bret Mulvey       |
+| Lookup3       | 1.2 GB/s    |    9    | Bob Jenkins       |
+| CityHash64    | 1.05 GB/s   |   10    | Pike & Alakuijala |
+| FNV           | 0.55 GB/s   |    5    | Fowler, Noll, Vo  |
+| CRC32         | 0.43 GB/s † |    9    |                   |
+| MD5-32        | 0.33 GB/s   |   10    | Ronald L.Rivest   |
+| SHA1-32       | 0.28 GB/s   |   10    |                   |
 
 [xxHash]: http://www.xxhash.com
 
+Note †: SMHasher's CRC32 implementation is known to be slow. Faster implementations exist.
+
 Q.Score is a measure of quality of the hash function.
 It depends on successfully passing SMHasher test set.
 10 is a perfect score.
@@ -104,21 +106,18 @@ Calling xxhash 64-bit variant from a C program :
 ```C
 #include "xxhash.h"
 
-unsigned long long calcul_hash(const void* buffer, size_t length)
-{
-    unsigned long long const seed = 0;   /* or any other value */
-    unsigned long long const hash = XXH64(buffer, length, seed);
-    return hash;
+    (...)
+    XXH64_hash_t hash = XXH64(buffer, size, seed);
 }
 ```
 
-Using streaming variant is more involved, but makes it possible to provide data in multiple rounds :
+Using streaming variant is more involved, but makes it possible to provide data incrementally :
 ```C
 #include "stdlib.h"   /* abort() */
 #include "xxhash.h"
 
 
-unsigned long long calcul_hash_streaming(someCustomType handler)
+XXH64_hash_t calcul_hash_streaming(FileHandler fh)
 {
     /* create a hash state */
     XXH64_state_t* const state = XXH64_createState();
@@ -129,28 +128,26 @@ unsigned long long calcul_hash_streaming(someCustomType handler)
     if (buffer==NULL) abort();
 
     /* Initialize state with selected seed */
-    unsigned long long const seed = 0;   /* or any other value */
-    XXH_errorcode const resetResult = XXH64_reset(state, seed);
-    if (resetResult == XXH_ERROR) abort();
+    XXH64_hash_t const seed = 0;   /* or any other value */
+    if (XXH64_reset(state, seed) == XXH_ERROR) abort();
 
     /* Feed the state with input data, any size, any number of times */
     (...)
     while ( /* any condition */ ) {
-        size_t const length = get_more_data(buffer, bufferSize, handler);   
-        XXH_errorcode const updateResult = XXH64_update(state, buffer, length);
-        if (updateResult == XXH_ERROR) abort();
+        size_t const length = get_more_data(buffer, bufferSize, fh);   
+        if (XXH64_update(state, buffer, length) == XXH_ERROR) abort();
         (...)
     }
     (...)
 
     /* Get the hash */
     XXH64_hash_t const hash = XXH64_digest(state);
 
-    /* State can then be re-used; in this example, it is simply freed  */
+    /* State can be re-used; in this example, it is simply freed  */
     free(buffer);
     XXH64_freeState(state);
 
-    return (unsigned long long)hash;
+    return hash;
 }
 ```
 
@@ -167,9 +164,10 @@ as can be observed in following graphs :
 
 ![XXH3, latency, random size](https://user-images.githubusercontent.com/750081/61976089-aedeab00-af9f-11e9-9239-e5375d6c080f.png)
 
-The algorithm is currently labelled experimental, its return values can still change in a future version.
-It can be used for ephemeral data, and for tests, but avoid storing long-term hash values yet.
-To access it, one need to unlock its declaration using macro `XXH_STATIC_LINKING_ONLY`.
+The algorithm is currently labeled experimental, its return values can still change in future versions.
+It can already be used for ephemeral data, and for tests, but avoid storing long-term hash values yet.
+
+To access experimental prototypes, one need to unlock their declaration using macro `XXH_STATIC_LINKING_ONLY`.
 `XXH3` will be stabilized in a future version.
 This period is used to collect users' feedback.
 

diff --git a/appveyor.yml b/appveyor.yml
@@ -82,7 +82,7 @@ build_script:
 
   - if [%COMPILER%]==[visual] (
       cd cmake_unofficial &&
-      cmake . -DCMAKE_BUILD_TYPE=Release -A %ARCH% &&
+      cmake . -DCMAKE_BUILD_TYPE=Release -A %ARCH% -DXXHASH_C_FLAGS="/WX" &&
       cmake --build . --config Release
     )
 

diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt
@@ -93,6 +93,9 @@ endif(XXHASH_BUILD_XXHSUM)
 
 # Extra warning flags
 include (CheckCCompilerFlag)
+if (XXHASH_C_FLAGS)
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${XXHASH_C_FLAGS}")
+endif()
 foreach (flag
     -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow
     -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement
@@ -165,4 +168,3 @@ if(NOT XXHASH_BUNDLED_MODE)
     DESTINATION ${xxHash_CONFIG_INSTALL_DIR}
     NAMESPACE ${PROJECT_NAME}::)
 endif(NOT XXHASH_BUNDLED_MODE)
-