Merge pull request #320 from t20100/updates

Updated filters: `c-blosc` v1.21.6, `c-blosc2` v2.15.1, `fcidecomp` v2.1.1; Updated libraries: `lz4` v1.10.0, `zlib` v1.3.1

doc/information.rst

@@ -65,14 +65,14 @@ HDF5 compression filters and compression libraries sources were obtained from:
   (commit `49e3b65 <https://github.com/nexusformat/HDF5-External-Filter-Plugins/tree/49e3b65eca772bca77af13ba047d8b577673afba>`_)
   using LZ4.
 * `bitshuffle plugin <https://github.com/kiyo-masui/bitshuffle>`_ (v0.5.1) using LZ4 and ZStd.
-* bzip2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.9.2)
+* bzip2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.10.1)
   using `BZip2 <https://sourceware.org/git/bzip2.git>`_ (v1.0.8).
 * `hdf5-blosc plugin <https://github.com/Blosc/hdf5-blosc>`_ (v1.0.1)
-  using `c-blosc <https://github.com/Blosc/c-blosc>`_ (v1.21.5), LZ4, Snappy, ZLib and ZStd.
-* hdf5-blosc2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.9.2)
-  using `c-blosc2 <https://github.com/Blosc/c-blosc2>`_ (v2.15.0), LZ4, ZLib and ZStd.
+  using `c-blosc <https://github.com/Blosc/c-blosc>`_ (v1.21.6), LZ4, Snappy, ZLib and ZStd.
+* hdf5-blosc2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.10.1)
+  using `c-blosc2 <https://github.com/Blosc/c-blosc2>`_ (v2.15.1), LZ4, ZLib and ZStd.
 * `FCIDECOMP plugin <https://gitlab.eumetsat.int/open-source/data-tailor-plugins/fcidecomp>`_
-  (`v2.0.1 <https://gitlab.eumetsat.int/open-source/data-tailor-plugins/fcidecomp/-/tree/e88f83c03bafcd0769c167dca14aa7aabf728e1b>`_)
+  (`v2.1.1 <https://gitlab.eumetsat.int/open-source/data-tailor-plugins/fcidecomp/-/tree/2.1.1>`_)
   using `CharLS <https://github.com/team-charls/charls>`_ (v2.1.0).
 * `SZ plugin <https://github.com/szcompressor/SZ>`_
   (commit `f466775 <https://github.com/szcompressor/SZ/tree/f4667759ead6a902110e80ff838ccdfddbc8dcd7>`_)
@@ -88,10 +88,10 @@ HDF5 compression filters and compression libraries sources were obtained from:
 
 Sources of compression libraries shared accross multiple filters were obtained from:
 
-* `LZ4 v1.9.4 <https://github.com/Blosc/c-blosc2/tree/v2.15.0/internal-complibs/lz4-1.9.4>`_
+* `LZ4 v1.10.0 <https://github.com/Blosc/c-blosc2/tree/v2.15.1/internal-complibs/lz4-1.10.0>`_
 * `Snappy v1.2.1 <https://github.com/google/snappy>`_
-* `ZStd v1.5.6 <https://github.com/Blosc/c-blosc2/tree/v2.15.0/internal-complibs/zstd-1.5.6>`_
-* `ZLib v1.2.13 <https://github.com/Blosc/c-blosc/tree/v1.21.5/internal-complibs/zlib-1.2.13>`_
+* `ZStd v1.5.6 <https://github.com/Blosc/c-blosc2/tree/v2.15.1/internal-complibs/zstd-1.5.6>`_
+* `ZLib v1.3.1 <https://github.com/Blosc/c-blosc/tree/v1.21.6/internal-complibs/zlib-1.3.1>`_
 
 When compiled with Intel IPP, the LZ4 compression library is replaced with `LZ4 v1.9.3 <https://github.com/lz4/lz4/releases/tag/v1.9.3>`_ patched with a patch from Intel IPP 2021.7.0.
 

setup.py

@@ -942,15 +942,17 @@ def get_blosc2_plugin():
     ]
     define_macros = [
         ('HAVE_PLUGINS', 1),
+        ('SHUFFLE_SSE2_ENABLED', 1),
+        ('SHUFFLE_AVX2_ENABLED', 1),
         ('SHUFFLE_AVX512_ENABLED', 1),
         ('SHUFFLE_NEON_ENABLED', 1),
+        ('SHUFFLE_ALTIVEC_ENABLED', 1),
     ]
     extra_compile_args = []
     extra_link_args = []
     libraries = []
 
     if platform.machine() == 'ppc64le':
-        define_macros.append(('SHUFFLE_ALTIVEC_ENABLED', 1))
         define_macros.append(('NO_WARN_X86_INTRINSICS', None))
     if HostConfig.ARCH == 'ARM_8':
         extra_compile_args += ['-flax-vector-conversions']

src/PyTables/hdf5-blosc2/src/blosc2_filter.c

@@ -298,6 +298,7 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
   /* Filter params that are only set for B2ND */
   int ndim = -1;
   int32_t chunkshape[BLOSC2_MAX_DIM];
+  size_t chunksize = typesize;
   if (cd_nelmts >= 8) {
     /* Get chunk shape for B2ND */
     ndim = cd_values[7];
@@ -321,6 +322,7 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
     }
     for (int i = 0; i < ndim; i++) {
       chunkshape[i] = cd_values[8 + i];
+      chunksize *= (size_t) cd_values[8 + i];
     }
   }
 
@@ -367,6 +369,14 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
 
     blosc2_storage storage = {.cparams=&cparams, .contiguous=false};
 
+    if (ndim > 1 && nbytes != chunksize) {
+      BLOSC_TRACE_INFO("Filter input size %lu does not match chunk data size %lu "
+                       "(e.g. Fletcher32 checksum added before compression step), "
+                       "using plain Blosc2 instead of B2ND",
+                       nbytes, chunksize);
+      ndim = -1;
+    }
+
     if (ndim > 1) {
 
       b2nd_context_t *ctx = NULL;
@@ -453,6 +463,15 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
     /* declare dummy variables */
     int32_t cbytes;
 
+    /* It would be cool to have a check to detect whether the buffer contains
+     * an unexpected amout of bytes because of the application of a previous
+     * filter (e.g. a Fletcher32 checksum), thus disabling B2ND optimizations.
+     * However, we cannot know before parsing the super-chunk, and that
+     * operation accepts trailing bytes without reporting their presence.
+     * Thus such a test can only happen outside of the filter, with more
+     * information available about the whole pipeline setup.
+     */
+
     blosc2_schunk* schunk = blosc2_schunk_from_buffer(*buf, (int64_t)nbytes, false);
     if (schunk == NULL) {
       PUSH_ERR("blosc2_filter", H5E_CALLBACK, "Cannot get super-chunk from buffer");
@@ -549,6 +568,7 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
       }
 
       blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
+      dparams.schunk = schunk;
       dctx = blosc2_create_dctx(dparams);
       status = blosc2_decompress_ctx(dctx, chunk, cbytes, outbuf, (int32_t) outbuf_size);
       if (status <= 0) {

src/c-blosc/.github/workflows/cmake.yml

@@ -115,7 +115,7 @@ jobs:
             compiler: gcc
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Install packages (Ubuntu)
       if: runner.os == 'Linux' && matrix.packages

src/c-blosc/.github/workflows/fuzz.yml

@@ -16,7 +16,7 @@ jobs:
         fuzz-seconds: 600
         dry-run: false
     - name: Upload Crash
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: artifacts

src/c-blosc/ANNOUNCE.rst

@@ -1,13 +1,14 @@
 ===============================================================
- Announcing C-Blosc 1.21.5
+ Announcing C-Blosc 1.21.6
  A blocking, shuffling and lossless compression library for C
 ===============================================================
 
 What is new?
 ============
 
-This is a maintenance release. Here, we have provided a fix for a SSE2/AVX2
-build issue. Thanks to Thomas VINCENT and Mark Kittisopikul.
+This is a maintenance release. Here, we are providing new versions of
+internal Zlib and Zstd compressors, which are now 1.3.1 and 1.5.6
+respectively.
 
 For more info, please see the release notes in:
 

src/c-blosc/RELEASE_NOTES.rst

@@ -3,9 +3,18 @@
 ===========================
 
 
+Changes from 1.21.5 to 1.21.6
+=============================
+
+* Zlib updated to 1.3.1.  Thanks to Lachlan Deakin.
+* Zstd updated to 1.5.6
+* Fixed many typos.  Thanks to Dimitri Papadopoulos.
+
+
 Changes from 1.21.4 to 1.21.5
 =============================
 
+
 * Fix SSE2/AVX2 build issue.  Fixes #352. Thanks to Thomas VINCENT
   and Mark Kittisopikul.
 

src/c-blosc/bench/Makefile

@@ -25,7 +25,7 @@ LDFLAGS += -DHAVE_SNAPPY -lsnappy
 LDFLAGS += -DHAVE_ZLIB -lz
 
 # Support for internal Zlib
-#ZLIB_DIR = ../internal-complibs/zlib-1.2.13
+#ZLIB_DIR = ../internal-complibs/zlib-1.3.1
 #CFLAGS += -DHAVE_ZLIB -I$(ZLIB_DIR)
 #SOURCES += $(wildcard $(ZLIB_DIR)/*.c)
 

src/c-blosc/bench/Makefile.mingw

@@ -31,7 +31,7 @@ LDFLAGS=-DHAVE_ZLIB -I$(ZLIB_ROOT)/include -lzdll -L$(ZLIB_ROOT)/lib
 #LDFLAGS=-DHAVE_ZLIB -I$(ZLIB_ROOT)/include -lz -L$(ZLIB_ROOT)/lib
 
 # Support for internal Zlib
-#ZLIB_DIR = ../internal-complibs/zlib-1.2.13
+#ZLIB_DIR = ../internal-complibs/zlib-1.3.1
 #CFLAGS += -DHAVE_ZLIB -I$(ZLIB_DIR)
 #SOURCES += $(wildcard $(ZLIB_DIR)/*.c)
 

src/c-blosc/blosc/CMakeLists.txt

@@ -31,7 +31,7 @@ if(NOT DEACTIVATE_ZLIB)
     if (ZLIB_FOUND)
         set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR})
     else(ZLIB_FOUND)
-        set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/zlib-1.2.13)
+        set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/zlib-1.3.1)
         set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_LOCAL_DIR})
     endif(ZLIB_FOUND)
 endif(NOT DEACTIVATE_ZLIB)
@@ -40,7 +40,7 @@ if (NOT DEACTIVATE_ZSTD)
     if (ZSTD_FOUND)
         set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIR})
     else (ZSTD_FOUND)
-        set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.5.5)
+        set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.5.6)
         set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_LOCAL_DIR} ${ZSTD_LOCAL_DIR}/common)
     endif (ZSTD_FOUND)
 endif (NOT DEACTIVATE_ZSTD)

src/c-blosc/blosc/blosc.h

@@ -19,11 +19,11 @@ extern "C" {
 /* Version numbers */
 #define BLOSC_VERSION_MAJOR    1    /* for major interface/format changes  */
 #define BLOSC_VERSION_MINOR    21   /* for minor interface/format changes  */
-#define BLOSC_VERSION_RELEASE  5    /* for tweaks, bug-fixes, or development */
+#define BLOSC_VERSION_RELEASE  6    /* for tweaks, bug-fixes, or development */
 
-#define BLOSC_VERSION_STRING   "1.21.5"  /* string version.  Sync with above! */
+#define BLOSC_VERSION_STRING   "1.21.6"  /* string version.  Sync with above! */
 #define BLOSC_VERSION_REVISION "$Rev$"   /* revision version */
-#define BLOSC_VERSION_DATE     "$Date:: 2023-05-16 #$"    /* date version */
+#define BLOSC_VERSION_DATE     "$Date:: 2024-06-24 #$"    /* date version */
 
 /* The *_FORMAT symbols should be just 1-byte long */
 #define BLOSC_VERSION_FORMAT    2   /* Blosc format version, starting at 1 */