Merge pull request #3000 from easybuilders/4.8.x

release EasyBuild v4.8.1

RELEASE_NOTES

@@ -3,7 +3,47 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.
 
-The latest version of easybuild-easyblocks provides 249 software-specific easyblocks and 42 generic easyblocks.
+The latest version of easybuild-easyblocks provides 251 software-specific easyblocks and 42 generic easyblocks.
+
+
+v4.8.1 (11 September 2023)
+--------------------------
+
+update/bugfix release
+
+- new custom easyblock for sympy (#2949) and tensorflow-compression (#2990)
+- minor enhancements and updates, including:
+  - drop unnecessary CUDA stub libraries from $LIBRARY_PATH (#2793)
+  - update Score-P easyblock to use `--with-nocross-compiler-suite=nvhpc` for recent software versions (#2928)
+  - unset `$CPPFLAGS`, `$LDFLAGS`, `$LIB` which may interfere with Score-P configure magic (#2928)
+  - update Clang easyblock for versions >= 16 + run tests only for final stage of bootstrap build (#2929)
+  - handle new directory structure for Intel Advisor (#2942)
+  - use `DCPU_BASELINE=DETECT` for OpenCV when default optarch compiler option is used (#2954)
+  - update MXNet easyblock + don't try to install R extension by default for MXNet >= 1.0 (#2955)
+  - use checkMCR.sh to determine if we have the correct MCR for FreeSurfer (#2962)
+  - add options to `MesonNinja` easyblock to customize `build_cmd`, `install_cmd`, `builddir` (#2963, #2993)
+  - add support for building CP2K with libvori support (#2967)
+  - enable system `pybind11` for PyTorch 1.10+ to make sure `pybind11` provided as dependency is used (#2968)
+  - update LLVM easyblock for LLVM v16: symlink `third-party` to `third-party-<version>.src` (#2970, #2994)
+  - update scipy easyblock for scipy >= 1.11.0 (#2971, #2980)
+  - update sanity check for Mesa >= 22.3 (#2973)
+  - update sanity check for OpenFOAM 11 (#2978)
+  - add support to `PerlModule` easyblock to customize prefix option used in installation command (#2979)
+  - update TensorFlow easyblock for v2.13 since LMDB is no longer a dependency (#2982)
+  - enhance PyTorch easyblock to print individual failed tests (#2983)
+  - enhance PETSc easyblock to support using custom `$PETSC_ARCH` (#2987)
+- various bug fixes, including:
+  - correctly determine path to active binutils in TensorFlow easyblock (#2218)
+  - patch Java binaries/libraries when using alternate sysroot to ensure correct glibc & co are picked up + add custom sanity check (#2557, #2995)
+  - update OpenMPI easyblock to fix sanity check for Clang-based compilers (#2774)
+  - improve depot management in `JuliaPackage` easyblock (#2935)
+  - disable disk space check in STAR-CCM+ installer (#2956)
+  - fix type check for `optarch` value in `Cargo` easyblock (#2969)
+  - conditionally add `-Wno-unused-command-line-argument` to `$CFLAGS` to fix error when installing `imkl-FFTW` with RPATH (#2975)
+  - enhance PythonPackage easyblock to deal with `posix_local` installation scheme used by Python in recent Debian/Ubuntu versions (#2977, #2988)
+  - don't add MATLAB libraries to `$LD_LIBRARY_PATH` (#2981)
+  - enhance Mesa easyblock to append EGL vendor library directory path to `$__EGL_VENDOR_LIBRARY_DIRS` (#2985)
+  - fix typo in TensorFlow easyblock when finding libdir of OpenSSL (#2989)
 
 
 v4.8.0 (7 July 2023)

easybuild/easyblocks/__init__.py

@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.8.0')
+VERSION = LooseVersion('4.8.1')
 UNKNOWN = 'UNKNOWN'
 
 

easybuild/easyblocks/a/advisor.py

@@ -30,6 +30,7 @@
 @author: Josef Dvoracek (Institute of Physics, Czech Academy of Sciences)
 """
 
+import os
 from distutils.version import LooseVersion
 
 from easybuild.easyblocks.generic.intelbase import IntelBase
@@ -45,8 +46,10 @@ def __init__(self, *args, **kwargs):
         super(EB_Advisor, self).__init__(*args, **kwargs)
         if LooseVersion(self.version) < LooseVersion('2017'):
             self.subdir = 'advisor_xe'
-        else:
+        elif LooseVersion(self.version) < LooseVersion('2021'):
             self.subdir = 'advisor'
+        else:
+            self.subdir = os.path.join('advisor', 'latest')
 
     def prepare_step(self, *args, **kwargs):
         """Since 2019u3 there is no license required."""

easybuild/easyblocks/c/clang.py

@@ -550,14 +550,6 @@ def build_with_prev_stage(self, prev_obj, next_obj):
         # restore $PATH
         setvar('PATH', orig_path)
 
-    def run_clang_tests(self, obj_dir):
-        """Run Clang tests in specified directory (unless disabled)."""
-        if not self.cfg['skip_all_tests']:
-            change_dir(obj_dir)
-
-            self.log.info("Running tests")
-            run_cmd("make %s check-all" % self.make_parallel_opts, log_all=True)
-
     def build_step(self):
         """Build Clang stage 1, 2, 3"""
 
@@ -567,23 +559,20 @@ def build_step(self):
         super(EB_Clang, self).build_step()
 
         if self.cfg['bootstrap']:
-            # Stage 1: run tests.
-            self.run_clang_tests(self.llvm_obj_dir_stage1)
-
             self.log.info("Building stage 2")
             self.build_with_prev_stage(self.llvm_obj_dir_stage1, self.llvm_obj_dir_stage2)
-            self.run_clang_tests(self.llvm_obj_dir_stage2)
 
             self.log.info("Building stage 3")
             self.build_with_prev_stage(self.llvm_obj_dir_stage2, self.llvm_obj_dir_stage3)
-            # Don't run stage 3 tests here, do it in the test step.
 
     def test_step(self):
-        """Run Clang tests."""
-        if self.cfg['bootstrap']:
-            self.run_clang_tests(self.llvm_obj_dir_stage3)
-        else:
-            self.run_clang_tests(self.llvm_obj_dir_stage1)
+        """Run Clang tests on final stage (unless disabled)."""
+        if not self.cfg['skip_all_tests']:
+            if self.cfg['bootstrap']:
+                change_dir(self.llvm_obj_dir_stage3)
+            else:
+                change_dir(self.llvm_obj_dir_stage1)
+            run_cmd("make %s check-all" % self.make_parallel_opts, log_all=True)
 
     def install_step(self):
         """Install stage 3 binaries."""
@@ -633,6 +622,11 @@ def sanity_check_step(self):
         custom_commands = ['clang --help', 'clang++ --help', 'llvm-config --cxxflags']
         shlib_ext = get_shared_lib_ext()
 
+        # Clang v16+ only use the major version number for the resource dir
+        resdir_version = self.version
+        if LooseVersion(self.version) >= LooseVersion('16'):
+            resdir_version = self.version.split('.')[0]
+
         # Detect OpenMP support for CPU architecture
         arch = get_cpu_architecture()
         # Check architecture explicitly since Clang uses potentially
@@ -662,9 +656,9 @@ def sanity_check_step(self):
             'files': [
                 "bin/clang", "bin/clang++", "bin/llvm-ar", "bin/llvm-nm", "bin/llvm-as", "bin/opt", "bin/llvm-link",
                 "bin/llvm-config", "bin/llvm-symbolizer", "include/llvm-c/Core.h", "include/clang-c/Index.h",
-                "lib/libclang.%s" % shlib_ext, "lib/clang/%s/include/stddef.h" % self.version,
+                "lib/libclang.%s" % shlib_ext, "lib/clang/%s/include/stddef.h" % resdir_version,
             ],
-            'dirs': ["include/clang", "include/llvm", "lib/clang/%s/lib" % self.version],
+            'dirs': ["include/clang", "include/llvm", "lib/clang/%s/lib" % resdir_version],
         }
         if self.cfg['static_analyzer']:
             custom_paths['files'].extend(["bin/scan-build", "bin/scan-view"])
@@ -697,7 +691,7 @@ def sanity_check_step(self):
             custom_commands.extend(["%s --help" % flang_compiler])
 
         if LooseVersion(self.version) >= LooseVersion('3.8'):
-            custom_paths['files'].extend(["lib/libomp.%s" % shlib_ext, "lib/clang/%s/include/omp.h" % self.version])
+            custom_paths['files'].extend(["lib/libomp.%s" % shlib_ext, "lib/clang/%s/include/omp.h" % resdir_version])
 
         if LooseVersion(self.version) >= LooseVersion('12'):
             omp_target_libs = ["lib/libomptarget.%s" % shlib_ext, "lib/libomptarget.rtl.%s.%s" % (arch, shlib_ext)]

easybuild/easyblocks/c/cp2k.py

@@ -440,6 +440,16 @@ def configure_common(self):
         else:
             self.log.info("libxc module not loaded, so building without libxc support")
 
+        libvori = get_software_root('libvori')
+        if libvori:
+            if LooseVersion(self.version) >= LooseVersion('8.1'):
+                options['LIBS'] += ' -lvori'
+                options['DFLAGS'] += ' -D__LIBVORI'
+            else:
+                raise EasyBuildError("This version of CP2K does not suppport libvori")
+        else:
+            self.log.info("libvori module not loaded, so building without support for Voronoi integration")
+
         return options
 
     def configure_intel_based(self):

easybuild/easyblocks/c/cuda.py

@@ -234,6 +234,17 @@ def create_wrapper(wrapper_name, wrapper_comp):
             raise EasyBuildError("Unable to find 'ldconfig' in $PATH (%s), nor in any of %s", path, sbin_dirs)
 
         stubs_dir = os.path.join(self.installdir, 'lib64', 'stubs')
+
+        # Remove stubs which are not required as the full library is in $EBROOTCUDA/lib64 because this duplication
+        # causes issues (e.g. CMake warnings) when using this module (see $LIBRARY_PATH & $LD_LIBRARY_PATH)
+        for stub_lib in expand_glob_paths([os.path.join(stubs_dir, '*.*')]):
+            real_lib = os.path.join(self.installdir, 'lib64', os.path.basename(stub_lib))
+            if os.path.exists(real_lib):
+                self.log.debug("Removing unnecessary stub library %s", stub_lib)
+                remove_file(stub_lib)
+            else:
+                self.log.debug("Keeping stub library %s", stub_lib)
+
         # Run ldconfig to create missing symlinks in the stubs directory (libcuda.so.1, etc)
         cmd = ' '.join([ldconfig, '-N', stubs_dir])
         run_cmd(cmd)
@@ -243,7 +254,7 @@ def create_wrapper(wrapper_name, wrapper_comp):
         # See e.g. https://github.com/easybuilders/easybuild-easyconfigs/issues/12348
         # Workaround: Create a copy that matches this pattern
         new_stubs_dir = os.path.join(self.installdir, 'stubs')
-        copy_dir(stubs_dir, os.path.join(new_stubs_dir, 'lib64'))
+        copy_dir(stubs_dir, os.path.join(new_stubs_dir, 'lib64'), symlinks=True)
         # Also create the lib dir as a symlink
         symlink('lib64', os.path.join(new_stubs_dir, 'lib'), use_abspath_source=False)
 

easybuild/easyblocks/f/freesurfer.py

@@ -30,6 +30,8 @@
 
 import os
 
+from distutils.version import LooseVersion
+
 from easybuild.easyblocks.generic.tarball import Tarball
 from easybuild.framework.easyconfig import MANDATORY
 from easybuild.tools.filetools import write_file
@@ -92,4 +94,8 @@ def sanity_check_step(self):
             'dirs': ['bin', 'lib', 'mni'],
         }
 
-        super(EB_FreeSurfer, self).sanity_check_step(custom_paths=custom_paths)
+        custom_commands = []
+        if LooseVersion(self.version) >= LooseVersion("7.2"):
+            custom_commands.append('checkMCR.sh')
+
+        super(EB_FreeSurfer, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)

easybuild/easyblocks/generic/cargo.py

@@ -74,19 +74,18 @@ def rustc_optarch(self):
 
         optarch = build_option('optarch')
         if optarch:
-            if type(optarch) == dict:
+            if isinstance(optarch, dict):
                 if 'rustc' in optarch:
                     rust_optarch = optarch['rustc']
                     if rust_optarch == OPTARCH_GENERIC:
                         return generic
                     else:
                         return '-' + rust_optarch
                 self.log.info("no rustc information in the optarch dict, so using %s" % optimal)
+            elif optarch == OPTARCH_GENERIC:
+                return generic
             else:
-                if optarch == OPTARCH_GENERIC:
-                    return generic
-                else:
-                    self.log.warning("optarch is ignored as there is no translation for rustc, so using %s" % optimal)
+                self.log.warning("optarch is ignored as there is no translation for rustc, so using %s" % optimal)
         return optimal
 
     def __init__(self, *args, **kwargs):

easybuild/easyblocks/generic/cmakeninja.py

@@ -38,9 +38,13 @@ class CMakeNinja(CMakeMake, MesonNinja):
 
     @staticmethod
     def extra_options(extra_vars=None):
-        """Define extra easyconfig parameters specific to CMakeMake."""
+        """Define extra easyconfig parameters specific to CMakeNinja."""
         extra_vars = CMakeMake.extra_options(extra_vars)
         extra_vars['generator'][0] = 'Ninja'
+        extra_vars.update({
+            key: value for key, value in MesonNinja.extra_options().items()
+            if key.startswith('build_') or key.startswith('install_')
+        })
         return extra_vars
 
     def configure_step(self, *args, **kwargs):

easybuild/easyblocks/generic/juliabundle.py

@@ -86,9 +86,12 @@ def prepare_step(self, *args, **kwargs):
             raise EasyBuildError("Julia not included as dependency!")
 
     def make_module_extra(self, *args, **kwargs):
-        """Prepend installation directory to JULIA_DEPOT_PATH in module file."""
+        """
+        Module has to append installation directory to JULIA_DEPOT_PATH to keep
+        the user depot in the top entry. See issue easybuilders/easybuild-easyconfigs#17455
+        """
         txt = super(JuliaBundle, self).make_module_extra()
-        txt += self.module_generator.prepend_paths('JULIA_DEPOT_PATH', [''])
+        txt += self.module_generator.append_paths('JULIA_DEPOT_PATH', [''])
         return txt
 
     def sanity_check_step(self, *args, **kwargs):

easybuild/easyblocks/generic/juliapackage.py

@@ -28,6 +28,7 @@
 @author: Alex Domingo (Vrije Universiteit Brussel)
 """
 import os
+import re
 
 from distutils.version import LooseVersion
 
@@ -40,6 +41,7 @@
 from easybuild.tools.run import run_cmd
 
 EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "")
+USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?$")
 
 
 class JuliaPackage(ExtensionEasyBlock):
@@ -56,6 +58,26 @@ def extra_options(extra_vars=None):
         })
         return extra_vars
 
+    def set_depot_path(self):
+        """
+        Top directory in JULIA_DEPOT_PATH is target installation directory
+        Prepend installation directory to JULIA_DEPOT_PATH
+        Remove user depot from JULIA_DEPOT_PATH during installation
+        see https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH
+        """
+        depot_path = os.getenv('JULIA_DEPOT_PATH', [])
+
+        if depot_path:
+            depot_path = depot_path.split(os.pathsep)
+        if len(depot_path) > 0:
+            # strip user depot path (top entry by definition)
+            if USER_DEPOT_PATTERN.search(depot_path[0]):
+                self.log.debug('Temporary disabling Julia user depot: %s', depot_path[0])
+                del depot_path[0]
+
+        depot_path.insert(0, self.installdir)
+        env.setvar('JULIA_DEPOT_PATH', os.pathsep.join(depot_path))
+
     def set_pkg_offline(self):
         """Enable offline mode of Julia Pkg"""
         if get_software_root('Julia') is None:
@@ -79,6 +101,7 @@ def prepare_step(self, *args, **kwargs):
         """Prepare for installing Julia package."""
         super(JuliaPackage, self).prepare_step(*args, **kwargs)
         self.set_pkg_offline()
+        self.set_depot_path()
 
     def configure_step(self):
         """No separate configuration for JuliaPackage."""
@@ -95,16 +118,6 @@ def test_step(self):
     def install_step(self):
         """Install Julia package with Pkg"""
 
-        # prepend installation directory to Julia DEPOT_PATH
-        # extensions in a bundle can share their DEPOT_PATH
-        # see https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH
-        depot_path = os.getenv('JULIA_DEPOT_PATH', [])
-        if depot_path:
-            depot_path = depot_path.split(os.pathsep)
-        if self.installdir not in depot_path:
-            depot_path = os.pathsep.join([depot for depot in [self.installdir] + depot_path if depot])
-            env.setvar('JULIA_DEPOT_PATH', depot_path)
-
         # command sequence for Julia.Pkg
         julia_pkg_cmd = ['using Pkg']
         if os.path.isdir(os.path.join(self.start_dir, '.git')):
@@ -147,6 +160,7 @@ def run(self):
         ExtensionEasyBlock.run(self, unpack_src=True)
 
         self.set_pkg_offline()
+        self.set_depot_path()  # all extensions share common depot in installdir
         self.install_step()
 
     def sanity_check_step(self, *args, **kwargs):
@@ -163,7 +177,10 @@ def sanity_check_step(self, *args, **kwargs):
         return ExtensionEasyBlock.sanity_check_step(self, EXTS_FILTER_JULIA_PACKAGES, *args, **kwargs)
 
     def make_module_extra(self):
-        """Prepend installation directory to JULIA_DEPOT_PATH in module file."""
+        """
+        Module has to append installation directory to JULIA_DEPOT_PATH to keep
+        the user depot in the top entry. See issue easybuilders/easybuild-easyconfigs#17455
+        """
         txt = super(JuliaPackage, self).make_module_extra()
-        txt += self.module_generator.prepend_paths('JULIA_DEPOT_PATH', [''])
+        txt += self.module_generator.append_paths('JULIA_DEPOT_PATH', [''])
         return txt