Merge pull request #2761 from easybuilders/4.6.x

release EasyBuild v4.6.0

RELEASE_NOTES

@@ -3,7 +3,30 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.
 
-The latest version of easybuild-easyblocks provides 245 software-specific easyblocks and 37 generic easyblocks.
+The latest version of easybuild-easyblocks provides 246 software-specific easyblocks and 37 generic easyblocks.
+
+
+v4.6.0 (July 8th 2022)
+----------------------
+
+feature release
+
+- new software-specific easyblock for STAR-CCM+ (#1613)
+- minor enhancements and updates, including:
+  - update Siesta EasyBlock to support GCC 10+ by adding -fallow-argument-mismatch Fortran compiler option (#2690)
+  - enable building of shared library for Libint 2.7+ (#2738)
+  - allow some PyTorch tests to fail + print warning if one or more tests fail (#2742)
+  - also support OpenSSL 3.0 in OpenSSL wrapper easyblock (#2746)
+  - add more logging to install_pc_files method of OpenSSL wrapper easyblock (#2752)
+  - make WPS easyblock aware of (pre)buildopts (#2754)
+  - add Abseil system dependency for TensorFlow 2.9+ (#2757)
+  - disable altivec when building FFTW versions < 3.4 with single-precision with GCC on POWER (#2758)
+- various bug fixes, including:
+  - make VEP easyblock compatible with --sanity-check-only (#2743)
+  - update Rosetta easyblock to take into account that $LD_LIBRARY_PATH, $CPATH, $PATH may not be defined (#2744)
+  - only load temporary module file during sanity check for pybind11 for stand-alone installations, so it can be installed as extension (#2747)
+  - make sure that CMakeMakeCp uses correct build dir (#2748)
+  - enhance Bazel easyblock to avoid writing to $HOME in sanity check (#2756)
 
 
 v4.5.5 (June 8th 2022)

easybuild/easyblocks/__init__.py

@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.5.5')
+VERSION = LooseVersion('4.6.0')
 UNKNOWN = 'UNKNOWN'
 
 

easybuild/easyblocks/b/bazel.py

@@ -119,7 +119,24 @@ def prepare_step(self, *args, **kwargs):
         """Setup bazel output root"""
         super(EB_Bazel, self).prepare_step(*args, **kwargs)
         self.bazel_tmp_dir = tempfile.mkdtemp(suffix='-bazel-tmp', dir=self.builddir)
-        self.output_user_root = tempfile.mkdtemp(suffix='-bazel-root', dir=self.builddir)
+        self._make_output_user_root()
+
+    def _make_output_user_root(self):
+        if not os.path.isdir(self.builddir):
+            # Can happen on module-only or sanity-check-only runs
+            self.log.info("Using temporary folder for user_root as builddir doesn't exist")
+            dir = None  # Will use the EB created temp dir
+        else:
+            dir = self.builddir
+        self._output_user_root = tempfile.mkdtemp(suffix='-bazel-root', dir=dir)
+
+    @property
+    def output_user_root(self):
+        try:
+            return self._output_user_root
+        except AttributeError:
+            self._make_output_user_root()
+            return self._output_user_root
 
     def extract_step(self):
         """Extract Bazel sources."""
@@ -207,6 +224,7 @@ def sanity_check_step(self):
         }
         custom_commands = []
         if LooseVersion(self.version) >= LooseVersion('1.0'):
-            custom_commands.append("bazel --help")
+            # Avoid writes to $HOME
+            custom_commands.append("bazel --output_user_root=%s --help" % self.output_user_root)
 
         super(EB_Bazel, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)

easybuild/easyblocks/f/fftw.py

@@ -203,8 +203,9 @@ def run_all_steps(self, *args, **kwargs):
                 comp_fam = self.toolchain.comp_family()
                 fftw_ver = LooseVersion(self.version)
                 if cpu_arch == POWER and comp_fam == TC_CONSTANT_GCC:
-                    # See https://github.com/FFTW/fftw3/issues/59 which applies to GCC 5.x - 10.x
-                    if prec == 'single' and fftw_ver <= LooseVersion('3.3.9'):
+                    # See https://github.com/FFTW/fftw3/issues/59 which applies to GCC 5 and above
+                    # Upper bound of 3.4 (as of yet unreleased) in hope there will eventually be a fix.
+                    if prec == 'single' and fftw_ver < LooseVersion('3.4'):
                         self.log.info("Disabling altivec for single precision on POWER with GCC for FFTW/%s"
                                       % self.version)
                         prec_configopts.append('--disable-altivec')

easybuild/easyblocks/generic/cmakemake.py

@@ -92,6 +92,7 @@ def __init__(self, *args, **kwargs):
         """Constructor for CMakeMake easyblock"""
         super(CMakeMake, self).__init__(*args, **kwargs)
         self._lib_ext = None
+        self.separate_build_dir = None
 
     @property
     def lib_ext(self):
@@ -122,7 +123,8 @@ def configure_step(self, srcdir=None, builddir=None):
         setup_cmake_env(self.toolchain)
 
         if builddir is None and self.cfg.get('separate_build_dir', True):
-            builddir = create_unused_dir(self.builddir, 'easybuild_obj')
+            self.separate_build_dir = create_unused_dir(self.builddir, 'easybuild_obj')
+            builddir = self.separate_build_dir
 
         if builddir:
             mkdir(builddir, parents=True)

easybuild/easyblocks/generic/cmakemakecp.py

@@ -50,5 +50,9 @@ def configure_step(self, srcdir=None, builddir=None):
     def install_step(self):
         """Install by copying specified files and directories."""
         if self.cfg.get('separate_build_dir', False):
-            self.cfg['start_dir'] = os.path.join(self.builddir, 'easybuild_obj')
+            if self.separate_build_dir:
+                self.cfg['start_dir'] = self.separate_build_dir
+            else:
+                self.cfg['start_dir'] = os.path.join(self.builddir, 'easybuild_obj')
+
         return MakeCp.install_step(self)

easybuild/easyblocks/generic/pythonpackage.py

@@ -639,8 +639,12 @@ def build_step(self):
             # We consider the build and install output together as downloads likely happen here if this is run
             self.install_cmd_output += out
 
-    def test_step(self):
-        """Test the built Python package."""
+    def test_step(self, return_output_ec=False):
+        """
+        Test the built Python package.
+
+        :param return_output: return output and exit code of test command
+        """
 
         if isinstance(self.cfg['runtest'], string_type):
             self.testcmd = self.cfg['runtest']
@@ -649,6 +653,8 @@ def test_step(self):
             extrapath = ""
             testinstalldir = None
 
+            out, ec = (None, None)
+
             if self.testinstall:
                 # install in test directory and export PYTHONPATH
 
@@ -670,12 +676,24 @@ def test_step(self):
 
             if self.testcmd:
                 testcmd = self.testcmd % {'python': self.python_cmd}
-                cmd = ' '.join([extrapath, self.cfg['pretestopts'], testcmd, self.cfg['testopts']])
-                run_cmd(cmd, log_all=True, simple=True)
+                cmd = ' '.join([
+                    extrapath,
+                    self.cfg['pretestopts'],
+                    testcmd,
+                    self.cfg['testopts'],
+                ])
+
+                if return_output_ec:
+                    (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False)
+                else:
+                    run_cmd(cmd, log_all=True, simple=True)
 
             if testinstalldir:
                 remove_dir(testinstalldir)
 
+            if return_output_ec:
+                return (out, ec)
+
     def install_step(self):
         """Install Python package to a custom path using setup.py"""
 

easybuild/easyblocks/l/libint.py

@@ -119,6 +119,9 @@ def configure_step(self):
             if self.cfg['with_fortran']:
                 self.cfg.update('configopts', '-DENABLE_FORTRAN=ON')
 
+            # also build shared libraries (not enabled by default)
+            self.cfg.update('configopts', "-DLIBINT2_BUILD_SHARED_AND_STATIC_LIBS=ON")
+
             # specify current directory as source directory (that contains CMakeLists.txt),
             # since that's the path to the unpacked source tarball for Libint library (created by 'make export')
             super(EB_Libint, self).configure_step(srcdir=os.getcwd())

easybuild/easyblocks/o/openssl_wrapper.py

@@ -112,6 +112,10 @@ def __init__(self, *args, **kwargs):
                 LINUX: ('so.1.1', ),
                 DARWIN: ('1.1.dylib', ),
             },
+            '3.0': {
+                LINUX: ('so.3', ),
+                DARWIN: ('3.dylib', ),
+            },
         }
 
         os_type = get_os_type()
@@ -135,6 +139,7 @@ def __init__(self, *args, **kwargs):
         openssl_engines = {
             '1.0': 'engines',
             '1.1': 'engines-1.1',
+            '3.0': 'engines-3',
         }
         self.target_ssl_engine = openssl_engines[self.majmin_version]
 
@@ -421,15 +426,19 @@ def install_pc_files(self):
                 pc_name_suffix = pc_name + '11'
                 pc_exists_cmd = "pkg-config --exists %s" % pc_name_suffix
                 if run_cmd(pc_exists_cmd, simple=True, log_ok=False, log_all=False):
+                    self.log.info("%s exists", pc_name_suffix)
                     pc_name = pc_name_suffix
 
             # get requires from pkg-config
             pc_file['requires'] = []
             for require_type in ['Requires', 'Requires.private']:
                 require_print = require_type.lower().replace('.', '-')
-                requires, _ = run_cmd("pkg-config --print-%s %s" % (require_print, pc_name), simple=False, log_ok=False)
+                pc_print_cmd = "pkg-config --print-%s %s" % (require_print, pc_name)
+                out, _ = run_cmd(pc_print_cmd, simple=False, log_ok=False)
+                self.log.info("Output of '%s': %s", pc_print_cmd, out)
 
-                if requires:
+                if out:
+                    requires = out
                     # use unsuffixed names for components provided by this wrapper
                     for wrap_comp in openssl_components:
                         requires = re.sub(r'^%s[0-9]+$' % wrap_comp, wrap_comp, requires, flags=re.M)
@@ -445,9 +454,16 @@ def install_pc_files(self):
                 pc_file['libs'] = "Libs: -L${libdir} -l%s" % c_lib_name
                 pc_file['cflags'] = "Cflags: -I${includedir}"
                 # infer private libs through pkg-config
-                linker_libs, _ = run_cmd("pkg-config --libs %s" % pc_name, simple=False, log_ok=False)
-                all_libs, _ = run_cmd("pkg-config --libs --static %s" % pc_name, simple=False, log_ok=False)
-                libs_priv = "%s " % all_libs.rstrip()
+                pc_libs_cmd = "pkg-config --libs %s" % pc_name
+                out, _ = run_cmd(pc_libs_cmd, simple=False, log_ok=False)
+                self.log.info("Output of '%s': %s", pc_libs_cmd, out)
+                linker_libs = out
+
+                pc_libs_static_cmd = "pkg-config --libs --static %s" % pc_name
+                out, _ = run_cmd(pc_libs_static_cmd, simple=False, log_ok=False)
+                self.log.info("Output of '%s': %s", pc_libs_static_cmd, out)
+
+                libs_priv = "%s " % out.rstrip()
                 for flag in linker_libs.rstrip().split(' '):
                     libs_priv = libs_priv.replace("%s " % flag, '')
                 pc_file['libs'] += "\nLibs.private: %s" % libs_priv

easybuild/easyblocks/p/pybind11.py

@@ -91,13 +91,17 @@ def sanity_check_step(self):
         # don't add user site directory to sys.path (equivalent to python -s)
         env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
         # Get python includes
-        fake_mod_data = self.load_fake_module(purge=True)
+        if not self.is_extension:
+            # only load fake module for stand-alone installations (not for extensions),
+            # since for extension the necessary modules should already be loaded at this point
+            fake_mod_data = self.load_fake_module(purge=True)
         cmd = "%s -c 'import pybind11; print(pybind11.get_include())'" % self.python_cmd
         out, ec = run_cmd(cmd, simple=False)
         if ec:
             raise EasyBuildError("Failed to get pybind11 includes!")
         python_include = out.strip()
-        self.clean_up_fake_module(fake_mod_data)
+        if not self.is_extension:
+            self.clean_up_fake_module(fake_mod_data)
 
         # Check for CMake config and includes
         custom_paths = {

easybuild/easyblocks/p/pytorch.py

@@ -31,15 +31,16 @@
 import os
 import re
 import tempfile
+import easybuild.tools.environment as env
 from distutils.version import LooseVersion
 from easybuild.easyblocks.generic.pythonpackage import PythonPackage
 from easybuild.framework.easyconfig import CUSTOM
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.tools.config import build_option
-import easybuild.tools.environment as env
+from easybuild.tools.filetools import symlink, apply_regex_substitutions
 from easybuild.tools.modules import get_software_root, get_software_version
 from easybuild.tools.systemtools import POWER, get_cpu_architecture
-from easybuild.tools.filetools import symlink, apply_regex_substitutions
+from easybuild.tools.utilities import nub
 
 
 class EB_PyTorch(PythonPackage):
@@ -49,9 +50,10 @@ class EB_PyTorch(PythonPackage):
     def extra_options():
         extra_vars = PythonPackage.extra_options()
         extra_vars.update({
-            'excluded_tests': [{}, 'Mapping of architecture strings to list of tests to be excluded', CUSTOM],
-            'custom_opts': [[], 'List of options for the build/install command. Can be used to change the defaults ' +
-                                'set by the PyTorch EasyBlock, for example ["USE_MKLDNN=0"].', CUSTOM],
+            'custom_opts': [[], "List of options for the build/install command. Can be used to change the defaults " +
+                                "set by the PyTorch EasyBlock, for example ['USE_MKLDNN=0'].", CUSTOM],
+            'excluded_tests': [{}, "Mapping of architecture strings to list of tests to be excluded", CUSTOM],
+            'max_failed_tests': [0, "Maximum number of failing tests", CUSTOM],
         })
         extra_vars['download_dep_fail'][0] = True
         extra_vars['sanity_pip_check'][0] = True
@@ -253,7 +255,42 @@ def test_step(self):
             'python': self.python_cmd,
             'excluded_tests': ' '.join(excluded_tests)
         })
-        super(EB_PyTorch, self).test_step()
+
+        (tests_out, tests_ec) = super(EB_PyTorch, self).test_step(return_output_ec=True)
+
+        ran_tests_hits = re.findall(r"^Ran (?P<test_cnt>[0-9]+) tests in", tests_out, re.M)
+        test_cnt = 0
+        for hit in ran_tests_hits:
+            test_cnt += int(hit)
+
+        failed_tests = nub(re.findall(r"^(?P<failed_test_name>.*) failed!\s*$", tests_out, re.M))
+        failed_test_cnt = len(failed_tests)
+
+        if failed_test_cnt:
+            max_failed_tests = self.cfg['max_failed_tests']
+
+            test_or_tests = 'tests' if failed_test_cnt > 1 else 'test'
+            msg = "%d %s (out of %d) failed:\n" % (failed_test_cnt, test_or_tests, test_cnt)
+            msg += '\n'.join('* %s' % t for t in sorted(failed_tests))
+
+            if max_failed_tests == 0:
+                raise EasyBuildError(msg)
+            else:
+                msg += '\n\n' + ' '.join([
+                    "The PyTorch test suite is known to include some flaky tests,",
+                    "which may fail depending on the specifics of the system or the context in which they are run.",
+                    "For this PyTorch installation, EasyBuild allows up to %d tests to fail." % max_failed_tests,
+                    "We recommend to double check that the failing tests listed above ",
+                    "are known to be flaky, or do not affect your intended usage of PyTorch.",
+                    "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).",
+                ])
+                print_warning(msg)
+
+                if failed_test_cnt > max_failed_tests:
+                    raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d",
+                                         failed_test_cnt, max_failed_tests)
+        elif tests_ec:
+            raise EasyBuildError("Test command had non-zero exit code (%s), but no failed tests found?!", tests_ec)
 
     def test_cases_step(self):
         # Make PyTorch tests not use the user home

easybuild/easyblocks/r/rosetta.py

@@ -121,9 +121,12 @@ def configure_step(self):
         self.log.debug("List of extra environment variables to pass down: %s" % str(env_vars))
 
         # create user.settings file
-        paths = os.getenv('PATH').split(':')
-        ld_library_paths = os.getenv('LD_LIBRARY_PATH').split(':')
-        cpaths = os.getenv('CPATH').split(':')
+        paths = os.getenv('PATH')
+        paths = paths.split(':') if paths else []
+        ld_library_paths = os.getenv('LD_LIBRARY_PATH')
+        ld_library_paths = ld_library_paths.split(':') if ld_library_paths else []
+        cpaths = os.getenv('CPATH')
+        cpaths = cpaths.split(':') if cpaths else []
         flags = [str(f).strip('-') for f in self.toolchain.variables['CXXFLAGS'].copy()]
 
         txt = '\n'.join([

easybuild/easyblocks/s/siesta.py

@@ -108,6 +108,12 @@ def configure_step(self):
             (r"^(FPPFLAGS_free_F90\s*=.*)$", r"\1 -ffree-line-length-none"),
         ]
 
+        gfortran_flags = ''
+        gcc_version = get_software_version('GCCcore') or get_software_version('GCC')
+        if LooseVersion(gcc_version) >= LooseVersion('10.0') and LooseVersion(self.version) <= LooseVersion('4.1.5'):
+            # -fallow-argument-mismatch is required when compiling with GCC 10.x & more recent
+            gfortran_flags = '-fallow-argument-mismatch'
+
         netcdff_loc = get_software_root('netCDF-Fortran')
         if netcdff_loc:
             # Needed for gfortran at least
@@ -181,7 +187,7 @@ def configure_step(self):
             regex_subs.extend([
                 (r"^(LIBS\s*=).*$", r"\1 %s" % complibs),
                 # Needed for a couple of the utils
-                (r"^(FFLAGS\s*=\s*).*$", r"\1 -fPIC %s" % os.environ['FCFLAGS']),
+                (r"^(FFLAGS\s*=\s*).*$", r"\1 -fPIC %s %s" % (os.environ['FCFLAGS'], gfortran_flags)),
             ])
             regex_newlines.append((r"^(COMP_LIBS\s*=.*)$", r"\1\nWXML = libwxml.a"))