diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 6346d95977..bfa19cc1a8 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -1,5 +1,13 @@ name: Static Analysis on: [push, pull_request] + +permissions: + contents: read # to fetch code (actions/checkout) + +concurrency: + group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} + cancel-in-progress: true + jobs: python-linting: runs-on: ubuntu-20.04 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 551c78bafb..b1cf6dc64a 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -1,16 +1,24 @@ name: easyblocks unit tests on: [push, pull_request] + +permissions: + contents: read # to fetch code (actions/checkout) + +concurrency: + group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-20.04 strategy: matrix: - python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10'] + python: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] modules_tool: [Lmod-6.6.3, Lmod-7.8.22, Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] module_syntax: [Lua, Tcl] # exclude some configuration for non-Lmod modules tool: # - don't test with Lua module syntax (only supported in Lmod) - # - don't test with Python 3.5 and 3.7+ (only with 2.7 and 3.6), to limit test configurations + # - don't test with Python 3.5 and 3.7+ (only with 3.6), to limit test configurations exclude: - modules_tool: modules-tcl-1.147 module_syntax: Lua @@ -28,6 +36,8 @@ jobs: python: 3.9 - modules_tool: modules-tcl-1.147 python: '3.10' + - modules_tool: modules-tcl-1.147 + python: '3.11' - modules_tool: modules-3.2.10 python: 3.5 - modules_tool: modules-3.2.10 @@ -38,6 +48,8 @@ jobs: python: 3.9 - modules_tool: modules-3.2.10 python: '3.10' + - modules_tool: modules-3.2.10 + python: '3.11' - modules_tool: modules-4.1.4 python: 3.5 - modules_tool: modules-4.1.4 @@ -48,6 +60,8 @@ jobs: python: 3.9 - modules_tool: modules-4.1.4 python: '3.10' + - modules_tool: modules-4.1.4 + python: '3.11' fail-fast: false steps: - uses: actions/checkout@v2 diff --git a/README.rst b/README.rst index 240714ed02..fd11b4fe04 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,10 @@ -.. image:: https://github.com/easybuilders/easybuild-easyblocks/workflows/easyblocks%20unit%20tests/badge.svg?branch=develop - -.. image:: https://easybuilders.github.io/easybuild/images/easybuild_logo_small.png +.. image:: https://github.com/easybuilders/easybuild/raw/develop/logo/png/easybuild_logo_2022_horizontal_dark_bg_transparent.png :align: center + :height: 400px + +.. image:: https://github.com/easybuilders/easybuild-easyblocks/workflows/easyblocks%20unit%20tests/badge.svg?branch=develop -`EasyBuild `_ is a software build +`EasyBuild `_ is a software build and installation framework that allows you to manage (scientific) software on High Performance Computing (HPC) systems in an efficient way. @@ -12,7 +13,7 @@ EasyBuild. Easyblocks are Python modules that implement the install procedure fo (group of) software package(s). Together with the EasyBuild framework, they allow to easily build and install supported software packages. -The EasyBuild documentation is available at http://easybuild.readthedocs.org/. +The EasyBuild documentation is available at http://docs.easybuild.io/. The easybuild-easyblocks source code is hosted on GitHub, along with an issue tracker for bug reports and feature requests, see diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 7e9f4fd022..f117aa07a1 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,7 +3,131 @@ For more detailed information, please see the git log. These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. -The latest version of easybuild-easyblocks provides 246 software-specific easyblocks and 37 generic easyblocks. +The latest version of easybuild-easyblocks provides 248 software-specific easyblocks and 41 generic easyblocks. + + +v4.7.2 (27 May 2023) +-------------------- + +- new generic easyblock for installing Rust crates with cargo: Cargo and CargoPythonPackage (#2902, #2934) +- minor enhancements and updates, including: + - let MATLAB easyblock raise an error if the MATLAB installation key is not provided (#2905) + - print message to inform that GPU package (instead of Kokkos) is used for LAMMPS (#2906) + - enhance PyTorch easyblock to use FlexiBLAS for PyTorch >= 1.11.0 (#2915) +- various bug fixes, including: + - use custom RPATH sanity check for Go packages that doesn't actually check for an RPATH section in the binary (#2913) + - use string '0' to avoid problems when openssl version is not determined (#2914) + - update GCC easyblock to ensure that --sysroot is passed to linker (but only when it needs to be) (#2921) + - add output log to MATLAB installs, actually parse for common errors (#2924) + - enhance Gurobi easyblock to allow using $EB_GUROBI_LICENSE_FILE environment variable (#2926) + - force building torchvision with CUDA support if CUDA is included as dependency by setting `$FORCE_CUDA` (#2931) + - fix exec permission on files in arch bindir for COMSOL (#2932) + +v4.7.1 (March 20th 2023) +------------------------ + +update/bugfix release + +- minor enhancements and updates, including: + - fix TensorFlow easyblock for new versions of Bazel & TensorFlow (#2854) + - make NAMD easyblock aware of (pre)testopts (#2856) + - update MesonNinja easyblock for Meson >=0.64.0 (#2861) + - update scipy easyblock for scipy >= 1.9.0 to use meson/ninja (#2862, #2903) + - modify logic in QScintilla easyblock to find the PyQt5 sipdir in more places (#2868) + - add `testinstall` custom easyconfig parameter to PythonPackage easyblock (#2872) + - use -x option for "go install" in GoPackage generic easyblock, to print commands as they are executed (#2878) + - allow disabling pybind11 tests with `runtest = False` (#2892) + - call parent post_install_step in EasyBuildMeta easyblock (so postinstallcmds are taken into account) (#2893) + - update and enhance Maple easyblock for recent versions (#2895) + - relax glob pattern to find Mathematica install script (#2896) + - implement CUDA support in the ELPA EasyBlock & fix CPP configure issue on newer ELPA versions (#2898) + - update Trilinos easyblock to allow disabling of building tests and forward deps + support Trilinos v13.x (#2900) + - enhance Python easyblock to create non-versioned symlink for python-config + check for bin/python and bin/python-config in sanity check (#2904) +- various bug fixes, including: + - do not use -g77 option when installing NVHPC 22.9+ (#2819) + - check that sanity_check_module_loaded attribute exists before querying it in PythonPackage easyblock (#2865) + - fix $JULIA_DEPOT_PATH in installation of multiple JuliaPackage extensions (#2869) + - fix checking of CUDA/ROCR-Runtime dependencies for Clang to determine default build targets (#2873) + - show template values of exts_default_options in PythonBundle (#2874) + - fix missing initialization of CMakeMake in CMakePythonPackage (#2876) + - fix error when failing pip version check during PythonPackage sanity check (#2877) + - handle templating correctly in CMakeMake when playing with configopts (#2882) + - avoid crash in test step of PyTorch easyblock if runtest is not a command (#2883) + - fix check configure option in FlexiBLAS easyblock (#2886) + - use older `ncgen -H` for older netCDF (#2889) + - fix linking numexpr with Intel MKL's VML library for imkl >= 2021.x (#2897) +- other changes: + - only give read permissions in GitHub Actions workflows (#2863) + - use start dir of extension to install R packages (#2867) + - fix website/docs links in README (#2870) + - add deprecation notice to RPackage extensions with relative paths in start_dir (#2879) + + +v4.7.0 (January 9th 2023) +------------------------- + +feature release + +- add generic easyblocks for installing (bundle of) Julia packages: JuliaPackage (#2816) and JuliaBundle (#2830) +- minor enhancements and updates, including: + - enhance TensorFlow easyblock to take into account provided OpenSSL dependency (#2575) + - add fix_shebang to install_step of PythonPackage easyblock so that we can fix shebangs when installing extensions (#2680) + - update PETSc easyblock for newer versions (>= 3.17) (#2796) + - update Clang easyblock to add support for new directory structure in Clang versions >= 14 + support Flang (#2800) + - update Xmipp easyblock since versions >= 3.20.07 use `noAsk` option to configure (#2809) + - add include/opencv4 to $CPATH for OpenCV versions >= 4.0 (#2818) + - add extra option for disabling LAPACK in ESMF (#2821) + - enable building of static libraries for libxml2 >= 2.10 (#2825) + - update Xmipp easyblock to handle effects of CUDA at SYSTEM level and newer CUDA version requirements for stdc++ (#2831) + - update LLVM easyblock to put 'cmake' symlink in place so separate CMake modules required for LLVM 15+ can be found (#2832) + - set $TEMPDIRPATH for testsuite in the BerkeleyGW easyblock, to avoid polluting /tmp (#2836) + - add `configure_no_prefix` option to skip addition of prefix to configure command in ConfigureMake easyblock (#2842) + - update qscintilla.py to be compatible with EB install of PyQt5 >= 5.15 (#2845) + - add UCC to known_dependencies in OpenMPI EasyBlock (#2847) + - update Clang-AOMP easyblock to handle version 5.2 and newer (#2851) +- various bug fixes, including: + - fix installing of Clang with RPATH linking (#2799) + - fix --module-only for Clang + fix sanity check for Clang 11.x (#2800) + - create $XDG_CACHE_HOME for PyTorch tests (#2806) + - make PythonPackage easyblock compatible with --sanity-check-only by loading module early during sanity check step (#2828) + - fix docstring of PythonBundle generic easyblock (#2833) + - fix counting of failures in PyTorch tests (#2834, #2840) + - make sure that ANSYS INSTALL script has execute permissions (#2852, #2853) +- other changes: + - remove useless -openmp build option for MRtrix v3.x (#2822) + - update HDF5 easyblock to use --enable-threadsafe configure option to make C API thread safe (#2824) + - use new EasyBuild logo in README (#2827) + - automatically cancel Github Action workflow runs for outdated commits (#2835) + - use fixed names for bazel/wrapper subdirectories used when building TensorFlow, to make debugging easier (#2841) + - also run unit tests with Python 3.11 (#2844) + - tweak docstring in some generic easyblocks so it renders nicely in auto-generated documentation (#2849) + - update copyright lines for 2023 (#2850) + + +v4.6.2 (October 21st 2022) +-------------------------- + +update/bugfix release + +- 2 new software-specific easyblock: + - CUDA compatibility libraries (#2764) and mamba (#2808) +- minor enhancements and updates, including: + - update OpenFOAM easyblock to support OpenFOAM 10 + clean up variant/version checks (#2766) + - added support for ESMPy in ESMF (#2789) + - enhance OpenBLAS easyblock to support running LAPACK test suite + checking how many tests fail (#2801) + - make numexpr easyblock aware of toolchain with GCC + imkl (#2810) + - add sanity check commands for netCDF (#2811) +- various bug fixes, including: + - handle problems copying symlink that points to CUDA folder that is not created for non CUDA builds of SuiteSparse (#2790) + - don't install docs (to avoid trouble with Java) + add Rocky support for ABAQUS (#2792) + - correctly count the number of failing tests (not failing test suites) in PyTorch builds (#2794, #2803) + - fix docstring for PyTorch easyblock (#2795) + - handle iterative builds with MakeCp easyblock (#2798) + - accept both None and empty value for optarch to let OpenCV detect host CPU (#2804) + - enhance EasyBuildMeta easyblock: auto-enable installing with pip + fix setup.py of easyconfigs package so installation with setuptools >= 61.0 works (#2805) + - use `python -m pip` instead of `pip` in PythonPackage easyblock (#2807) +- other changes: + - make the test output from PythonPackage less verbose by disabling default search for error patterns done by run_cmd (2797) v4.6.1 (September 12th 2022) diff --git a/easybuild/__init__.py b/easybuild/__init__.py index e87b3e3c76..62a579f2dd 100644 --- a/easybuild/__init__.py +++ b/easybuild/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index c546b0843b..445fac1632 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.6.2.dev0') +VERSION = LooseVersion('4.7.3.dev0') UNKNOWN = 'UNKNOWN' diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index 91ba89d56d..50404d75db 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -44,6 +44,7 @@ from easybuild.tools.filetools import change_dir, symlink, write_file from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd_qa +from easybuild.tools.systemtools import get_os_name from easybuild.tools.py2vs3 import OrderedDict @@ -78,6 +79,9 @@ def extract_step(self): def configure_step(self): """Configure ABAQUS installation.""" if LooseVersion(self.version) >= LooseVersion('2016'): + # Rocky Linux isn't recognized; faking it as RHEL + if get_os_name() in ['Rocky Linux', 'AlmaLinux']: + setvar('DISTRIB_ID', 'RedHatEnterpriseServer') # skip checking of Linux version setvar('DSY_Force_OS', 'linux_a64') # skip checking of license server @@ -126,10 +130,9 @@ def install_step(self): # rather than a regular dictionary (where there's no guarantee on key order in general) std_qa = OrderedDict() - # Enable Extended Product Documentation - std_qa[selectionstr % (r"\[ \]", "Extended Product Documentation")] = "%(nr)s" - # Enable Abaqus CAE (docs) - std_qa[selectionstr % (r"\[ \]", "Abaqus CAE")] = "%(nr)s" + # Disable Extended Product Documentation because it has a troublesome Java dependency + std_qa[selectionstr % (r"\[*\]", "Extended Product Documentation")] = "%(nr)s" + installed_docs = False # hard disabled, previous support was actually incomplete # enable all ABAQUS components std_qa[selectionstr % (r"\[ \]", "Abaqus.*")] = "%(nr)s" @@ -146,9 +149,9 @@ def install_step(self): # Disable/enable Tosca if self.cfg['with_tosca']: - std_qa[selectionstr % (r"\[\ \]", "Tosca")] = "%(nr)s" + std_qa[selectionstr % (r"\[\ \]", "Tosca.*")] = "%(nr)s" else: - std_qa[selectionstr % (r"\[\*\]", "Tosca")] = "%(nr)s" + std_qa[selectionstr % (r"\[\*\]", "Tosca.*")] = "%(nr)s" # disable CloudView std_qa[r"(?P[0-9]+) \[X\] Search using CloudView\nEnter selection:"] = '%(cloudview)s\n\n' @@ -170,7 +173,7 @@ def install_step(self): cae_subdir = os.path.join(self.installdir, 'cae') sim_subdir = os.path.join(self.installdir, 'sim') std_qa[r"Default.*SIMULIA/EstProducts.*:"] = cae_subdir - std_qa[r"SIMULIA[0-9]*doc.*:"] = os.path.join(self.installdir, 'doc') + std_qa[r"SIMULIA[0-9]*doc.*:"] = os.path.join(self.installdir, 'doc') # if docs are installed std_qa[r"SimulationServices.*:"] = sim_subdir std_qa[r"Choose the CODE installation directory.*:\n.*\n\n.*:"] = sim_subdir std_qa[r"SIMULIA/CAE.*:"] = cae_subdir @@ -197,13 +200,13 @@ def install_step(self): std_qa[r"Please choose an action:"] = '1' - if LooseVersion(self.version) >= LooseVersion('2022'): + if LooseVersion(self.version) >= LooseVersion('2022') and installed_docs: java_root = get_software_root('Java') if java_root: std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[\]:"] = java_root std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[.+\]:"] = '' else: - raise EasyBuildError("Java is a required dependency for ABAQUS versions >= 2022, but it is missing") + raise EasyBuildError("Java is required for ABAQUS docs versions >= 2022, but it is missing") # Continue std_qa[nextstr] = '' @@ -330,7 +333,7 @@ def sanity_check_step(self): custom_commands = [] if LooseVersion(self.version) >= LooseVersion('2016'): - custom_paths['dirs'].extend(['cae', 'Commands', 'doc']) + custom_paths['dirs'].extend(['cae', 'Commands']) if LooseVersion(self.version) < LooseVersion('2020'): custom_paths['dirs'].extend(['sim']) # 'all' also check license server, but lmstat is usually not available diff --git a/easybuild/easyblocks/a/acml.py b/easybuild/easyblocks/a/acml.py index 5a0a5cc2e7..b20704a53e 100644 --- a/easybuild/easyblocks/a/acml.py +++ b/easybuild/easyblocks/a/acml.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/adf.py b/easybuild/easyblocks/a/adf.py index 09aeac12c7..b3e4f9166e 100644 --- a/easybuild/easyblocks/a/adf.py +++ b/easybuild/easyblocks/a/adf.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2022 Ghent University +# Copyright 2016-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/advisor.py b/easybuild/easyblocks/a/advisor.py index f87ffe6fe0..952c94642e 100644 --- a/easybuild/easyblocks/a/advisor.py +++ b/easybuild/easyblocks/a/advisor.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/aladin.py b/easybuild/easyblocks/a/aladin.py index 02ccaa41cf..aa248b831b 100644 --- a/easybuild/easyblocks/a/aladin.py +++ b/easybuild/easyblocks/a/aladin.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/allinea.py b/easybuild/easyblocks/a/allinea.py index 9a5f8c7f23..e15d9bc3b2 100644 --- a/easybuild/easyblocks/a/allinea.py +++ b/easybuild/easyblocks/a/allinea.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/amber.py b/easybuild/easyblocks/a/amber.py index e6c1efda90..0140ac8d10 100644 --- a/easybuild/easyblocks/a/amber.py +++ b/easybuild/easyblocks/a/amber.py @@ -1,6 +1,6 @@ ## -# Copyright 2009-2022 Ghent University -# Copyright 2015-2022 Stanford University +# Copyright 2009-2023 Ghent University +# Copyright 2015-2023 Stanford University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/anaconda.py b/easybuild/easyblocks/a/anaconda.py index 7c6793dfc1..a792c5e498 100644 --- a/easybuild/easyblocks/a/anaconda.py +++ b/easybuild/easyblocks/a/anaconda.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/ansys.py b/easybuild/easyblocks/a/ansys.py index 3473eae8c8..0c4b834846 100644 --- a/easybuild/easyblocks/a/ansys.py +++ b/easybuild/easyblocks/a/ansys.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -56,6 +56,8 @@ def install_step(self): if licport is None: licport = os.getenv('EB_ANSYS_LICENSE_SERVER_PORT', '2325:1055') + # Sources (e.g. iso files) may drop the execute permissions + adjust_permissions('INSTALL', stat.S_IXUSR) cmd = "./INSTALL -silent -install_dir %s -licserverinfo %s:%s" % (self.installdir, licport, licserv) run_cmd(cmd, log_all=True, simple=True) diff --git a/easybuild/easyblocks/a/ant.py b/easybuild/easyblocks/a/ant.py index a9555afdc1..a345118490 100644 --- a/easybuild/easyblocks/a/ant.py +++ b/easybuild/easyblocks/a/ant.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index a8a88d3b8d..8ccdaf1d86 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2022 Forschungszentrum Juelich GmbH +# Copyright 2020-2023 Forschungszentrum Juelich GmbH # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/easyblocks/a/aomp.py b/easybuild/easyblocks/a/aomp.py index 4a39adacfa..7ff4126d6d 100644 --- a/easybuild/easyblocks/a/aomp.py +++ b/easybuild/easyblocks/a/aomp.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2022 Ghent University +# Copyright 2021-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/arb.py b/easybuild/easyblocks/a/arb.py index bcc8eb3162..3d184d88bf 100644 --- a/easybuild/easyblocks/a/arb.py +++ b/easybuild/easyblocks/a/arb.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/armadillo.py b/easybuild/easyblocks/a/armadillo.py index 286f29bafe..a2faa04317 100644 --- a/easybuild/easyblocks/a/armadillo.py +++ b/easybuild/easyblocks/a/armadillo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/atlas.py b/easybuild/easyblocks/a/atlas.py index d7ed1f42cd..da8fdefbc9 100644 --- a/easybuild/easyblocks/a/atlas.py +++ b/easybuild/easyblocks/a/atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/bamtools.py b/easybuild/easyblocks/b/bamtools.py index 63ab90a9eb..0a11f2e4cb 100644 --- a/easybuild/easyblocks/b/bamtools.py +++ b/easybuild/easyblocks/b/bamtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 The Cyprus Institute +# Copyright 2009-2023 The Cyprus Institute # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/bazel.py b/easybuild/easyblocks/b/bazel.py index f2798184d9..e626457001 100644 --- a/easybuild/easyblocks/b/bazel.py +++ b/easybuild/easyblocks/b/bazel.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/berkeleygw.py b/easybuild/easyblocks/b/berkeleygw.py index 2a2fbfaf59..e9a9b87f8e 100644 --- a/easybuild/easyblocks/b/berkeleygw.py +++ b/easybuild/easyblocks/b/berkeleygw.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -178,6 +178,7 @@ def test_step(self): if self.cfg['runtest'] is not False: self.cfg['runtest'] = 'check' setvar('OMP_NUM_THREADS', '4') + setvar('TEMPDIRPATH', os.path.join(self.builddir, 'tmp')) super(EB_BerkeleyGW, self).test_step() def sanity_check_step(self): diff --git a/easybuild/easyblocks/b/binutils.py b/easybuild/easyblocks/b/binutils.py index 6bc27f5413..306a535d99 100644 --- a/easybuild/easyblocks/b/binutils.py +++ b/easybuild/easyblocks/b/binutils.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/bioconductor.py b/easybuild/easyblocks/b/bioconductor.py index d1e4209404..c343d75773 100644 --- a/easybuild/easyblocks/b/bioconductor.py +++ b/easybuild/easyblocks/b/bioconductor.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/bisearch.py b/easybuild/easyblocks/b/bisearch.py index 57fa97a729..975f5c391f 100644 --- a/easybuild/easyblocks/b/bisearch.py +++ b/easybuild/easyblocks/b/bisearch.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/blacs.py b/easybuild/easyblocks/b/blacs.py index ae3f570259..a600c5285c 100644 --- a/easybuild/easyblocks/b/blacs.py +++ b/easybuild/easyblocks/b/blacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/blat.py b/easybuild/easyblocks/b/blat.py index 15468627fc..53d01c2b8f 100755 --- a/easybuild/easyblocks/b/blat.py +++ b/easybuild/easyblocks/b/blat.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 the Cyprus Institute +# Copyright 2009-2023 the Cyprus Institute # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/blender.py b/easybuild/easyblocks/b/blender.py index cc61453c04..500b65a0e6 100644 --- a/easybuild/easyblocks/b/blender.py +++ b/easybuild/easyblocks/b/blender.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/boost.py b/easybuild/easyblocks/b/boost.py index 53acc8c489..5110105cd8 100644 --- a/easybuild/easyblocks/b/boost.py +++ b/easybuild/easyblocks/b/boost.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/bowtie.py b/easybuild/easyblocks/b/bowtie.py index 0b789e8ccd..4daa8512ba 100644 --- a/easybuild/easyblocks/b/bowtie.py +++ b/easybuild/easyblocks/b/bowtie.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/bowtie2.py b/easybuild/easyblocks/b/bowtie2.py index 97b547c078..285f9b98db 100644 --- a/easybuild/easyblocks/b/bowtie2.py +++ b/easybuild/easyblocks/b/bowtie2.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/b/bwa.py b/easybuild/easyblocks/b/bwa.py index fe6552c8fe..646cc85e4e 100644 --- a/easybuild/easyblocks/b/bwa.py +++ b/easybuild/easyblocks/b/bwa.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Kenneth Hoste # Authors:: George Tsouloupas , Fotis Georgatos # License:: MIT/GPL diff --git a/easybuild/easyblocks/b/bwise.py b/easybuild/easyblocks/b/bwise.py index dab0c0d312..723805ccaf 100644 --- a/easybuild/easyblocks/b/bwise.py +++ b/easybuild/easyblocks/b/bwise.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2022 Free University of Brussels +# Copyright 2018-2023 Free University of Brussels # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/bzip2.py b/easybuild/easyblocks/b/bzip2.py index 84fecdc554..5c5d34b8da 100644 --- a/easybuild/easyblocks/b/bzip2.py +++ b/easybuild/easyblocks/b/bzip2.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cblas.py b/easybuild/easyblocks/c/cblas.py index 6b4539aa4f..14b1364b74 100644 --- a/easybuild/easyblocks/c/cblas.py +++ b/easybuild/easyblocks/c/cblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cfdemcoupling.py b/easybuild/easyblocks/c/cfdemcoupling.py index 6f2a2b329c..2264ffdeb3 100644 --- a/easybuild/easyblocks/c/cfdemcoupling.py +++ b/easybuild/easyblocks/c/cfdemcoupling.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2022 Ghent University +# Copyright 2018-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cgal.py b/easybuild/easyblocks/c/cgal.py index df0bde1aa1..6cb2937cd8 100644 --- a/easybuild/easyblocks/c/cgal.py +++ b/easybuild/easyblocks/c/cgal.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/chapel.py b/easybuild/easyblocks/c/chapel.py index a962e1a5f1..fc7ddebfb7 100644 --- a/easybuild/easyblocks/c/chapel.py +++ b/easybuild/easyblocks/c/chapel.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/c/charmm.py b/easybuild/easyblocks/c/charmm.py index 5a17c1232c..6c92cae548 100644 --- a/easybuild/easyblocks/c/charmm.py +++ b/easybuild/easyblocks/c/charmm.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index 062a48dff2..172811566d 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -1,6 +1,6 @@ ## -# Copyright 2013 Dmitri Gribenko -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Dmitri Gribenko +# Copyright 2013-2023 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -32,6 +32,7 @@ @author: Dmitri Gribenko (National Technical University of Ukraine "KPI") @author: Ward Poelmans (Ghent University) @author: Alan O'Cais (Juelich Supercomputing Centre) +@author: Maxime Boissonneault (Digital Research Alliance of Canada, Universite Laval) """ import glob @@ -41,10 +42,11 @@ from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM +from easybuild.toolchains.compiler.clang import Clang from easybuild.tools import run from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option -from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir +from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir, which from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH32, AARCH64, POWER, X86_64 @@ -73,6 +75,13 @@ CUDA_TOOLKIT_SUPPORT = ['80', '90', '91', '92', '100', '101', '102', '110', '111', '112'] +# When extending the lists below, make sure to add additional sanity checks! +# List of the known LLVM projects +KNOWN_LLVM_PROJECTS = ['llvm', 'clang', 'polly', 'lld', 'lldb', 'clang-tools-extra', 'flang'] +# List of the known LLVM runtimes +KNOWN_LLVM_RUNTIMES = ['compiler-rt', 'libunwind', 'libcxx', 'libcxxabi', 'openmp'] + + class EB_Clang(CMakeMake): """Support for bootstrapping Clang.""" @@ -86,6 +95,7 @@ def extra_options(): 'bootstrap': [True, "Bootstrap Clang using GCC", CUSTOM], 'build_extra_clang_tools': [False, "Build extra Clang tools", CUSTOM], 'build_lld': [False, "Build the LLVM lld linker", CUSTOM], + 'build_lldb': [False, "Build the LLVM lldb debugger", CUSTOM], 'build_targets': [None, "Build targets for LLVM (host architecture if None). Possible values: " + ', '.join(CLANG_TARGETS), CUSTOM], 'default_cuda_capability': [None, "Default CUDA capability specified for clang, e.g. '7.5'", CUSTOM], @@ -98,6 +108,8 @@ def extra_options(): # The sanitizer tests often fail on HPC systems due to the 'weird' environment. 'skip_sanitizer_tests': [True, "Do not run the sanitizer tests", CUSTOM], 'usepolly': [False, "Build Clang with polly", CUSTOM], + 'llvm_projects': [[], "LLVM projects to install", CUSTOM], + 'llvm_runtimes': [[], "LLVM runtimes to install", CUSTOM], }) # disable regular out-of-source build, too simplistic for Clang to work extra_vars['separate_build_dir'][0] = False @@ -112,6 +124,94 @@ def __init__(self, *args, **kwargs): self.llvm_obj_dir_stage2 = None self.llvm_obj_dir_stage3 = None self.make_parallel_opts = "" + self.runtime_lib_path = "lib" + + if not self.cfg['llvm_projects']: + self.cfg['llvm_projects'] = [] + if not self.cfg['llvm_runtimes']: + self.cfg['llvm_runtimes'] = [] + + # Be forgiving if someone places a runtime under projects since it is pretty new + for project in [p for p in self.cfg['llvm_projects'] if p in KNOWN_LLVM_RUNTIMES]: + msg = "LLVM project %s included but this should be a runtime, moving to runtime list!" % project + self.log.warning(msg) + self.cfg.update('llvm_runtimes', [project], allow_duplicate=False) + # No cleaner way to remove an element from the list + self.cfg['llvm_projects'] = [p for p in self.cfg['llvm_projects'] if p != project] + print_warning(msg) + + for project in [p for p in self.cfg['llvm_projects'] if p not in KNOWN_LLVM_PROJECTS]: + msg = "LLVM project %s included but not recognised, this project will NOT be sanity checked!" % project + self.log.warning(msg) + print_warning(msg) + + for runtime in [r for r in self.cfg['llvm_runtimes'] if r not in KNOWN_LLVM_RUNTIMES]: + msg = "LLVM runtime %s included but not recognised, this runtime will NOT be sanity checked!" % runtime + self.log.warning(msg) + print_warning(msg) + + # keep compatibility between using llvm_projects/llvm_runtimes vs using flags + if LooseVersion(self.version) >= LooseVersion('14'): + self.cfg.update('llvm_projects', ['llvm', 'clang'], allow_duplicate=False) + self.cfg.update('llvm_runtimes', ['compiler-rt', 'openmp'], allow_duplicate=False) + if self.cfg['usepolly']: + self.cfg.update('llvm_projects', 'polly', allow_duplicate=False) + if self.cfg['build_lld']: + self.cfg.update('llvm_projects', ['lld'], allow_duplicate=False) + self.cfg.update('llvm_runtimes', ['libunwind'], allow_duplicate=False) + if self.cfg['build_lldb']: + self.cfg.update('llvm_projects', 'lldb', allow_duplicate=False) + if self.cfg['libcxx']: + self.cfg.update('llvm_runtimes', ['libcxx', 'libcxxabi'], allow_duplicate=False) + if self.cfg['build_extra_clang_tools']: + self.cfg.update('llvm_projects', 'clang-tools-extra', allow_duplicate=False) + + # ensure libunwind is there if lld is there + if 'lld' in self.cfg['llvm_projects']: + self.cfg.update('llvm_runtimes', 'libunwind', allow_duplicate=False) + + # ensure libcxxabi is there if libcxx is there + if 'libcxx' in self.cfg['llvm_runtimes']: + self.cfg.update('llvm_runtimes', 'libcxxabi', allow_duplicate=False) + + build_targets = self.cfg['build_targets'] + # define build_targets if not set + if build_targets is None: + deps = [dep['name'].lower() for dep in self.cfg.dependencies()] + arch = get_cpu_architecture() + try: + default_targets = DEFAULT_TARGETS_MAP[arch][:] + # If CUDA is included as a dep, add NVPTX as a target + # There are (old) toolchains with CUDA as part of the toolchain + cuda_toolchain = hasattr(self.toolchain, 'COMPILER_CUDA_FAMILY') + if 'cuda' in deps or cuda_toolchain: + default_targets += ['NVPTX'] + # For AMDGPU support we need ROCR-Runtime and + # ROCT-Thunk-Interface, however, since ROCT is a dependency of + # ROCR we only check for the ROCR-Runtime here + # https://openmp.llvm.org/SupportAndFAQ.html#q-how-to-build-an-openmp-amdgpu-offload-capable-compiler + if 'rocr-runtime' in deps: + default_targets += ['AMDGPU'] + self.cfg['build_targets'] = build_targets = default_targets + self.log.debug("Using %s as default build targets for CPU/GPU architecture %s.", default_targets, arch) + except KeyError: + raise EasyBuildError("No default build targets defined for CPU architecture %s.", arch) + + # carry on with empty list from this point forward if no build targets are specified + if build_targets is None: + self.cfg['build_targets'] = build_targets = [] + + unknown_targets = [target for target in build_targets if target not in CLANG_TARGETS] + + if unknown_targets: + raise EasyBuildError("Some of the chosen build targets (%s) are not in %s.", + ', '.join(unknown_targets), ', '.join(CLANG_TARGETS)) + + if LooseVersion(self.version) < LooseVersion('3.4') and "R600" in build_targets: + raise EasyBuildError("Build target R600 not supported in < Clang-3.4") + + if LooseVersion(self.version) > LooseVersion('3.3') and "MBlaze" in build_targets: + raise EasyBuildError("Build target MBlaze is not supported anymore in > Clang-3.3") def check_readiness_step(self): """Fail early on RHEL 5.x and derivatives because of known bug in libc.""" @@ -123,20 +223,7 @@ def check_readiness_step(self): def extract_step(self): """ - Prepare a combined LLVM source tree. The layout is: - llvm/ Unpack llvm-*.tar.gz here - projects/ - compiler-rt/ Unpack compiler-rt-*.tar.gz here - openmp/ Unpack openmp-*.tar.xz here - tools/ - clang/ Unpack clang-*.tar.gz here - tools/ - extra/ Unpack clang-tools-extra-*.tar.gz here - polly/ Unpack polly-*.tar.gz here - libcxx/ Unpack libcxx-*.tar.gz here - libcxxabi/ Unpack libcxxabi-*.tar.gz here - lld/ Unpack lld-*.tar.gz here - libunwind/ Unpack libunwind-*.tar.gz here + Prepare a combined LLVM source tree. The layout is different for versions earlier and later than 14. """ # Extract everything into separate directories. @@ -164,27 +251,52 @@ def find_source_dir(globpatterns, targetdir): glob_src_dirs) src_dirs[glob_src_dirs[0]] = targetdir - find_source_dir('compiler-rt-*', os.path.join(self.llvm_src_dir, 'projects', 'compiler-rt')) - - if self.cfg["usepolly"]: - find_source_dir('polly-*', os.path.join(self.llvm_src_dir, 'tools', 'polly')) - - if self.cfg["build_lld"]: - find_source_dir('lld-*', os.path.join(self.llvm_src_dir, 'tools', 'lld')) - if LooseVersion(self.version) >= LooseVersion('12.0.1'): - find_source_dir('libunwind-*', os.path.normpath(os.path.join(self.llvm_src_dir, '..', 'libunwind'))) - - if self.cfg["libcxx"]: - find_source_dir('libcxx-*', os.path.join(self.llvm_src_dir, 'projects', 'libcxx')) - find_source_dir('libcxxabi-*', os.path.join(self.llvm_src_dir, 'projects', 'libcxxabi')) - - find_source_dir(['clang-[1-9]*', 'cfe-*'], os.path.join(self.llvm_src_dir, 'tools', 'clang')) - - if self.cfg["build_extra_clang_tools"]: - find_source_dir('clang-tools-extra-*', os.path.join(self.llvm_src_dir, 'tools', 'clang', 'tools', 'extra')) - - if LooseVersion(self.version) >= LooseVersion('3.8'): - find_source_dir('openmp-*', os.path.join(self.llvm_src_dir, 'projects', 'openmp')) + if any([x['name'].startswith('llvm-project') for x in self.src]): + # if sources contain 'llvm-project*', we use the full tarball + find_source_dir("../llvm-project-*", os.path.join(self.llvm_src_dir, "llvm-project-%s" % self.version)) + self.cfg.update('configopts', '-DLLVM_ENABLE_PROJECTS="%s"' % ';'.join(self.cfg['llvm_projects'])) + self.cfg.update('configopts', '-DLLVM_ENABLE_RUNTIMES="%s"' % ';'.join(self.cfg['llvm_runtimes'])) + else: + # Layout for previous versions + # llvm/ Unpack llvm-*.tar.gz here + # projects/ + # compiler-rt/ Unpack compiler-rt-*.tar.gz here + # openmp/ Unpack openmp-*.tar.xz here + # tools/ + # clang/ Unpack clang-*.tar.gz here + # tools/ + # extra/ Unpack clang-tools-extra-*.tar.gz here + # polly/ Unpack polly-*.tar.gz here + # libcxx/ Unpack libcxx-*.tar.gz here + # libcxxabi/ Unpack libcxxabi-*.tar.gz here + # lld/ Unpack lld-*.tar.gz here + # lldb/ Unpack lldb-*.tar.gz here + # libunwind/ Unpack libunwind-*.tar.gz here + find_source_dir('compiler-rt-*', os.path.join(self.llvm_src_dir, 'projects', 'compiler-rt')) + + if 'polly' in self.cfg['llvm_projects']: + find_source_dir('polly-*', os.path.join(self.llvm_src_dir, 'tools', 'polly')) + + if 'lld' in self.cfg['llvm_projects']: + find_source_dir('lld-*', os.path.join(self.llvm_src_dir, 'tools', 'lld')) + if LooseVersion(self.version) >= LooseVersion('12.0.1'): + find_source_dir('libunwind-*', os.path.normpath(os.path.join(self.llvm_src_dir, '..', 'libunwind'))) + + if 'lldb' in self.cfg['llvm_projects']: + find_source_dir('lldb-*', os.path.join(self.llvm_src_dir, 'tools', 'lldb')) + + if 'libcxx' in self.cfg['llvm_runtimes']: + find_source_dir('libcxx-*', os.path.join(self.llvm_src_dir, 'projects', 'libcxx')) + find_source_dir('libcxxabi-*', os.path.join(self.llvm_src_dir, 'projects', 'libcxxabi')) + + find_source_dir(['clang-[1-9]*', 'cfe-*'], os.path.join(self.llvm_src_dir, 'tools', 'clang')) + + if 'clang-tools-extra' in self.cfg['llvm_projects']: + find_source_dir('clang-tools-extra-*', + os.path.join(self.llvm_src_dir, 'tools', 'clang', 'tools', 'extra')) + + if LooseVersion(self.version) >= LooseVersion('3.8'): + find_source_dir('openmp-*', os.path.join(self.llvm_src_dir, 'projects', 'openmp')) for src in self.src: for (dirname, new_path) in src_dirs.items(): @@ -197,45 +309,6 @@ def find_source_dir(globpatterns, targetdir): src['finalpath'] = new_path break - def prepare_step(self, *args, **kwargs): - """Prepare build environment.""" - super(EB_Clang, self).prepare_step(*args, **kwargs) - - build_targets = self.cfg['build_targets'] - if build_targets is None: - arch = get_cpu_architecture() - try: - default_targets = DEFAULT_TARGETS_MAP[arch][:] - # If CUDA is included as a dep, add NVPTX as a target - if get_software_root("CUDA"): - default_targets += ["NVPTX"] - # For AMDGPU support we need ROCR-Runtime and - # ROCT-Thunk-Interface, however, since ROCT is a dependency of - # ROCR we only check for the ROCR-Runtime here - # https://openmp.llvm.org/SupportAndFAQ.html#q-how-to-build-an-openmp-amdgpu-offload-capable-compiler - if get_software_root("ROCR-Runtime"): - default_targets += ["AMDGPU"] - self.cfg['build_targets'] = build_targets = default_targets - self.log.debug("Using %s as default build targets for CPU/GPU architecture %s.", default_targets, arch) - except KeyError: - raise EasyBuildError("No default build targets defined for CPU architecture %s.", arch) - - # carry on with empty list from this point forward if no build targets are specified - if build_targets is None: - self.cfg['build_targets'] = build_targets = [] - - unknown_targets = [target for target in build_targets if target not in CLANG_TARGETS] - - if unknown_targets: - raise EasyBuildError("Some of the chosen build targets (%s) are not in %s.", - ', '.join(unknown_targets), ', '.join(CLANG_TARGETS)) - - if LooseVersion(self.version) < LooseVersion('3.4') and "R600" in build_targets: - raise EasyBuildError("Build target R600 not supported in < Clang-3.4") - - if LooseVersion(self.version) > LooseVersion('3.3') and "MBlaze" in build_targets: - raise EasyBuildError("Build target MBlaze is not supported anymore in > Clang-3.3") - def configure_step(self): """Run CMake for stage 1 Clang.""" @@ -306,8 +379,12 @@ def configure_step(self): else: self.cfg.update('configopts', "-DLLVM_ENABLE_ASSERTIONS=OFF") - if self.cfg["usepolly"]: - self.cfg.update('configopts', "-DLINK_POLLY_INTO_TOOLS=ON") + if 'polly' in self.cfg['llvm_projects']: + # Not exactly sure when this change took place, educated guess + if LooseVersion(self.version) >= LooseVersion('14'): + self.cfg.update('configopts', "-DLLVM_POLLY_LINK_INTO_TOOLS=ON") + else: + self.cfg.update('configopts', "-DLINK_POLLY_INTO_TOOLS=ON") # If Z3 is included as a dep, enable support in static analyzer (if enabled) if self.cfg["static_analyzer"] and LooseVersion(self.version) >= LooseVersion('9.0.0'): @@ -318,7 +395,7 @@ def configure_step(self): build_targets = self.cfg['build_targets'] - if self.cfg["usepolly"] and "NVPTX" in build_targets: + if 'polly' in self.cfg['llvm_projects'] and "NVPTX" in build_targets: self.cfg.update('configopts', "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON") self.cfg.update('configopts', '-DLLVM_TARGETS_TO_BUILD="%s"' % ';'.join(build_targets)) @@ -368,7 +445,12 @@ def configure_step(self): self.cfg.update('configopts', '-DLIBOMPTARGET_AMDGCN_GFXLIST=%s' % ' '.join(ec_amdgfx)) self.log.info("Configuring") - super(EB_Clang, self).configure_step(srcdir=self.llvm_src_dir) + + # directory structure has changed in version 14.x, cmake must start in llvm sub directory + if LooseVersion(self.version) >= LooseVersion('14'): + super(EB_Clang, self).configure_step(srcdir=os.path.join(self.llvm_src_dir, "llvm")) + else: + super(EB_Clang, self).configure_step(srcdir=self.llvm_src_dir) def disable_sanitizer_tests(self): """Disable the tests of all the sanitizers by removing the test directories from the build system""" @@ -383,7 +465,7 @@ def disable_sanitizer_tests(self): regex_subs = [(r'.*add_subdirectory\(lit_tests\).*', '')] apply_regex_substitutions(cmakelists, regex_subs) - # There is a common part seperate for the specific saniters, we disable all the common tests + # There is a common part seperate for the specific sanitizers, we disable all the common tests cmakelists = os.path.join('projects', 'compiler-rt', 'lib', 'sanitizer_common', 'CMakeLists.txt') regex_subs = [(r'.*add_subdirectory\(tests\).*', '')] apply_regex_substitutions(cmakelists, regex_subs) @@ -391,7 +473,10 @@ def disable_sanitizer_tests(self): else: # In Clang 3.6, the sanitizer tests are grouped together in one CMakeLists # We patch out adding the subdirectories with the sanitizer tests - cmakelists_tests = os.path.join(self.llvm_src_dir, 'projects', 'compiler-rt', 'test', 'CMakeLists.txt') + if LooseVersion(self.version) >= LooseVersion('14'): + cmakelists_tests = os.path.join(self.llvm_src_dir, 'compiler-rt', 'test', 'CMakeLists.txt') + else: + cmakelists_tests = os.path.join(self.llvm_src_dir, 'projects', 'compiler-rt', 'test', 'CMakeLists.txt') regex_subs = [] if LooseVersion(self.version) >= LooseVersion('5.0'): regex_subs.append((r'compiler_rt_test_runtime.*san.*', '')) @@ -407,21 +492,63 @@ def build_with_prev_stage(self, prev_obj, next_obj): mkdir(next_obj) change_dir(next_obj) - # Configure. - CC = os.path.join(prev_obj, 'bin', 'clang') - CXX = os.path.join(prev_obj, 'bin', 'clang++') + # Make sure clang and clang++ compilers from the previous stage are (temporarily) in PATH + # The call to prepare_rpath_wrappers() requires the compilers-to-be-wrapped on the PATH + # Also, the call to 'which' in later in this current function also requires that + orig_path = os.getenv('PATH') + prev_obj_path = os.path.join(prev_obj, 'bin') + setvar('PATH', prev_obj_path + ":" + orig_path) + + # If building with rpath, create RPATH wrappers for the Clang compilers for stage 2 and 3 + if build_option('rpath'): + my_clang_toolchain = Clang(name='Clang', version='1') + my_clang_toolchain.prepare_rpath_wrappers() + self.log.info("Prepared clang rpath wrappers") + + # RPATH wrappers add -Wl,rpath arguments to all command lines, including when it is just compiling + # Clang by default warns about that, and then some configure tests use -Werror which turns those warnings + # into errors. As a result, those configure tests fail, even though the compiler supports the requested + # functionality (e.g. the test that checks if -fPIC is supported would fail, and it compiles without + # resulting in relocation errors). + # See https://github.com/easybuilders/easybuild-easyblocks/pull/2799#issuecomment-1270621100 + # Here, we add -Wno-unused-command-line-argument to CXXFLAGS to avoid these warnings alltogether + cflags = os.getenv('CFLAGS') + cxxflags = os.getenv('CXXFLAGS') + setvar('CFLAGS', "%s %s" % (cflags, '-Wno-unused-command-line-argument')) + setvar('CXXFLAGS', "%s %s" % (cxxflags, '-Wno-unused-command-line-argument')) + + # determine full path to clang/clang++ (which may be wrapper scripts in case of RPATH linking) + clang = which('clang') + clangxx = which('clang++') - options = "-DCMAKE_INSTALL_PREFIX=%s " % self.installdir - options += "-DCMAKE_C_COMPILER='%s' " % CC - options += "-DCMAKE_CXX_COMPILER='%s' " % CXX - options += self.cfg['configopts'] - options += "-DCMAKE_BUILD_TYPE=%s" % self.build_type + # Configure. + options = [ + "-DCMAKE_INSTALL_PREFIX=%s " % self.installdir, + "-DCMAKE_C_COMPILER='%s' " % clang, + "-DCMAKE_CXX_COMPILER='%s' " % clangxx, + self.cfg['configopts'], + "-DCMAKE_BUILD_TYPE=%s " % self.build_type, + ] + + # Cmake looks for llvm-link by default in the same directory as the compiler + # However, when compiling with rpath, the clang 'compiler' is not actually the compiler, but the wrapper + # Clearly, the wrapper directory won't llvm-link. Thus, we pass the linker to be used by full path. + # See https://github.com/easybuilders/easybuild-easyblocks/pull/2799#issuecomment-1275916186 + if build_option('rpath'): + llvm_link = which('llvm-link') + options.append("-DLIBOMPTARGET_NVPTX_BC_LINKER=%s" % llvm_link) self.log.info("Configuring") - run_cmd("cmake %s %s" % (options, self.llvm_src_dir), log_all=True) + if LooseVersion(self.version) >= LooseVersion('14'): + run_cmd("cmake %s %s" % (' '.join(options), os.path.join(self.llvm_src_dir, "llvm")), log_all=True) + else: + run_cmd("cmake %s %s" % (' '.join(options), self.llvm_src_dir), log_all=True) self.log.info("Building") - run_cmd("make %s" % self.make_parallel_opts, log_all=True) + run_cmd("make %s VERBOSE=1" % self.make_parallel_opts, log_all=True) + + # restore $PATH + setvar('PATH', orig_path) def run_clang_tests(self, obj_dir): """Run Clang tests in specified directory (unless disabled).""" @@ -493,14 +620,44 @@ def post_install_step(self): # copy Python bindings here in post-install step so that it is not done more than once in multi_deps context if self.cfg['python_bindings']: - python_bindings_source_dir = os.path.join(self.llvm_src_dir, "tools", "clang", "bindings", "python") + if LooseVersion(self.version) >= LooseVersion('14'): + python_bindings_source_dir = os.path.join(self.llvm_src_dir, "clang", "bindings", "python") + else: + python_bindings_source_dir = os.path.join(self.llvm_src_dir, "tools", "clang", "bindings", "python") python_bindins_target_dir = os.path.join(self.installdir, 'lib', 'python') shutil.copytree(python_bindings_source_dir, python_bindins_target_dir) def sanity_check_step(self): """Custom sanity check for Clang.""" + custom_commands = ['clang --help', 'clang++ --help', 'llvm-config --cxxflags'] shlib_ext = get_shared_lib_ext() + + # Detect OpenMP support for CPU architecture + arch = get_cpu_architecture() + # Check architecture explicitly since Clang uses potentially + # different names + if arch == X86_64: + arch = 'x86_64' + elif arch == POWER: + arch = 'ppc64' + elif arch == AARCH64: + arch = 'aarch64' + else: + print_warning("Unknown CPU architecture (%s) for OpenMP and runtime libraries check!" % arch) + + if LooseVersion(self.version) >= LooseVersion('14'): + glob_pattern = os.path.join(self.installdir, 'lib', '%s-*' % arch) + matches = glob.glob(glob_pattern) + if matches: + directory = os.path.basename(matches[0]) + self.runtime_lib_path = os.path.join("lib", directory) + else: + print_warning("Could not find runtime library directory") + self.runtime_lib_path = "lib" + else: + self.runtime_lib_path = "lib" + custom_paths = { 'files': [ "bin/clang", "bin/clang++", "bin/llvm-ar", "bin/llvm-nm", "bin/llvm-as", "bin/opt", "bin/llvm-link", @@ -512,37 +669,42 @@ def sanity_check_step(self): if self.cfg['static_analyzer']: custom_paths['files'].extend(["bin/scan-build", "bin/scan-view"]) - if self.cfg['build_extra_clang_tools'] and LooseVersion(self.version) >= LooseVersion('3.4'): + if 'clang-tools-extra' in self.cfg['llvm_projects'] and LooseVersion(self.version) >= LooseVersion('3.4'): custom_paths['files'].extend(["bin/clang-tidy"]) - if self.cfg["usepolly"]: + if 'polly' in self.cfg['llvm_projects']: custom_paths['files'].extend(["lib/LLVMPolly.%s" % shlib_ext]) custom_paths['dirs'].extend(["include/polly"]) - if self.cfg["build_lld"]: + if 'lld' in self.cfg['llvm_projects']: custom_paths['files'].extend(["bin/lld"]) - if self.cfg["libcxx"]: - custom_paths['files'].extend(["lib/libc++.%s" % shlib_ext]) - custom_paths['files'].extend(["lib/libc++abi.%s" % shlib_ext]) + if 'lldb' in self.cfg['llvm_projects']: + custom_paths['files'].extend(["bin/lldb"]) + + if 'libunwind' in self.cfg['llvm_runtimes']: + custom_paths['files'].extend([os.path.join(self.runtime_lib_path, "libunwind.%s" % shlib_ext)]) + + if 'libcxx' in self.cfg['llvm_runtimes']: + custom_paths['files'].extend([os.path.join(self.runtime_lib_path, "libc++.%s" % shlib_ext)]) + + if 'libcxxabi' in self.cfg['llvm_runtimes']: + custom_paths['files'].extend([os.path.join(self.runtime_lib_path, "libc++abi.%s" % shlib_ext)]) + + if 'flang' in self.cfg['llvm_projects'] and LooseVersion(self.version) >= LooseVersion('15'): + flang_compiler = 'flang-new' + custom_paths['files'].extend(["bin/%s" % flang_compiler]) + custom_commands.extend(["%s --help" % flang_compiler]) if LooseVersion(self.version) >= LooseVersion('3.8'): custom_paths['files'].extend(["lib/libomp.%s" % shlib_ext, "lib/clang/%s/include/omp.h" % self.version]) - # Detect OpenMP support for CPU architecture - arch = get_cpu_architecture() - # Check architecture explicitly since Clang uses potentially - # different names - if arch == X86_64: - arch = 'x86_64' - elif arch == POWER: - arch = 'ppc64' - elif arch == AARCH64: - arch = 'aarch64' + if LooseVersion(self.version) >= LooseVersion('12'): + omp_target_libs = ["lib/libomptarget.%s" % shlib_ext, "lib/libomptarget.rtl.%s.%s" % (arch, shlib_ext)] else: - print_warning("Unknown CPU architecture (%s) for OpenMP library check!" % arch) - custom_paths['files'].extend(["lib/libomptarget.%s" % shlib_ext, - "lib/libomptarget.rtl.%s.%s" % (arch, shlib_ext)]) + omp_target_libs = ["lib/libomptarget.%s" % shlib_ext] + custom_paths['files'].extend(omp_target_libs) + # If building for CUDA check that OpenMP target library was created if 'NVPTX' in self.cfg['build_targets']: custom_paths['files'].append("lib/libomptarget.rtl.cuda.%s" % shlib_ext) @@ -582,7 +744,6 @@ def sanity_check_step(self): custom_paths['files'].extend(["lib/libomptarget-new-amdgpu-%s.bc" % gfx for gfx in self.cfg['amd_gfx_list']]) - custom_commands = ['clang --help', 'clang++ --help', 'llvm-config --cxxflags'] if self.cfg['python_bindings']: custom_paths['files'].extend([os.path.join("lib", "python", "clang", "cindex.py")]) custom_commands.extend(["python -c 'import clang'"]) @@ -598,3 +759,15 @@ def make_module_extra(self): if self.cfg['python_bindings']: txt += self.module_generator.prepend_paths('PYTHONPATH', os.path.join("lib", "python")) return txt + + def make_module_req_guess(self): + """ + Clang can find its own headers and libraries but the .so's need to be in LD_LIBRARY_PATH + """ + guesses = super(EB_Clang, self).make_module_req_guess() + guesses.update({ + 'CPATH': [], + 'LIBRARY_PATH': [], + 'LD_LIBRARY_PATH': ['lib', 'lib64', self.runtime_lib_path], + }) + return guesses diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 04ebda0da0..0104ab64f9 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -163,8 +163,7 @@ def sanity_check_step(self): custom_paths = { 'files': ['bin/clang', 'bin/lld', 'lib/libomp.%s' % shlib_ext, 'lib/libomptarget.rtl.amdgpu.%s' % shlib_ext, 'lib/libomptarget.%s' % shlib_ext], - 'dirs': ['amdgcn/bitcode', 'include/clang', 'include/lld', 'include/llvm', - 'lib/libdevice'], + 'dirs': ['amdgcn/bitcode', 'include/clang', 'include/lld', 'include/llvm'], } custom_commands = ['clang --help', 'clang++ --help'] @@ -176,7 +175,8 @@ def sanity_check_step(self): # Check that all AMD GFX libraries were built for gfx in self.amd_gfx_archs: - custom_paths['files'].extend([os.path.join(libdevice, 'lib%s-amdgcn-%s.bc' % (x, gfx)) for x in libs]) + if LooseVersion(self.version) < LooseVersion("5.2"): + custom_paths['files'].extend([os.path.join(libdevice, 'lib%s-amdgcn-%s.bc' % (x, gfx)) for x in libs]) if LooseVersion(self.version) >= LooseVersion("5"): custom_paths['files'].append(os.path.join('lib', 'libomptarget-amdgcn-%s.bc' % gfx)) custom_paths['files'].append(os.path.join('lib', 'libomptarget-new-amdgpu-%s.bc' % gfx)) @@ -267,6 +267,11 @@ def _configure_omp(self, component): '', ]) + if LooseVersion(self.version) >= LooseVersion("5.2"): + component['configopts'] += ' '.join([ + "-DDEVICELIBS_ROOT=%s" % self.device_lib_path, + ]) + if get_software_root('CUDA'): llvm_link = os.path.join(self.installdir, 'bin', 'llvm-link') cuda_path = os.path.join(self.installdir, 'bin', 'clang++') @@ -288,4 +293,10 @@ def _configure_aomp_extras(self, component): "-DAOMP_STANDALONE_BUILD=1", "-DROCDL=%s" % self.device_lib_path, "-DROCM_DIR=%s" % self.installdir, + '', ]) + + if LooseVersion(self.version) >= LooseVersion("5.2"): + component['configopts'] += ' '.join([ + "-DAOMP_VERSION_STRING=%s" % self.version, + ]) diff --git a/easybuild/easyblocks/c/cmake.py b/easybuild/easyblocks/c/cmake.py index 5d3ff515d1..2c9ff85ddc 100644 --- a/easybuild/easyblocks/c/cmake.py +++ b/easybuild/easyblocks/c/cmake.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2022 Alexander Grund +# Copyright 2020-2023 Alexander Grund # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/code_server.py b/easybuild/easyblocks/c/code_server.py index 6ae209c498..b975340bf8 100644 --- a/easybuild/easyblocks/c/code_server.py +++ b/easybuild/easyblocks/c/code_server.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/comsol.py b/easybuild/easyblocks/c/comsol.py index 4fa7cdfbfd..38d778a623 100644 --- a/easybuild/easyblocks/c/comsol.py +++ b/easybuild/easyblocks/c/comsol.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -114,6 +114,11 @@ def install_step(self): # make sure setup script is executable adjust_permissions(setup_script, stat.S_IXUSR) + # make sure binaries in arch bindir is executable + archpath = os.path.join(self.start_dir, 'bin', 'glnxa64') + adjust_permissions(os.path.join(archpath, 'inflate'), stat.S_IXUSR) + adjust_permissions(os.path.join(archpath, 'setuplauncher'), stat.S_IXUSR) + # make sure $DISPLAY is not defined, which may lead to (hard to trace) problems # this is a workaround for not being able to specify --nodisplay to the install scripts env.unset_env_vars(['DISPLAY']) diff --git a/easybuild/easyblocks/c/cp2k.py b/easybuild/easyblocks/c/cp2k.py index acfbee3a39..c9f4171921 100644 --- a/easybuild/easyblocks/c/cp2k.py +++ b/easybuild/easyblocks/c/cp2k.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cplex.py b/easybuild/easyblocks/c/cplex.py index 02c0efee3a..cc6ec0e52f 100644 --- a/easybuild/easyblocks/c/cplex.py +++ b/easybuild/easyblocks/c/cplex.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cppcheck.py b/easybuild/easyblocks/c/cppcheck.py index b70d819ce4..ec43ceda09 100644 --- a/easybuild/easyblocks/c/cppcheck.py +++ b/easybuild/easyblocks/c/cppcheck.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2022 Forschungszentrum Juelich GmbH +# Copyright 2016-2023 Forschungszentrum Juelich GmbH # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/crispr_dav.py b/easybuild/easyblocks/c/crispr_dav.py index 86c172926b..b528fd1056 100644 --- a/easybuild/easyblocks/c/crispr_dav.py +++ b/easybuild/easyblocks/c/crispr_dav.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2022 Ghent University +# Copyright 2020-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cryptography.py b/easybuild/easyblocks/c/cryptography.py index a785ecac84..bfb6688358 100644 --- a/easybuild/easyblocks/c/cryptography.py +++ b/easybuild/easyblocks/c/cryptography.py @@ -1,5 +1,5 @@ ## -# Copyright 2017-2022 Ghent University +# Copyright 2017-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index 5fcf093ab3..32629aab40 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py new file mode 100644 index 0000000000..e97db8e2c0 --- /dev/null +++ b/easybuild/easyblocks/c/cudacompat.py @@ -0,0 +1,266 @@ +## +# Copyright 2012-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for CUDA compat libraries, implemented as an easyblock + +Ref: https://docs.nvidia.com/deploy/cuda-compatibility/index.html#manually-installing-from-runfile + +@author: Alexander Grund (TU Dresden) +""" + +import os +from distutils.version import LooseVersion + +from easybuild.easyblocks.generic.binary import Binary +from easybuild.framework.easyconfig import CUSTOM, MANDATORY +from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.config import build_option, IGNORE +from easybuild.tools.filetools import copy_file, find_glob_pattern, mkdir, symlink, which +from easybuild.tools.run import run_cmd + + +class EB_CUDAcompat(Binary): + """ + Support for installing CUDA compat libraries. + """ + + @staticmethod + def extra_options(): + """Add variable for driver version""" + extra_vars = Binary.extra_options() + extra_vars.update({ + 'compatible_driver_versions': [None, "Minimum (system) CUDA driver versions which are compatible", CUSTOM], + 'nv_version': [None, "Version of the driver package", MANDATORY], + }) + # We don't need the extract and install step from the Binary EasyBlock + del extra_vars['extract_sources'] + del extra_vars['install_cmd'] + # And also no need to modify the PATH + del extra_vars['prepend_to_path'] + return extra_vars + + def __init__(self, *args, **kwargs): + """Initialize custom class variables for CUDACompat.""" + super(EB_CUDAcompat, self).__init__(*args, **kwargs) + self._has_nvidia_smi = None + + @property + def has_nvidia_smi(self): + """Return whether the system has nvidia-smi and print a warning once if not""" + if self._has_nvidia_smi is None: + self._has_nvidia_smi = which('nvidia-smi', on_error=IGNORE) is not None + if not self._has_nvidia_smi: + print_warning('Could not find nvidia-smi. Assuming a system without GPUs and skipping checks!') + return self._has_nvidia_smi + + def _run_nvidia_smi(self, args): + """ + Run nvidia-smi with the given argument(s) and return the output. + + Also does sensible logging. + Raises RuntimeError on failure. + """ + if not self.has_nvidia_smi: + raise RuntimeError('Could not find nvidia-smi.') + cmd = 'nvidia-smi ' + args + out, ec = run_cmd(cmd, log_ok=False, log_all=False, regexp=False) + if ec != 0: + raise RuntimeError("`%s` returned exit code %s with output:\n%s" % (cmd, ec, out)) + else: + self.log.info('`%s` succeeded with output:\n%s' % (cmd, out)) + return out.strip().split('\n') + + def prepare_step(self, *args, **kwargs): + """Parse and check the compatible_driver_versions value of the EasyConfig""" + compatible_driver_versions = self.cfg.get('compatible_driver_versions') + if compatible_driver_versions: + try: + # Create a dictionary with the major version as the keys + self.compatible_driver_version_map = { + int(v.split('.', 1)[0]): v + for v in compatible_driver_versions + } + except ValueError: + raise EasyBuildError("Invalid format of compatible_driver_versions. " + "Expected numeric major versions, got '%s'", compatible_driver_versions) + else: + self.compatible_driver_version_map = None + if 'LD_LIBRARY_PATH' in (build_option('filter_env_vars') or []): + raise EasyBuildError("This module relies on setting $LD_LIBRARY_PATH, " + "so you need to remove this variable from --filter-env-vars") + super(EB_CUDAcompat, self).prepare_step(*args, **kwargs) + + def fetch_step(self, *args, **kwargs): + """Check for EULA acceptance prior to getting sources.""" + # EULA for NVIDIA driver must be accepted via --accept-eula-for EasyBuild configuration option, + # or via 'accept_eula = True' in easyconfig file + self.check_accepted_eula( + name='NVIDIA-driver', + more_info='https://www.nvidia.com/content/DriverDownload-March2009/licence.php?lang=us' + ) + return super(EB_CUDAcompat, self).fetch_step(*args, **kwargs) + + def extract_step(self): + """Extract the files without running the installer.""" + execpath = self.src[0]['path'] + tmpdir = os.path.join(self.builddir, 'tmp') + targetdir = os.path.join(self.builddir, 'extracted') + run_cmd("/bin/sh " + execpath + " --extract-only --tmpdir='%s' --target '%s'" % (tmpdir, targetdir)) + self.src[0]['finalpath'] = targetdir + + def test_step(self): + """ + Check for a compatible driver version if the EC has that information. + + This can be skipped with `--skip-test-step`. + """ + + if self.compatible_driver_version_map and self.has_nvidia_smi: + try: + out_lines = self._run_nvidia_smi('--query-gpu=driver_version --format=csv,noheader') + if not out_lines or not out_lines[0]: + raise RuntimeError('nvidia-smi did not find any GPUs on the system') + driver_version = out_lines[0] + version_parts = driver_version.split('.') + if len(version_parts) < 3 or any(not v.isdigit() for v in version_parts): + raise RuntimeError("Expected the version to be in format x.y.z (all numeric) " + "but got '%s'" % driver_version) + except RuntimeError as err: + self.log.warning("Failed to get the CUDA driver version: %s", err) + driver_version = None + + if not driver_version: + print_warning('Failed to determine the CUDA driver version, so skipping the compatibility check!') + else: + driver_version_major = int(driver_version.split('.', 1)[0]) + compatible_driver_versions = ', '.join(sorted(self.compatible_driver_version_map.values())) + try: + min_required_version = self.compatible_driver_version_map[driver_version_major] + except KeyError: + raise EasyBuildError('The installed CUDA driver %s is not a supported branch/major version for ' + '%s %s. Supported drivers: %s', + driver_version, self.name, self.version, compatible_driver_versions) + if LooseVersion(driver_version) < min_required_version: + raise EasyBuildError('The installed CUDA driver %s is to old for %s %s, ' + 'need at least %s. Supported drivers: %s', + driver_version, self.name, self.version, + min_required_version, compatible_driver_versions) + else: + self.log.info('The installed CUDA driver %s appears to be supported.', driver_version) + + return super(EB_CUDAcompat, self).test_step() + + def install_step(self): + """Install CUDA compat libraries by copying library files and creating the symlinks.""" + libdir = os.path.join(self.installdir, 'lib') + mkdir(libdir) + + # From https://docs.nvidia.com/deploy/cuda-compatibility/index.html#installing-from-network-repo: + # The cuda-compat package consists of the following files: + # - libcuda.so.* - the CUDA Driver + # - libnvidia-nvvm.so.* - JIT LTO ( CUDA 11.5 and later only) + # - libnvidia-ptxjitcompiler.so.* - the JIT (just-in-time) compiler for PTX files + + library_globs = [ + 'libcuda.so.*', + 'libnvidia-ptxjitcompiler.so.*', + ] + if LooseVersion(self.version) >= '11.5': + library_globs.append('libnvidia-nvvm.so.*') + + startdir = self.cfg['start_dir'] + nv_version = self.cfg['nv_version'] + for library_glob in library_globs: + library_path = find_glob_pattern(os.path.join(startdir, library_glob)) + library = os.path.basename(library_path) + # Sanity check the version + if library_glob == 'libcuda.so.*': + library_version = library.split('.', 2)[2] + if library_version != nv_version: + raise EasyBuildError('Expected driver version %s (from nv_version) but found %s ' + '(determined from file %s)', nv_version, library_version, library_path) + + copy_file(library_path, os.path.join(libdir, library)) + if library.endswith('.' + nv_version): + # E.g. libcuda.so.510.73.08 -> libcuda.so.1 + versioned_symlink = library[:-len(nv_version)] + '1' + else: + # E.g. libnvidia-nvvm.so.4.0.0 -> libnvidia-nvvm.so.4 + versioned_symlink = library.rsplit('.', 2)[0] + symlink(library, os.path.join(libdir, versioned_symlink), use_abspath_source=False) + # E.g. libcuda.so.1 -> libcuda.so + unversioned_symlink = versioned_symlink.rsplit('.', 1)[0] + symlink(versioned_symlink, os.path.join(libdir, unversioned_symlink), use_abspath_source=False) + + def make_module_req_guess(self): + """Don't try to guess anything.""" + return dict() + + def make_module_extra(self): + """Skip the changes from the Binary EasyBlock and (only) set LD_LIBRARY_PATH.""" + + txt = super(Binary, self).make_module_extra() + txt += self.module_generator.prepend_paths('LD_LIBRARY_PATH', 'lib') + return txt + + def sanity_check_step(self): + """Check for core files (unversioned libs, symlinks)""" + libraries = [ + 'libcuda.so', + 'libnvidia-ptxjitcompiler.so', + ] + if LooseVersion(self.version) >= '11.5': + libraries.append('libnvidia-nvvm.so') + custom_paths = { + 'files': [os.path.join(self.installdir, 'lib', x) for x in libraries], + 'dirs': ['lib', 'lib64'], + } + super(EB_CUDAcompat, self).sanity_check_step(custom_paths=custom_paths) + + if self.has_nvidia_smi: + fake_mod_data = None + + # skip loading of fake module when using --sanity-check-only, load real module instead + if build_option('sanity_check_only'): + self.load_module() + elif not self.dry_run: + fake_mod_data = self.load_fake_module(purge=True, verbose=True) + + try: + out_lines = self._run_nvidia_smi('--query --display=COMPUTE') + + if fake_mod_data: + self.clean_up_fake_module(fake_mod_data) + + cuda_version = next((line.rsplit(' ', 1)[1] for line in out_lines if line.startswith('CUDA')), None) + if not cuda_version: + raise RuntimeError('Failed to find CUDA version!') + self.log.info('CUDA version (as per nvidia-smi) after loading the module: ' + cuda_version) + if LooseVersion(cuda_version) != self.version: + raise RuntimeError('Reported CUDA version %s is not %s' % (cuda_version, self.version)) + except RuntimeError as err: + if fake_mod_data: + self.clean_up_fake_module(fake_mod_data) + raise EasyBuildError('Version check via nvidia-smi after loading the module failed: %s', err) diff --git a/easybuild/easyblocks/c/cudnn.py b/easybuild/easyblocks/c/cudnn.py index a843f8ae78..39e04fe8d5 100644 --- a/easybuild/easyblocks/c/cudnn.py +++ b/easybuild/easyblocks/c/cudnn.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cufflinks.py b/easybuild/easyblocks/c/cufflinks.py index d1542a70cc..62f2cb7bf2 100644 --- a/easybuild/easyblocks/c/cufflinks.py +++ b/easybuild/easyblocks/c/cufflinks.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/d/db.py b/easybuild/easyblocks/d/db.py index f62f02c116..065df60a14 100644 --- a/easybuild/easyblocks/d/db.py +++ b/easybuild/easyblocks/d/db.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/d/dl_poly_classic.py b/easybuild/easyblocks/d/dl_poly_classic.py index 6975df2747..823df870b9 100644 --- a/easybuild/easyblocks/d/dl_poly_classic.py +++ b/easybuild/easyblocks/d/dl_poly_classic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/d/dm_reverb.py b/easybuild/easyblocks/d/dm_reverb.py index 18012d98bb..802edca1d2 100644 --- a/easybuild/easyblocks/d/dm_reverb.py +++ b/easybuild/easyblocks/d/dm_reverb.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -94,7 +94,8 @@ def build_step(self, *args, **kwargs): # print full compilation commands bazel_build_opts += " --subcommands" - bazel_cmd = "bazel %s build %s %s" % (bazel_opts, bazel_build_opts, bazel_build_pkg) + bazel_cmd = "%s bazel %s build %s %s" % (self.cfg['prebuildopts'], bazel_opts, bazel_build_opts, + bazel_build_pkg) return run_cmd(bazel_cmd, log_all=True, simple=True, log_output=True) @@ -121,6 +122,7 @@ def install_step(self, *args, **kwargs): 'installopts': self.cfg['installopts'], 'loc': whl_path, 'prefix': self.installdir, + 'python': self.python_cmd, } return super(EB_dm_minus_reverb, self).install_step(*args, **kwargs) diff --git a/easybuild/easyblocks/d/dolfin.py b/easybuild/easyblocks/d/dolfin.py index c9e78d23dc..6cb323636c 100644 --- a/easybuild/easyblocks/d/dolfin.py +++ b/easybuild/easyblocks/d/dolfin.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/d/doris.py b/easybuild/easyblocks/d/doris.py index 86f0a27601..cac0909ba1 100644 --- a/easybuild/easyblocks/d/doris.py +++ b/easybuild/easyblocks/d/doris.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/d/doxygen.py b/easybuild/easyblocks/d/doxygen.py index 8498fed336..7a8a16365d 100644 --- a/easybuild/easyblocks/d/doxygen.py +++ b/easybuild/easyblocks/d/doxygen.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index 5949a181f8..f4b52ff921 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,9 +33,9 @@ import sys from distutils.version import LooseVersion -from easybuild.easyblocks.generic.pythonpackage import PythonPackage +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pip_version from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import read_file +from easybuild.tools.filetools import apply_regex_substitutions, change_dir, read_file from easybuild.tools.modules import get_software_root_env_var_name from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.utilities import flatten @@ -66,6 +66,27 @@ def __init__(self, *args, **kwargs): # consider setuptools first, in case it is listed as a sources self.easybuild_pkgs.insert(0, 'setuptools') + # opt-in to using pip for recent version of EasyBuild, if: + # - EasyBuild is being installed for Python >= 3.6; + # - pip is available, and recent enough (>= 21.0); + # - use_pip is not specified; + pyver = sys.version.split(' ')[0] + self.log.info("Python version: %s", pyver) + if sys.version_info >= (3, 6) and self.cfg['use_pip'] is None: + # try to determine pip version, ignore any failures that occur while doing so; + # problems may occur due changes in environment ($PYTHONPATH, etc.) + pip_version = None + try: + pip_version = det_pip_version(python_cmd=sys.executable) + self.log.info("Found Python v%s + pip: %s", pyver, pip_version) + except Exception as err: + self.log.warning("Failed to determine pip version: %s", err) + + if pip_version and LooseVersion(pip_version) >= LooseVersion('21.0'): + self.log.info("Auto-enabling use of pip to install EasyBuild!") + self.cfg['use_pip'] = True + self.determine_install_command() + # Override this function since we want to respect the user choice for the python installation to use # (which can be influenced by EB_PYTHON and EB_INSTALLPYTHON) def prepare_python(self): @@ -92,6 +113,21 @@ def build_step(self): """No building for EasyBuild packages.""" pass + def fix_easyconfigs_setup_py_setuptools61(self): + """ + Patch setup.py of easybuild-easyconfigs package if needed to make sure that installation works + for recent setuptools versions (>= 61.0). + """ + # cfr. https://github.com/easybuilders/easybuild-easyconfigs/pull/15206 + cwd = os.getcwd() + regex = re.compile(r'packages=\[\]') + setup_py_txt = read_file('setup.py') + if regex.search(setup_py_txt) is None: + self.log.info("setup.py at %s needs to be fixed to install with setuptools >= 61.0", cwd) + apply_regex_substitutions('setup.py', [(r'^setup\(', 'setup(packages=[],')]) + else: + self.log.info("setup.py at %s does not need to be fixed to install with setuptools >= 61.0", cwd) + def install_step(self): """Install EasyBuild packages one by one.""" try: @@ -108,7 +144,11 @@ def install_step(self): else: self.log.info("Installing package %s", pkg) - os.chdir(os.path.join(self.builddir, seldirs[0])) + change_dir(os.path.join(self.builddir, seldirs[0])) + + if pkg == 'easybuild-easyconfigs': + self.fix_easyconfigs_setup_py_setuptools61() + super(EB_EasyBuildMeta, self).install_step() except OSError as err: @@ -117,6 +157,8 @@ def install_step(self): def post_install_step(self): """Remove setuptools.pth file that hard includes a system-wide (site-packages) path, if it is there.""" + super(EB_EasyBuildMeta, self).post_install_step() + setuptools_pth = os.path.join(self.installdir, self.pylibdir, 'setuptools.pth') if os.path.exists(setuptools_pth): setuptools_pth_txt = read_file(setuptools_pth) diff --git a/easybuild/easyblocks/e/egglib.py b/easybuild/easyblocks/e/egglib.py index 3233fd0ca2..f4aa262042 100644 --- a/easybuild/easyblocks/e/egglib.py +++ b/easybuild/easyblocks/e/egglib.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/e/eigen.py b/easybuild/easyblocks/e/eigen.py index acdfbc168d..70dc6a4e36 100644 --- a/easybuild/easyblocks/e/eigen.py +++ b/easybuild/easyblocks/e/eigen.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index cd488cade1..41b81e15e8 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -1,6 +1,6 @@ ## -# Copyright 2009-2022 Ghent University -# Copyright 2019-2022 Micael Oliveira +# Copyright 2009-2023 Ghent University +# Copyright 2019-2023 Micael Oliveira # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,17 +30,21 @@ @author: Kenneth Hoste (Ghent University) """ import os +from distutils.version import LooseVersion +import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM +from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC +from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions +from easybuild.tools.modules import get_software_root from easybuild.tools.systemtools import get_cpu_features, get_shared_lib_ext from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC from easybuild.tools.utilities import nub - ELPA_CPU_FEATURE_FLAGS = ['avx', 'avx2', 'avx512f', 'vsx', 'sse4_2'] @@ -176,6 +180,51 @@ def run_all_steps(self, *args, **kwargs): return super(EB_ELPA, self).run_all_steps(*args, **kwargs) + def configure_step(self): + """Configure step for ELPA""" + + # Add nvidia GPU support if requested + cuda_root = get_software_root('CUDA') + self.log.info("Got cuda root: %s", cuda_root) + if cuda_root: + self.cfg.update('configopts', '--enable-nvidia-gpu') + self.cfg.update('configopts', '--with-cuda-path=%s' % cuda_root) + self.cfg.update('configopts', '--with-cuda-sdk-path=%s' % cuda_root) + + cuda_cc = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities'] + if not cuda_cc: + raise EasyBuildError('List of CUDA compute capabilities must be specified, either via ' + 'cuda_compute_capabilities easyconfig parameter or via ' + '--cuda-compute-capabilities') + + # ELPA's --with-NVIDIA-GPU-compute-capability only accepts a single architecture + if len(cuda_cc) != 1: + raise EasyBuildError('ELPA currently only supports specifying one CUDA architecture when ' + 'building. You specified cuda-compute-capabilities: %s', cuda_cc) + cuda_cc = cuda_cc[0] + cuda_cc_string = cuda_cc.replace('.', '') + self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=sm_%s' % cuda_cc_string) + self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) + # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards + if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'): + self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') + + # From v2022.05.001 onwards, the config complains if CPP is not set, resulting in non-zero exit of configure + # C preprocessor to use for given comp_fam + cpp_dict = { + TC_CONSTANT_GCC: 'cpp', + TC_CONSTANT_INTELCOMP: 'cpp', + } + comp_fam = self.toolchain.comp_family() + if comp_fam in cpp_dict: + env.setvar('CPP', cpp_dict[comp_fam]) + else: + raise EasyBuildError('ELPA EasyBlock does not know which C preprocessor to use for the ' + 'current compiler family (%s). Please add the correct preprocessor ' + 'for this compiler family to cpp_dict in the ELPA EasyBlock', comp_fam) + + super(EB_ELPA, self).configure_step() + def patch_step(self, *args, **kwargs): """Patch manual_cpp script to avoid using hardcoded /usr/bin/python.""" super(EB_ELPA, self).patch_step(*args, **kwargs) diff --git a/easybuild/easyblocks/e/elsi.py b/easybuild/easyblocks/e/elsi.py index e216833760..e3e26dea00 100644 --- a/easybuild/easyblocks/e/elsi.py +++ b/easybuild/easyblocks/e/elsi.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/e/epd.py b/easybuild/easyblocks/e/epd.py index d278afaa3b..26e2610fdb 100644 --- a/easybuild/easyblocks/e/epd.py +++ b/easybuild/easyblocks/e/epd.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index cc3986ca45..03ff400619 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,11 +39,20 @@ from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.framework.easyconfig import CUSTOM class EB_ESMF(ConfigureMake): """Support for building/installing ESMF.""" + @staticmethod + def extra_options(): + """Custom easyconfig parameters for ESMF""" + extra_vars = { + 'disable_lapack': [False, 'Disable external LAPACK - True or False', CUSTOM] + } + return ConfigureMake.extra_options(extra_vars) + def configure_step(self): """Custom configuration procedure for ESMF through environment variables.""" @@ -75,13 +84,19 @@ def configure_step(self): env.setvar('ESMF_COMM', comm) # specify decent LAPACK lib - env.setvar('ESMF_LAPACK', 'user') - ldflags = os.getenv('LDFLAGS') - liblapack = os.getenv('LIBLAPACK_MT') or os.getenv('LIBLAPACK') - if liblapack is None: - raise EasyBuildError("$LIBLAPACK(_MT) not defined, no BLAS/LAPACK in %s toolchain?", self.toolchain.name) + if self.cfg['disable_lapack']: + env.setvar('ESMF_LAPACK', 'OFF') + # MOAB can't be built without LAPACK + env.setvar('ESMF_MOAB', 'OFF') else: - env.setvar('ESMF_LAPACK_LIBS', ldflags + ' ' + liblapack) + env.setvar('ESMF_LAPACK', 'user') + ldflags = os.getenv('LDFLAGS') + liblapack = os.getenv('LIBLAPACK_MT') or os.getenv('LIBLAPACK') + if liblapack is None: + msg = "$LIBLAPACK(_MT) not defined, no BLAS/LAPACK in %s toolchain?" + raise EasyBuildError(msg, self.toolchain.name) + else: + env.setvar('ESMF_LAPACK_LIBS', ldflags + ' ' + liblapack) # specify netCDF netcdf = get_software_root('netCDF') diff --git a/easybuild/easyblocks/e/espresso.py b/easybuild/easyblocks/e/espresso.py index b8510c974e..3d1795fe41 100644 --- a/easybuild/easyblocks/e/espresso.py +++ b/easybuild/easyblocks/e/espresso.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Josh Berryman , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/e/extrae.py b/easybuild/easyblocks/e/extrae.py index 39006eb433..fcb10677a1 100644 --- a/easybuild/easyblocks/e/extrae.py +++ b/easybuild/easyblocks/e/extrae.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/faststructure.py b/easybuild/easyblocks/f/faststructure.py index cfafbb7d65..5504335056 100644 --- a/easybuild/easyblocks/f/faststructure.py +++ b/easybuild/easyblocks/f/faststructure.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/fdtd_solutions.py b/easybuild/easyblocks/f/fdtd_solutions.py index 167885eaa1..4e044d2788 100644 --- a/easybuild/easyblocks/f/fdtd_solutions.py +++ b/easybuild/easyblocks/f/fdtd_solutions.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/ferret.py b/easybuild/easyblocks/f/ferret.py index 8bc8380939..37e38e9a0b 100644 --- a/easybuild/easyblocks/f/ferret.py +++ b/easybuild/easyblocks/f/ferret.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/fftw.py b/easybuild/easyblocks/f/fftw.py index 8594e278f2..6f10c711c9 100644 --- a/easybuild/easyblocks/f/fftw.py +++ b/easybuild/easyblocks/f/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/fftwmpi.py b/easybuild/easyblocks/f/fftwmpi.py index bb4e467287..3736362493 100644 --- a/easybuild/easyblocks/f/fftwmpi.py +++ b/easybuild/easyblocks/f/fftwmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/flex.py b/easybuild/easyblocks/f/flex.py index b65b008cc5..a7fd414f8d 100644 --- a/easybuild/easyblocks/f/flex.py +++ b/easybuild/easyblocks/f/flex.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/flexiblas.py b/easybuild/easyblocks/f/flexiblas.py index b6edc0226c..ab6fabfc50 100644 --- a/easybuild/easyblocks/f/flexiblas.py +++ b/easybuild/easyblocks/f/flexiblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2022 Ghent University +# Copyright 2021-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -138,7 +138,7 @@ def configure_step(self): # to allow easyconfig to override specifies settings for key, value in sorted(configopts.items()): opt = '-D%s=' % key - if key not in self.cfg['configopts']: + if opt not in self.cfg['configopts']: self.cfg.update('configopts', opt + "'%s'" % value) # specify compiler commands with absolute paths, to ensure that RPATH wrapper scripts are used diff --git a/easybuild/easyblocks/f/fluent.py b/easybuild/easyblocks/f/fluent.py index e67d891589..5976f8ea4f 100644 --- a/easybuild/easyblocks/f/fluent.py +++ b/easybuild/easyblocks/f/fluent.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/foldx.py b/easybuild/easyblocks/f/foldx.py index 1a4300f18b..8166c9b68a 100644 --- a/easybuild/easyblocks/f/foldx.py +++ b/easybuild/easyblocks/f/foldx.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/freefem.py b/easybuild/easyblocks/f/freefem.py index 7caad3acb5..15c0fdd31c 100644 --- a/easybuild/easyblocks/f/freefem.py +++ b/easybuild/easyblocks/f/freefem.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/freesurfer.py b/easybuild/easyblocks/f/freesurfer.py index 6419caf8f0..c60059c149 100644 --- a/easybuild/easyblocks/f/freesurfer.py +++ b/easybuild/easyblocks/f/freesurfer.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/freetype.py b/easybuild/easyblocks/f/freetype.py index feddc3a29f..fa3487a1cd 100644 --- a/easybuild/easyblocks/f/freetype.py +++ b/easybuild/easyblocks/f/freetype.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/fsl.py b/easybuild/easyblocks/f/fsl.py index 94a8350415..e8ebaa9b21 100644 --- a/easybuild/easyblocks/f/fsl.py +++ b/easybuild/easyblocks/f/fsl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/g2clib.py b/easybuild/easyblocks/g/g2clib.py index 48eaa80aae..15ad20f57b 100644 --- a/easybuild/easyblocks/g/g2clib.py +++ b/easybuild/easyblocks/g/g2clib.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/g2lib.py b/easybuild/easyblocks/g/g2lib.py index 55f708f8f1..8b3e9643de 100644 --- a/easybuild/easyblocks/g/g2lib.py +++ b/easybuild/easyblocks/g/g2lib.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/gamess_us.py b/easybuild/easyblocks/g/gamess_us.py index 4d658a7c0e..973309d0c9 100644 --- a/easybuild/easyblocks/g/gamess_us.py +++ b/easybuild/easyblocks/g/gamess_us.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/gate.py b/easybuild/easyblocks/g/gate.py index 51338d19e2..572204b836 100644 --- a/easybuild/easyblocks/g/gate.py +++ b/easybuild/easyblocks/g/gate.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index dbe1039c54..7ee469763e 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -48,7 +48,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions, change_dir, copy_file, move_file, symlink -from easybuild.tools.filetools import which, write_file +from easybuild.tools.filetools import which, read_file, write_file from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import RISCV, check_os_dependency, get_cpu_architecture, get_cpu_family @@ -357,8 +357,19 @@ def configure_step(self): # (see https://gcc.gnu.org/install/configure.html) self.cfg.update('configopts', '--with-sysroot=%s' % sysroot) - # avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c - apply_regex_substitutions(os.path.join('gcc', 'gcc.c'), [('--sysroot=%R', '')]) + libc_so_candidates = [os.path.join(sysroot, x, 'libc.so') for x in + ['lib', 'lib64', os.path.join('usr', 'lib'), os.path.join('usr', 'lib64')]] + for libc_so in libc_so_candidates: + if os.path.exists(libc_so): + # only patch gcc.c or gcc.cc if entries in libc.so are prefixed with sysroot + if '\nGROUP ( ' + sysroot in read_file(libc_so): + gccfile = os.path.join('gcc', 'gcc.c') + # renamed to gcc.cc in GCC 12 + if not os.path.exists(gccfile): + gccfile = os.path.join('gcc', 'gcc.cc') + # avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c* + apply_regex_substitutions(gccfile, [('--sysroot=%R', '')]) + break # prefix dynamic linkers with sysroot # this patches lines like: diff --git a/easybuild/easyblocks/g/gctf.py b/easybuild/easyblocks/g/gctf.py index 0e3ca2d16a..9e2ca40b86 100644 --- a/easybuild/easyblocks/g/gctf.py +++ b/easybuild/easyblocks/g/gctf.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2022 Ghent University +# Copyright 2019-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (https://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/geant4.py b/easybuild/easyblocks/g/geant4.py index 77b8c17f4d..ca431ca946 100644 --- a/easybuild/easyblocks/g/geant4.py +++ b/easybuild/easyblocks/g/geant4.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/ghc.py b/easybuild/easyblocks/g/ghc.py index 2112cd6c20..20c6edcd75 100644 --- a/easybuild/easyblocks/g/ghc.py +++ b/easybuild/easyblocks/g/ghc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/go.py b/easybuild/easyblocks/g/go.py index 2a19f2c468..932df72f3e 100644 --- a/easybuild/easyblocks/g/go.py +++ b/easybuild/easyblocks/g/go.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2022 Ghent University +# Copyright 2014-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index fbffd6535d..973b3f323a 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/gurobi.py b/easybuild/easyblocks/g/gurobi.py index 34e1eb9a2a..49147d9a2f 100644 --- a/easybuild/easyblocks/g/gurobi.py +++ b/easybuild/easyblocks/g/gurobi.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -55,22 +55,25 @@ def __init__(self, *args, **kwargs): """Easyblock constructor, define custom class variables specific to Gurobi.""" super(EB_Gurobi, self).__init__(*args, **kwargs) - self.license_file = self.cfg['license_file'] + # make sure license file is available + self.orig_license_file = self.cfg['license_file'] + if self.orig_license_file is None: + self.orig_license_file = os.getenv('EB_GUROBI_LICENSE_FILE', None) if self.cfg['copy_license_file']: self.license_file = os.path.join(self.installdir, 'gurobi.lic') + else: + self.license_file = self.orig_license_file def install_step(self): """Install Gurobi and license file.""" - - # make sure license file is available - if self.cfg['license_file'] is None or not os.path.exists(self.cfg['license_file']): - raise EasyBuildError("No existing license file specified: %s", self.cfg['license_file']) - super(EB_Gurobi, self).install_step() if self.cfg['copy_license_file']: - copy_file(self.cfg['license_file'], self.license_file) + if self.orig_license_file is None or not os.path.exists(self.orig_license_file): + raise EasyBuildError("No existing license file specified: %s", self.orig_license_file) + + copy_file(self.orig_license_file, self.license_file) if get_software_root('Python'): run_cmd("python setup.py install --prefix=%s" % self.installdir) diff --git a/easybuild/easyblocks/generic/__init__.py b/easybuild/easyblocks/generic/__init__.py index e87b3e3c76..62a579f2dd 100644 --- a/easybuild/easyblocks/generic/__init__.py +++ b/easybuild/easyblocks/generic/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/binariestarball.py b/easybuild/easyblocks/generic/binariestarball.py index 259f7d9073..fc74fbe313 100644 --- a/easybuild/easyblocks/generic/binariestarball.py +++ b/easybuild/easyblocks/generic/binariestarball.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/binary.py b/easybuild/easyblocks/generic/binary.py index cfc795acc9..d9ff3a5bb7 100644 --- a/easybuild/easyblocks/generic/binary.py +++ b/easybuild/easyblocks/generic/binary.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -59,6 +59,7 @@ def extra_options(extra_vars=None): extra_vars.update({ 'extract_sources': [False, "Whether or not to extract sources", CUSTOM], 'install_cmd': [None, "Install command to be used.", CUSTOM], + 'install_cmds': [None, "List of install commands to be used.", CUSTOM], # staged installation can help with the hard (potentially faulty) check on available disk space 'staged_install': [False, "Perform staged installation via subdirectory of build directory", CUSTOM], 'prepend_to_path': [PREPEND_TO_PATH_DEFAULT, "Prepend the given directories (relative to install-dir) to " @@ -104,7 +105,9 @@ def build_step(self): def install_step(self): """Copy all files in build directory to the install directory""" install_cmd = self.cfg.get('install_cmd', None) - if install_cmd is None: + install_cmds = self.cfg.get('install_cmds', []) + + if install_cmd is None and install_cmds is None: try: # shutil.copytree doesn't allow the target directory to exist already remove_dir(self.installdir) @@ -112,9 +115,21 @@ def install_step(self): except OSError as err: raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err) else: - cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']]) - self.log.info("Installing %s using command '%s'..." % (self.name, cmd)) - run_cmd(cmd, log_all=True, simple=True) + if install_cmd: + if not install_cmds: + install_cmds = [install_cmd] + install_cmd = None + else: + raise EasyBuildError("Don't use both install_cmds and install_cmd, pick one!") + + if isinstance(install_cmds, (list, tuple)): + for install_cmd in install_cmds: + cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']]) + self.log.info("Running install command for %s: '%s'..." % (self.name, cmd)) + run_cmd(cmd, log_all=True, simple=True) + else: + raise EasyBuildError("Incorrect value type for install_cmds, should be list or tuple: ", + install_cmds) def post_install_step(self): """Copy installation to actual installation directory in case of a staged installation.""" diff --git a/easybuild/easyblocks/generic/buildenv.py b/easybuild/easyblocks/generic/buildenv.py index 8f4200ef98..cb75e3a348 100644 --- a/easybuild/easyblocks/generic/buildenv.py +++ b/easybuild/easyblocks/generic/buildenv.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index a574ba2db4..61cc7125fd 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/cargo.py b/easybuild/easyblocks/generic/cargo.py new file mode 100644 index 0000000000..56cda395fb --- /dev/null +++ b/easybuild/easyblocks/generic/cargo.py @@ -0,0 +1,279 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for installing Cargo packages (Rust lang package system) + +@author: Mikael Oehman (Chalmers University of Technology) +""" + +import os + +import easybuild.tools.environment as env +import easybuild.tools.systemtools as systemtools +from easybuild.tools.build_log import EasyBuildError +from easybuild.framework.easyconfig import CUSTOM +from easybuild.framework.extensioneasyblock import ExtensionEasyBlock +from easybuild.tools.filetools import extract_file, change_dir +from easybuild.tools.run import run_cmd +from easybuild.tools.config import build_option +from easybuild.tools.filetools import write_file, compute_checksum +from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC + +CRATESIO_SOURCE = "https://crates.io/api/v1/crates" + + +class Cargo(ExtensionEasyBlock): + """Support for installing Cargo packages (Rust)""" + + @staticmethod + def extra_options(extra_vars=None): + """Define extra easyconfig parameters specific to Cargo""" + extra_vars = ExtensionEasyBlock.extra_options(extra_vars) + extra_vars.update({ + 'enable_tests': [True, "Enable building of tests", CUSTOM], + 'offline': [True, "Build offline", CUSTOM], + 'lto': [None, "Override default LTO flag ('fat', 'thin', 'off')", CUSTOM], + 'crates': [[], "List of (crate, version, [repo, rev]) tuples to use", CUSTOM], + }) + + return extra_vars + + def rustc_optarch(self): + """Determines what architecture to target. + Translates GENERIC optarch, and respects rustc specific optarch. + General optarchs are ignored as there is no direct translation. + """ + if systemtools.X86_64 == systemtools.get_cpu_architecture(): + generic = '-C target-cpu=x86-64' + else: + generic = '-C target-cpu=generic' + + optimal = '-C target-cpu=native' + + optarch = build_option('optarch') + if optarch: + if type(optarch) == dict: + if 'rustc' in optarch: + rust_optarch = optarch['rustc'] + if rust_optarch == OPTARCH_GENERIC: + return generic + else: + return '-' + rust_optarch + self.log.info("no rustc information in the optarch dict, so using %s" % optimal) + else: + if optarch == OPTARCH_GENERIC: + return generic + else: + self.log.warning("optarch is ignored as there is no translation for rustc, so using %s" % optimal) + return optimal + + def __init__(self, *args, **kwargs): + """Constructor for Cargo easyblock.""" + super(Cargo, self).__init__(*args, **kwargs) + self.cargo_home = os.path.join(self.builddir, '.cargo') + env.setvar('CARGO_HOME', self.cargo_home) + env.setvar('RUSTC', 'rustc') + env.setvar('RUSTDOC', 'rustdoc') + env.setvar('RUSTFMT', 'rustfmt') + env.setvar('RUSTFLAGS', self.rustc_optarch()) + env.setvar('RUST_LOG', 'DEBUG') + env.setvar('RUST_BACKTRACE', '1') + + # Populate sources from "crates" list of tuples (only once) + if self.cfg['crates']: + # copy list of crates, so we can wipe 'crates' easyconfig parameter, + # to avoid that creates are processed into 'sources' easyconfig parameter again + # when easyblock is initialized again using same parsed easyconfig + # (for example when check_sha256_checksums function is called, like in easyconfigs test suite) + self.crates = self.cfg['crates'][:] + sources = [] + for crate_info in self.cfg['crates']: + if len(crate_info) == 2: + crate, version = crate_info + sources.append({ + 'download_filename': crate + '/' + version + '/download', + 'filename': crate + '-' + version + '.tar.gz', + 'source_urls': [CRATESIO_SOURCE], + 'alt_location': 'crates.io', + }) + else: + crate, version, repo, rev = crate_info + url, repo_name_git = repo.rsplit('/', maxsplit=1) + sources.append({ + 'git_config': {'url': url, 'repo_name': repo_name_git[:-4], 'commit': rev}, + 'filename': crate + '-' + version + '.tar.gz', + 'source_urls': [CRATESIO_SOURCE], + }) + + self.cfg.update('sources', sources) + + # set 'crates' easyconfig parameter to empty list to prevent re-processing into sources + self.cfg['crates'] = [] + + def extract_step(self): + """ + Unpack the source files and populate them with required .cargo-checksum.json if offline + """ + if self.cfg['offline']: + self.log.info("Setting vendored crates dir") + # Replace crates-io with vendored sources using build dir wide toml file in CARGO_HOME + # because the rust source subdirectories might differ with python packages + config_toml = os.path.join(self.cargo_home, 'config.toml') + write_file(config_toml, '[source.vendored-sources]\ndirectory = "%s"\n\n' % self.builddir, append=True) + write_file(config_toml, '[source.crates-io]\nreplace-with = "vendored-sources"\n\n', append=True) + + # also vendor sources from other git sources (could be many crates for one git source) + git_sources = set() + for crate_info in self.crates: + if len(crate_info) == 4: + _, _, repo, rev = crate_info + git_sources.add((repo, rev)) + for repo, rev in git_sources: + write_file(config_toml, '[source."%s"]\ngit = "%s"\nrev = "%s"\n' + 'replace-with = "vendored-sources"\n\n' % (repo, repo, rev), append=True) + + # Use environment variable since it would also be passed along to builds triggered via python packages + env.setvar('CARGO_NET_OFFLINE', 'true') + + # More work is needed here for git sources to work, especially those repos with multiple packages. + for src in self.src: + existing_dirs = set(os.listdir(self.builddir)) + self.log.info("Unpacking source %s" % src['name']) + srcdir = extract_file(src['path'], self.builddir, cmd=src['cmd'], + extra_options=self.cfg['unpack_options'], change_into_dir=False) + change_dir(srcdir) + if srcdir: + self.src[self.src.index(src)]['finalpath'] = srcdir + else: + raise EasyBuildError("Unpacking source %s failed", src['name']) + + # Create checksum file for all sources required by vendored crates.io sources + new_dirs = set(os.listdir(self.builddir)) - existing_dirs + if self.cfg['offline'] and len(new_dirs) == 1: + cratedir = new_dirs.pop() + self.log.info('creating .cargo-checksums.json file for : %s', cratedir) + chksum = compute_checksum(src['path'], checksum_type='sha256') + chkfile = os.path.join(self.builddir, cratedir, '.cargo-checksum.json') + write_file(chkfile, '{"files":{},"package":"%s"}' % chksum) + + def configure_step(self): + """Empty configuration step.""" + pass + + @property + def profile(self): + return 'debug' if self.toolchain.options.get('debug', None) else 'release' + + def build_step(self): + """Build with cargo""" + parallel = '' + if self.cfg['parallel']: + parallel = "-j %s" % self.cfg['parallel'] + + tests = '' + if self.cfg['enable_tests']: + tests = "--tests" + + lto = '' + if self.cfg['lto'] is not None: + lto = '--config profile.%s.lto=\\"%s\\"' % (self.profile, self.cfg['lto']) + + run_cmd('rustc --print cfg', log_all=True, simple=True) # for tracking in log file + cmd = ' '.join([ + self.cfg['prebuildopts'], + 'cargo build', + '--profile=' + self.profile, + lto, + tests, + parallel, + self.cfg['buildopts'], + ]) + run_cmd(cmd, log_all=True, simple=True) + + def test_step(self): + """Test with cargo""" + if self.cfg['enable_tests']: + cmd = ' '.join([ + self.cfg['pretestopts'], + 'cargo test', + '--profile=' + self.profile, + self.cfg['testopts'], + ]) + run_cmd(cmd, log_all=True, simple=True) + + def install_step(self): + """Install with cargo""" + cmd = ' '.join([ + self.cfg['preinstallopts'], + 'cargo install', + '--profile=' + self.profile, + '--root=' + self.installdir, + '--path=.', + self.cfg['installopts'], + ]) + run_cmd(cmd, log_all=True, simple=True) + + +def generate_crate_list(sourcedir): + """Helper for generating crate list""" + import toml + + cargo_toml = toml.load(os.path.join(sourcedir, 'Cargo.toml')) + cargo_lock = toml.load(os.path.join(sourcedir, 'Cargo.lock')) + + app_name = cargo_toml['package']['name'] + deps = cargo_lock['package'] + + app_in_cratesio = False + crates = [] + other_crates = [] + for dep in deps: + name = dep['name'] + version = dep['version'] + if 'source' in dep: + if name == app_name: + app_in_cratesio = True # exclude app itself, needs to be first in crates list or taken from pypi + else: + if dep['source'] == 'registry+https://github.com/rust-lang/crates.io-index': + crates.append((name, version)) + else: + # Lock file has revision#revision in the url for some reason. + crates.append((name, version, dep['source'].rsplit('#', maxsplit=1)[0])) + else: + other_crates.append((name, version)) + return app_in_cratesio, crates, other_crates + + +if __name__ == '__main__': + import sys + app_in_cratesio, crates, other = generate_crate_list(sys.argv[1]) + print(other) + if app_in_cratesio or crates: + print('crates = [') + if app_in_cratesio: + print(' (name, version),') + for crate_info in crates: + print(" ('" + "', '".join(crate_info) + "'),") + print(']') diff --git a/easybuild/easyblocks/generic/cargopythonpackage.py b/easybuild/easyblocks/generic/cargopythonpackage.py new file mode 100644 index 0000000000..a935e190a0 --- /dev/null +++ b/easybuild/easyblocks/generic/cargopythonpackage.py @@ -0,0 +1,57 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for installing Cargo packages (Rust lang package system) + +@author: Mikael Oehman (Chalmers University of Technology) +""" + +from easybuild.easyblocks.generic.cargo import Cargo +from easybuild.easyblocks.generic.pythonpackage import PythonPackage + + +class CargoPythonPackage(PythonPackage, Cargo): # PythonPackage must come first to take precedence + """Build a Python package with setup from Cargo but build/install step from PythonPackage + + The cargo init step will set up the environment variables for rustc and vendor sources + but all the build steps are triggered via normal PythonPackage steps like normal. + """ + + @staticmethod + def extra_options(extra_vars=None): + """Define extra easyconfig parameters specific to Cargo""" + extra_vars = PythonPackage.extra_options(extra_vars) + extra_vars = Cargo.extra_options(extra_vars) # not all extra options here will used here + + return extra_vars + + def __init__(self, *args, **kwargs): + """Constructor for CargoPythonPackage easyblock.""" + Cargo.__init__(self, *args, **kwargs) + PythonPackage.__init__(self, *args, **kwargs) + + def extract_step(self): + """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums.""" + return Cargo.extract_step(self) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 0f1a7610cb..48cdae593a 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -141,7 +141,10 @@ def build_type(self): def prepend_config_opts(self, config_opts): """Prepends configure options (-Dkey=value) to configopts ignoring those already set""" - cfg_configopts = self.cfg['configopts'] + # need to disable template resolution or it will remain the same for all runs + with self.cfg.disable_templating(): + cfg_configopts = self.cfg['configopts'] + # All options are of the form '-D=' new_opts = ' '.join('-D%s=%s' % (key, value) for key, value in config_opts.items() if '-D%s=' % key not in cfg_configopts) diff --git a/easybuild/easyblocks/generic/cmakemakecp.py b/easybuild/easyblocks/generic/cmakemakecp.py index bfa4229fbe..866ffe9e86 100644 --- a/easybuild/easyblocks/generic/cmakemakecp.py +++ b/easybuild/easyblocks/generic/cmakemakecp.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/cmakeninja.py b/easybuild/easyblocks/generic/cmakeninja.py index 1b3d554ff7..db344470f1 100644 --- a/easybuild/easyblocks/generic/cmakeninja.py +++ b/easybuild/easyblocks/generic/cmakeninja.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2022 Ghent University +# Copyright 2019-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/cmakepythonpackage.py b/easybuild/easyblocks/generic/cmakepythonpackage.py index 1f76be306c..fac9aeced4 100644 --- a/easybuild/easyblocks/generic/cmakepythonpackage.py +++ b/easybuild/easyblocks/generic/cmakepythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -53,11 +53,6 @@ def extra_options(extra_vars=None): extra_vars['runtest'][0] = None return extra_vars - def __init__(self, *args, **kwargs): - """Initialize with PythonPackage.""" - PythonPackage.__init__(self, *args, **kwargs) - self._lib_ext = None # From CMakeMake.__init__ - def configure_step(self, *args, **kwargs): """Main configuration using cmake""" diff --git a/easybuild/easyblocks/generic/cmdcp.py b/easybuild/easyblocks/generic/cmdcp.py index be15601c39..5a91112a4e 100644 --- a/easybuild/easyblocks/generic/cmdcp.py +++ b/easybuild/easyblocks/generic/cmdcp.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2022 Ghent University +# Copyright 2014-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/conda.py b/easybuild/easyblocks/generic/conda.py index f40cc2d398..b3e0fed5b0 100644 --- a/easybuild/easyblocks/generic/conda.py +++ b/easybuild/easyblocks/generic/conda.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index 3771f61b64..a8527e1088 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -184,6 +184,8 @@ def extra_options(extra_vars=None): " False implies to leave it up to the configure script)", CUSTOM], 'configure_cmd': [DEFAULT_CONFIGURE_CMD, "Configure command to use", CUSTOM], 'configure_cmd_prefix': ['', "Prefix to be glued before ./configure", CUSTOM], + 'configure_without_installdir': [False, "Avoid passing an install directory to the configure command " + "(such as via --prefix)", CUSTOM], 'host_type': [None, "Value to provide to --host option of configure script, e.g., x86_64-pc-linux-gnu " "(determined by config.guess shipped with EasyBuild if None," " False implies to leave it up to the configure script)", CUSTOM], @@ -303,11 +305,19 @@ def configure_step(self, cmd_prefix=''): if host_type: build_and_host_options.append(' --host=' + host_type) + if self.cfg.get('configure_without_installdir'): + configure_prefix = '' + if self.cfg.get('prefix_opt'): + print_warning("Specified prefix_opt '%s' is ignored due to use of configure_without_installdir", + prefix_opt) + else: + configure_prefix = prefix_opt + self.installdir + cmd = ' '.join( [ self.cfg['preconfigopts'], configure_command, - prefix_opt + self.installdir, + configure_prefix, ] + build_and_host_options + [self.cfg['configopts']] ) diff --git a/easybuild/easyblocks/generic/configuremakepythonpackage.py b/easybuild/easyblocks/generic/configuremakepythonpackage.py index 8c5dc9f983..8da38021a2 100644 --- a/easybuild/easyblocks/generic/configuremakepythonpackage.py +++ b/easybuild/easyblocks/generic/configuremakepythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,9 +36,10 @@ class ConfigureMakePythonPackage(ConfigureMake, PythonPackage): """ - Build a Python package and module with 'python configure/make/make install'. + Build a Python package and module with ``python configure``/``make``/``make install``. Implemented by using: + - a custom implementation of configure_step - using the build_step and install_step from ConfigureMake - using the sanity_check_step and make_module_extra from PythonPackage @@ -54,13 +55,13 @@ def __init__(self, *args, **kwargs): PythonPackage.__init__(self, *args, **kwargs) def configure_step(self, *args, **kwargs): - """Configure build using 'python configure'.""" + """Configure build using ``python configure``.""" PythonPackage.configure_step(self, *args, **kwargs) cmd = ' '.join([self.cfg['preconfigopts'], self.python_cmd, self.cfg['configopts']]) run_cmd(cmd, log_all=True) def build_step(self, *args, **kwargs): - """Build Python package with 'make'.""" + """Build Python package with ``make``.""" return ConfigureMake.build_step(self, *args, **kwargs) def test_step(self, *args, **kwargs): @@ -68,7 +69,7 @@ def test_step(self, *args, **kwargs): PythonPackage.test_step(self, *args, **kwargs) def install_step(self, *args, **kargs): - """Install with 'make install'.""" + """Install with ``make install``.""" return ConfigureMake.install_step(self, *args, **kargs) def sanity_check_step(self, *args, **kwargs): diff --git a/easybuild/easyblocks/generic/craytoolchain.py b/easybuild/easyblocks/generic/craytoolchain.py index 41ae34643d..e324b72e7f 100644 --- a/easybuild/easyblocks/generic/craytoolchain.py +++ b/easybuild/easyblocks/generic/craytoolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/fortranpythonpackage.py b/easybuild/easyblocks/generic/fortranpythonpackage.py index 01270f0028..849dacf54b 100644 --- a/easybuild/easyblocks/generic/fortranpythonpackage.py +++ b/easybuild/easyblocks/generic/fortranpythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index cfde24848c..5c2d96bdc9 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -131,6 +131,9 @@ def install_step(self): self.cfg['preinstallopts'], 'go', 'install', + # print commands as they are executed, + # including downloading and installing of package deps as listed in the go.mod file + '-x', self.cfg['installopts'], ]) run_cmd(cmd, log_all=True, log_ok=True, simple=True) @@ -146,3 +149,6 @@ def sanity_check_step(self): custom_commands = ['%s --help' % self.name.lower()] super(GoPackage, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + + def sanity_check_rpath(self, rpath_dirs=None): + super(GoPackage, self).sanity_check_rpath(rpath_dirs=rpath_dirs, check_readelf_rpath=False) diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index 2913525749..cab94b770c 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -91,6 +91,7 @@ class IntelBase(EasyBlock): """ Base class for Intel software + - no configure/make : binary release - add license_file variable """ diff --git a/easybuild/easyblocks/generic/jar.py b/easybuild/easyblocks/generic/jar.py index dcfc772e31..55765f11ab 100644 --- a/easybuild/easyblocks/generic/jar.py +++ b/easybuild/easyblocks/generic/jar.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py new file mode 100644 index 0000000000..28c8035068 --- /dev/null +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -0,0 +1,100 @@ +## +# Copyright 2022-2023 Vrije Universiteit Brussel +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for bundles of Julia packages, implemented as an easyblock + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" +import os + +from easybuild.easyblocks.generic.bundle import Bundle +from easybuild.easyblocks.generic.juliapackage import EXTS_FILTER_JULIA_PACKAGES, JuliaPackage +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.modules import get_software_root + + +class JuliaBundle(Bundle): + """ + Bundle of JuliaPackages: install Julia packages as extensions in a bundle + Defines custom sanity checks and module environment + """ + + @staticmethod + def extra_options(extra_vars=None): + """Easyconfig parameters specific to bundles of Julia packages.""" + if extra_vars is None: + extra_vars = {} + # combine custom easyconfig parameters of Bundle & JuliaPackage + extra_vars = Bundle.extra_options(extra_vars) + return JuliaPackage.extra_options(extra_vars) + + def __init__(self, *args, **kwargs): + """Initialize JuliaBundle easyblock.""" + super(JuliaBundle, self).__init__(*args, **kwargs) + + self.cfg['exts_defaultclass'] = 'JuliaPackage' + self.cfg['exts_filter'] = EXTS_FILTER_JULIA_PACKAGES + + # need to disable templating to ensure that actual value for exts_default_options is updated... + with self.cfg.disable_templating(): + # set default options for extensions according to relevant top-level easyconfig parameters + jlpkg_keys = JuliaPackage.extra_options().keys() + for key in jlpkg_keys: + if key not in self.cfg['exts_default_options']: + self.cfg['exts_default_options'][key] = self.cfg[key] + + # Sources of Julia packages are commonly distributed from GitHub repos. + # By default, rename downloaded tarballs to avoid name collisions on + # packages sharing the same version string + if 'sources' not in self.cfg['exts_default_options']: + self.cfg['exts_default_options']['sources'] = [ + { + 'download_filename': 'v%(version)s.tar.gz', + 'filename': '%(name)s-%(version)s.tar.gz', + } + ] + + self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) + + def prepare_step(self, *args, **kwargs): + """Prepare for installing bundle of Julia packages.""" + super(JuliaBundle, self).prepare_step(*args, **kwargs) + + if get_software_root('Julia') is None: + raise EasyBuildError("Julia not included as dependency!") + + def make_module_extra(self, *args, **kwargs): + """Prepend installation directory to JULIA_DEPOT_PATH in module file.""" + txt = super(JuliaBundle, self).make_module_extra() + txt += self.module_generator.prepend_paths('JULIA_DEPOT_PATH', ['']) + return txt + + def sanity_check_step(self, *args, **kwargs): + """Custom sanity check for bundle of Julia packages""" + custom_paths = { + 'files': [], + 'dirs': [os.path.join('packages', self.name)], + } + super(JuliaBundle, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py new file mode 100644 index 0000000000..2e32047665 --- /dev/null +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -0,0 +1,169 @@ +## +# Copyright 2022-2023 Vrije Universiteit Brussel +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for Julia Packages, implemented as an easyblock + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" +import os + +from distutils.version import LooseVersion + +import easybuild.tools.environment as env +from easybuild.framework.easyconfig import CUSTOM +from easybuild.framework.extensioneasyblock import ExtensionEasyBlock +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.modules import get_software_root, get_software_version +from easybuild.tools.filetools import copy_dir +from easybuild.tools.run import run_cmd + +EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "") + + +class JuliaPackage(ExtensionEasyBlock): + """Builds and installs Julia Packages.""" + + @staticmethod + def extra_options(extra_vars=None): + """Extra easyconfig parameters specific to JuliaPackage.""" + extra_vars = ExtensionEasyBlock.extra_options(extra_vars=extra_vars) + extra_vars.update({ + 'download_pkg_deps': [ + False, "Let Julia download and bundle all needed dependencies for this installation", CUSTOM + ], + }) + return extra_vars + + def set_pkg_offline(self): + """Enable offline mode of Julia Pkg""" + if get_software_root('Julia') is None: + raise EasyBuildError("Julia not included as dependency!") + + if not self.cfg['download_pkg_deps']: + julia_version = get_software_version('Julia') + if LooseVersion(julia_version) >= LooseVersion('1.5'): + # Enable offline mode of Julia Pkg + # https://pkgdocs.julialang.org/v1/api/#Pkg.offline + env.setvar('JULIA_PKG_OFFLINE', 'true') + else: + errmsg = ( + "Cannot set offline mode in Julia v%s (needs Julia >= 1.5). " + "Enable easyconfig option 'download_pkg_deps' to allow installation " + "with any extra downloaded dependencies." + ) + raise EasyBuildError(errmsg, julia_version) + + def prepare_step(self, *args, **kwargs): + """Prepare for installing Julia package.""" + super(JuliaPackage, self).prepare_step(*args, **kwargs) + self.set_pkg_offline() + + def configure_step(self): + """No separate configuration for JuliaPackage.""" + pass + + def build_step(self): + """No separate build procedure for JuliaPackage.""" + pass + + def test_step(self): + """No separate (standard) test procedure for JuliaPackage.""" + pass + + def install_step(self): + """Install Julia package with Pkg""" + + # prepend installation directory to Julia DEPOT_PATH + # extensions in a bundle can share their DEPOT_PATH + # see https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH + depot_path = os.getenv('JULIA_DEPOT_PATH', []) + if depot_path: + depot_path = depot_path.split(os.pathsep) + if self.installdir not in depot_path: + depot_path = os.pathsep.join([depot for depot in [self.installdir] + depot_path if depot]) + env.setvar('JULIA_DEPOT_PATH', depot_path) + + # command sequence for Julia.Pkg + julia_pkg_cmd = ['using Pkg'] + if os.path.isdir(os.path.join(self.start_dir, '.git')): + # sources from git repos can be installed as any remote package + self.log.debug('Installing Julia package in normal mode (Pkg.add)') + + julia_pkg_cmd.extend([ + # install package from local path preserving existing dependencies + 'Pkg.add(url="%s"; preserve=Pkg.PRESERVE_ALL)' % self.start_dir, + ]) + else: + # plain sources have to be installed in develop mode + # copy sources to install directory and install + self.log.debug('Installing Julia package in develop mode (Pkg.develop)') + + install_pkg_path = os.path.join(self.installdir, 'packages', self.name) + copy_dir(self.start_dir, install_pkg_path) + + julia_pkg_cmd.extend([ + 'Pkg.develop(PackageSpec(path="%s"))' % install_pkg_path, + 'Pkg.build("%s")' % self.name, + ]) + + julia_pkg_cmd = ';'.join(julia_pkg_cmd) + cmd = ' '.join([ + self.cfg['preinstallopts'], + "julia -e '%s'" % julia_pkg_cmd, + self.cfg['installopts'], + ]) + (out, _) = run_cmd(cmd, log_all=True, simple=False) + + return out + + def run(self): + """Install Julia package as an extension.""" + + if not self.src: + errmsg = "No source found for Julia package %s, required for installation. (src: %s)" + raise EasyBuildError(errmsg, self.name, self.src) + ExtensionEasyBlock.run(self, unpack_src=True) + + self.set_pkg_offline() + self.install_step() + + def sanity_check_step(self, *args, **kwargs): + """Custom sanity check for JuliaPackage""" + + pkg_dir = os.path.join('packages', self.name) + + custom_paths = { + 'files': [], + 'dirs': [pkg_dir], + } + kwargs.update({'custom_paths': custom_paths}) + + return ExtensionEasyBlock.sanity_check_step(self, EXTS_FILTER_JULIA_PACKAGES, *args, **kwargs) + + def make_module_extra(self): + """Prepend installation directory to JULIA_DEPOT_PATH in module file.""" + txt = super(JuliaPackage, self).make_module_extra() + txt += self.module_generator.prepend_paths('JULIA_DEPOT_PATH', ['']) + return txt diff --git a/easybuild/easyblocks/generic/makecp.py b/easybuild/easyblocks/generic/makecp.py index f8603445b7..f02937570b 100644 --- a/easybuild/easyblocks/generic/makecp.py +++ b/easybuild/easyblocks/generic/makecp.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 the Cyprus Institute +# Copyright 2013-2023 the Cyprus Institute # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -26,6 +26,7 @@ @author: George Tsouloupas (The Cyprus Institute) @author: Fotis Georgatos (Uni.Lu, NTUA) @author: Kenneth Hoste (Ghent University) +@author: Maxime Boissonneault (Digital Research Alliance of Canada, Universite Laval) """ import os import glob @@ -70,6 +71,10 @@ def install_step(self): files_to_copy = self.cfg.get('files_to_copy') or [] self.log.debug("Starting install_step with files_to_copy: %s", files_to_copy) + + # if this is an iterative build directories will be copied multiple times + dirs_exist_ok = True if self.iter_opts else False + for fil in files_to_copy: if isinstance(fil, tuple): # ([src1, src2], targetdir) @@ -128,6 +133,6 @@ def install_step(self): elif os.path.isdir(filepath): self.log.debug("Copying directory %s to %s", filepath, target) fulltarget = os.path.join(target, os.path.basename(filepath)) - copy_dir(filepath, fulltarget, symlinks=self.cfg['keepsymlinks']) + copy_dir(filepath, fulltarget, symlinks=self.cfg['keepsymlinks'], dirs_exist_ok=dirs_exist_ok) else: raise EasyBuildError("Can't copy non-existing path %s to %s", filepath, target) diff --git a/easybuild/easyblocks/generic/mesonninja.py b/easybuild/easyblocks/generic/mesonninja.py index 2c097cbb67..c8ad0af51b 100644 --- a/easybuild/easyblocks/generic/mesonninja.py +++ b/easybuild/easyblocks/generic/mesonninja.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2022 Ghent University +# Copyright 2018-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,12 +28,16 @@ @author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, create_unused_dir, which +from easybuild.tools.modules import get_software_version from easybuild.tools.run import run_cmd +DEFAULT_CONFIGURE_CMD = 'meson' + class MesonNinja(EasyBlock): """ @@ -45,6 +49,7 @@ def extra_options(extra_vars=None): """Define extra easyconfig parameters specific to MesonNinja.""" extra_vars = EasyBlock.extra_options(extra_vars) extra_vars.update({ + 'configure_cmd': [DEFAULT_CONFIGURE_CMD, "Configure command to use", CUSTOM], 'separate_build_dir': [True, "Perform build in a separate directory", CUSTOM], }) return extra_vars @@ -73,8 +78,16 @@ def configure_step(self, cmd_prefix=''): if no_Dlibdir and no_libdir: self.cfg.update('configopts', '-Dlibdir=lib') - cmd = "%(preconfigopts)s meson --prefix %(installdir)s %(configopts)s %(sourcedir)s" % { + configure_cmd = self.cfg.get('configure_cmd') or DEFAULT_CONFIGURE_CMD + # Meson >= 0.64.0 has a deprecatation warning for running `meson [options]` + # instead of `meson setup [options]` + if (LooseVersion(get_software_version('Meson')) >= LooseVersion('0.64.0') and + configure_cmd == DEFAULT_CONFIGURE_CMD): + configure_cmd += ' setup' + + cmd = "%(preconfigopts)s %(configure_cmd)s --prefix %(installdir)s %(configopts)s %(sourcedir)s" % { 'configopts': self.cfg['configopts'], + 'configure_cmd': configure_cmd, 'installdir': self.installdir, 'preconfigopts': self.cfg['preconfigopts'], 'sourcedir': self.start_dir, diff --git a/easybuild/easyblocks/generic/modulerc.py b/easybuild/easyblocks/generic/modulerc.py index 8c2d1736bb..aa67e90fe3 100644 --- a/easybuild/easyblocks/generic/modulerc.py +++ b/easybuild/easyblocks/generic/modulerc.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2022 Ghent University +# Copyright 2018-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/ocamlpackage.py b/easybuild/easyblocks/generic/ocamlpackage.py index 6dbc168718..d0b460d2ca 100644 --- a/easybuild/easyblocks/generic/ocamlpackage.py +++ b/easybuild/easyblocks/generic/ocamlpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/octavepackage.py b/easybuild/easyblocks/generic/octavepackage.py index 11b1024abd..cff63cc9c6 100644 --- a/easybuild/easyblocks/generic/octavepackage.py +++ b/easybuild/easyblocks/generic/octavepackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/packedbinary.py b/easybuild/easyblocks/generic/packedbinary.py index fe6ac9c5ee..2520397b3e 100644 --- a/easybuild/easyblocks/generic/packedbinary.py +++ b/easybuild/easyblocks/generic/packedbinary.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/perlbundle.py b/easybuild/easyblocks/generic/perlbundle.py new file mode 100644 index 0000000000..fcb5ce48f8 --- /dev/null +++ b/easybuild/easyblocks/generic/perlbundle.py @@ -0,0 +1,94 @@ +## +# Copyright 2018-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for installing a bundle of Perl modules, implemented as a generic easyblock + +@author: Mikael Öhman (Chalmers University of Technology) +""" +import os + +from easybuild.easyblocks.generic.bundle import Bundle +from easybuild.easyblocks.generic.perlmodule import PerlModule +from easybuild.easyblocks.perl import get_major_perl_version, get_site_suffix +from easybuild.tools.config import build_option +from easybuild.tools.environment import setvar + + +class PerlBundle(Bundle): + """ + Bundle of perl modules + """ + + @staticmethod + def extra_options(): + """Easyconfig parameters specific to bundles of Perl modules.""" + # combine custom easyconfig parameters of Bundle & PerlModule + extra_vars = PerlModule.extra_options() + return Bundle.extra_options(extra_vars) + + def __init__(self, *args, **kwargs): + """Initialize PerlBundle easyblock.""" + super(PerlBundle, self).__init__(*args, **kwargs) + + self.cfg['exts_defaultclass'] = 'PerlModule' + self.cfg['exts_filter'] = ("perl -e 'require %(ext_name)s'", '') + + def extensions_step(self, *args, **kwargs): + """Install extensions""" + + setvar('INSTALLDIRS', 'site') + # define $OPENSSL_PREFIX to ensure that e.g. Net-SSLeay extension picks up OpenSSL + # from specified sysroot rather than from host OS + sysroot = build_option('sysroot') + if sysroot: + setvar('OPENSSL_PREFIX', sysroot) + + super(PerlBundle, self).extensions_step(*args, **kwargs) + + def test_step(self): + """No global test step for bundle of Perl modules.""" + # required since runtest is set to True by default + pass + + def sanity_check_step(self, *args, **kwargs): + """Custom sanity check for bundle of Perl modules.""" + + if not self.cfg['sanity_check_paths']: + majver = get_major_perl_version() + self.cfg['sanity_check_paths'] = { + 'files': [], + 'dirs': [os.path.join('lib', 'perl%s' % majver)], + } + + super(Bundle, self).sanity_check_step(*args, **kwargs) + + def make_module_extra(self): + """Extra module entries for Perl bundles.""" + majver = get_major_perl_version() + sitelibsuffix = get_site_suffix('sitelib') + + txt = super(Bundle, self).make_module_extra() + txt += self.module_generator.prepend_paths("PERL%sLIB" % majver, [sitelibsuffix]) + return txt diff --git a/easybuild/easyblocks/generic/perlmodule.py b/easybuild/easyblocks/generic/perlmodule.py index 28d8137475..f58163d6e8 100644 --- a/easybuild/easyblocks/generic/perlmodule.py +++ b/easybuild/easyblocks/generic/perlmodule.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index ba5a317473..292cc275aa 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2022 Ghent University +# Copyright 2018-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -41,7 +41,8 @@ class PythonBundle(Bundle): """ - Bundle of modules: only generate module files, nothing to build/install + Bundle of PythonPackages: install Python packages as extensions in a bundle + Defines custom sanity checks and module environment """ @staticmethod @@ -61,21 +62,17 @@ def __init__(self, *args, **kwargs): self.cfg['exts_filter'] = EXTS_FILTER_PYTHON_PACKAGES # need to disable templating to ensure that actual value for exts_default_options is updated... - prev_enable_templating = self.cfg.enable_templating - self.cfg.enable_templating = False + with self.cfg.disable_templating(): + # set default options for extensions according to relevant top-level easyconfig parameters + pypkg_keys = PythonPackage.extra_options().keys() + for key in pypkg_keys: + if key not in self.cfg['exts_default_options']: + self.cfg['exts_default_options'][key] = self.cfg[key] - # set default options for extensions according to relevant top-level easyconfig parameters - pypkg_keys = PythonPackage.extra_options().keys() - for key in pypkg_keys: - if key not in self.cfg['exts_default_options']: - self.cfg['exts_default_options'][key] = self.cfg[key] + self.cfg['exts_default_options']['download_dep_fail'] = True + self.log.info("Detection of downloaded extension dependencies is enabled") - self.cfg['exts_default_options']['download_dep_fail'] = True - self.log.info("Detection of downloaded extension dependencies is enabled") - - self.cfg.enable_templating = prev_enable_templating - - self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) + self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) self.pylibdir = None self.all_pylibdirs = [] diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index f84b601ef0..412d005270 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -60,7 +60,7 @@ # not 'easy_install' deliberately, to avoid that pkg installations listed in easy-install.pth get preference # '.' is required at the end when using easy_install/pip in unpacked source dir EASY_INSTALL_TARGET = "easy_install" -PIP_INSTALL_CMD = "pip install --prefix=%(prefix)s %(installopts)s %(loc)s" +PIP_INSTALL_CMD = "%(python)s -m pip install --prefix=%(prefix)s %(installopts)s %(loc)s" SETUP_PY_INSTALL_CMD = "%(python)s setup.py %(install_target)s --prefix=%(prefix)s %(installopts)s" UNKNOWN = 'UNKNOWN' @@ -204,14 +204,14 @@ def get_pylibdirs(python_cmd): return all_pylibdirs -def det_pip_version(): - """Determine version of currently active 'pip' command.""" +def det_pip_version(python_cmd='python'): + """Determine version of currently active 'pip' module.""" pip_version = None log = fancylogger.getLogger('det_pip_version', fname=False) log.info("Determining pip version...") - out, _ = run_cmd("pip --version", verbose=False, simple=False, trace=False) + out, _ = run_cmd("%s -m pip --version" % python_cmd, verbose=False, simple=False, trace=False) pip_version_regex = re.compile('^pip ([0-9.]+)') res = pip_version_regex.search(out) @@ -247,9 +247,10 @@ def extra_options(extra_vars=None): "the pip version check. Enabled by default when pip_ignore_installed=True", CUSTOM], 'req_py_majver': [None, "Required major Python version (only relevant when using system Python)", CUSTOM], 'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM], - 'sanity_pip_check': [False, "Run 'pip check' to ensure all required Python packages are installed " - "and check for any package with an invalid (0.0.0) version.", CUSTOM], + 'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are " + "installed and check for any package with an invalid (0.0.0) version.", CUSTOM], 'runtest': [True, "Run unit tests.", CUSTOM], # overrides default + 'testinstall': [False, "Install into temporary directory prior to running the tests.", CUSTOM], 'unpack_sources': [None, "Unpack sources prior to build/install. Defaults to 'True' except for whl files", CUSTOM], # A version of 0.0.0 is usually an error on installation unless the package does really not provide a @@ -261,8 +262,8 @@ def extra_options(extra_vars=None): # see https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras 'use_pip_extras': [None, "String with comma-separated list of 'extras' to install via pip", CUSTOM], 'use_pip_for_deps': [False, "Install dependencies using '%s'" % PIP_INSTALL_CMD, CUSTOM], - 'use_pip_requirement': [False, "Install using 'pip install --requirement'. The sources is expected " + - "to be the requirements file.", CUSTOM], + 'use_pip_requirement': [False, "Install using 'python -m pip install --requirement'. The sources is " + + "expected to be the requirements file.", CUSTOM], 'zipped_egg': [False, "Install as a zipped eggs", CUSTOM], }) # Use PYPI_SOURCE as the default for source_urls. @@ -283,7 +284,7 @@ def __init__(self, *args, **kwargs): self.sitecfgfn = 'site.cfg' self.sitecfglibdir = None self.sitecfgincdir = None - self.testinstall = False + self.testinstall = self.cfg['testinstall'] self.testcmd = None self.unpack_options = self.cfg['unpack_options'] @@ -318,6 +319,12 @@ def __init__(self, *args, **kwargs): # determine install command self.use_setup_py = False + self.determine_install_command() + + def determine_install_command(self): + """ + Determine install command to use. + """ if self.cfg.get('use_pip', False) or self.cfg.get('use_pip_editable', False): self.install_cmd = PIP_INSTALL_CMD @@ -361,7 +368,7 @@ def __init__(self, *args, **kwargs): else: raise EasyBuildError("Installing zipped eggs requires using easy_install or pip") - self.log.debug("Using '%s' as install command", self.install_cmd) + self.log.info("Using '%s' as install command", self.install_cmd) def set_pylibdirs(self): """Set Python lib directory-related class variables.""" @@ -468,7 +475,7 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None): using_pip = self.install_cmd.startswith(PIP_INSTALL_CMD) if using_pip: - pip_version = det_pip_version() + pip_version = det_pip_version(python_cmd=self.python_cmd) if pip_version: # pip 8.x or newer required, because of --prefix option being used if LooseVersion(pip_version) >= LooseVersion('8.0'): @@ -684,8 +691,8 @@ def test_step(self, return_output_ec=False): ]) if return_output_ec: - (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False) - # need to log seperately, since log_all and log_ok need to be false to retreive out and ec + (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False) + # need to log seperately, since log_all and log_ok need to be false to retrieve out and ec self.log.info("cmd '%s' exited with exit code %s and output:\n%s", cmd, ec, out) else: run_cmd(cmd, log_all=True, simple=True) @@ -724,6 +731,9 @@ def install_step(self): # (for iterated installations over multiply Python versions) self.install_cmd_output += out + # fix shebangs if specified + self.fix_shebang() + # restore env vars if it they were set for name in ('PYTHONPATH', 'PATH'): value = old_values[name] @@ -781,6 +791,14 @@ def sanity_check_step(self, *args, **kwargs): success, fail_msg = True, '' + # load module early ourselves rather than letting parent sanity_check_step method do so, + # since custom actions taken below require that environment is set up properly already + # (especially when using --sanity-check-only) + if hasattr(self, 'sanity_check_module_loaded') and not self.sanity_check_module_loaded: + extension = self.is_extension or kwargs.get('extension', False) + extra_modules = kwargs.get('extra_modules', None) + self.fake_mod_data = self.sanity_check_load_module(extension=extension, extra_modules=extra_modules) + # don't add user site directory to sys.path (equivalent to python -s) # see https://www.python.org/dev/peps/pep-0370/; # must be set here to ensure that it is defined when running sanity check for extensions, @@ -834,23 +852,31 @@ def sanity_check_step(self, *args, **kwargs): kwargs.update({'exts_filter': exts_filter}) if self.cfg.get('sanity_pip_check', False): - pip_version = det_pip_version() + pip_version = det_pip_version(python_cmd=python_cmd) + if pip_version: + pip_check_command = "%s -m pip check" % python_cmd + if LooseVersion(pip_version) >= LooseVersion('9.0.0'): if not self.is_extension: # for stand-alone Python package installations (not part of a bundle of extensions), - # we need to load the fake module file, otherwise the Python package being installed - # is not "in view", and we will overlook missing dependencies... - fake_mod_data = self.load_fake_module(purge=True) + # the (fake or real) module file must be loaded at this point, + # otherwise the Python package being installed is not "in view", + # and we will overlook missing dependencies... + loaded_modules = [x['mod_name'] for x in self.modules_tool.list()] + if self.short_mod_name not in loaded_modules: + self.log.debug("Currently loaded modules: %s", loaded_modules) + raise EasyBuildError("%s module is not loaded, this should never happen...", + self.short_mod_name) pip_check_errors = [] - pip_check_msg, ec = run_cmd("pip check", log_ok=False) + pip_check_msg, ec = run_cmd(pip_check_command, log_ok=False) if ec: - pip_check_errors.append('`pip check` failed:\n%s' % pip_check_msg) + pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg)) else: - self.log.info('`pip check` completed successfully') + self.log.info('`%s` completed successfully' % pip_check_command) # Also check for a common issue where the package version shows up as 0.0.0 often caused # by using setup.py as the installation method for a package which is released as a generic wheel @@ -869,8 +895,8 @@ def sanity_check_step(self, *args, **kwargs): version = next(pkg['version'] for pkg in pkgs if pkg['name'] == unversioned_package) except StopIteration: msg = ('Package %s in unversioned_packages was not found in the installed packages. ' - 'Check that the name from `pip list` is used which may be different than the ' - 'module name.' % unversioned_package) + 'Check that the name from `python -m pip list` is used which may be different ' + 'than the module name.' % unversioned_package) else: msg = ('Package %s in unversioned_packages has a version of %s which is valid. ' 'Please remove it from unversioned_packages.' % (unversioned_package, version)) @@ -888,16 +914,21 @@ def sanity_check_step(self, *args, **kwargs): ) % (faulty_version, '\n'.join(faulty_pkg_names)) pip_check_errors.append(msg) - if not self.is_extension: - self.clean_up_fake_module(fake_mod_data) - if pip_check_errors: raise EasyBuildError('\n'.join(pip_check_errors)) else: - raise EasyBuildError("pip >= 9.0.0 is required for running 'pip check', found %s", pip_version) + raise EasyBuildError("pip >= 9.0.0 is required for running '%s', found %s", + pip_check_command, + pip_version) else: raise EasyBuildError("Failed to determine pip version!") + # ExtensionEasyBlock handles loading modules correctly for multi_deps, so we clean up fake_mod_data + # and let ExtensionEasyBlock do its job + if 'Python' in self.cfg["multi_deps"] and self.fake_mod_data: + self.clean_up_fake_module(self.fake_mod_data) + self.sanity_check_module_loaded = False + parent_success, parent_fail_msg = super(PythonPackage, self).sanity_check_step(*args, **kwargs) if parent_fail_msg: diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 3caf702d22..897408503c 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -134,10 +134,14 @@ def make_cmdline_cmd(self, prefix=None): else: prefix = '' - if self.start_dir: - loc = os.path.join(self.ext_dir or os.path.sep, self.start_dir) - else: + loc = self.start_dir + if loc is None: loc = self.ext_dir or self.ext_src + elif not os.path.isabs(loc): + # TODO: deprecated behaviour in framework 4.7.1, remove after 5.0 + loc = os.path.join(self.ext_dir or os.path.sep, loc) + deprecation_msg = "Found relative path in start_dir, please upgrade to easybuild-framework>=4.7.1" + self.log.deprecated(deprecation_msg, '5.0') cmd = ' '.join([ self.cfg['preinstallopts'], diff --git a/easybuild/easyblocks/generic/rpm.py b/easybuild/easyblocks/generic/rpm.py index e127977c44..0a3f284c7f 100644 --- a/easybuild/easyblocks/generic/rpm.py +++ b/easybuild/easyblocks/generic/rpm.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -94,6 +94,7 @@ def rebuild_rpm(rpm_path, targetdir): class Rpm(Binary): """ Support for installing RPM files. + - sources is a list of rpms - installation is with --nodeps (so the sources list has to be complete) """ diff --git a/easybuild/easyblocks/generic/rubygem.py b/easybuild/easyblocks/generic/rubygem.py index 8297677534..58ecfa05c4 100644 --- a/easybuild/easyblocks/generic/rubygem.py +++ b/easybuild/easyblocks/generic/rubygem.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/scons.py b/easybuild/easyblocks/generic/scons.py index 4b2fe3de34..9820cb5855 100644 --- a/easybuild/easyblocks/generic/scons.py +++ b/easybuild/easyblocks/generic/scons.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/systemcompiler.py b/easybuild/easyblocks/generic/systemcompiler.py index 2d937897cd..def79c8543 100644 --- a/easybuild/easyblocks/generic/systemcompiler.py +++ b/easybuild/easyblocks/generic/systemcompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/systemmpi.py b/easybuild/easyblocks/generic/systemmpi.py index 3b4ad8915e..5ba5a3f812 100644 --- a/easybuild/easyblocks/generic/systemmpi.py +++ b/easybuild/easyblocks/generic/systemmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/tarball.py b/easybuild/easyblocks/generic/tarball.py index 09a55a84eb..5ad887cc81 100644 --- a/easybuild/easyblocks/generic/tarball.py +++ b/easybuild/easyblocks/generic/tarball.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -46,8 +46,7 @@ class Tarball(ExtensionEasyBlock): """ - Precompiled software supplied as a tarball: - - will unpack binary and copy it to the install dir + Precompiled software supplied as a tarball: will unpack binary and copy it to the install dir """ @staticmethod diff --git a/easybuild/easyblocks/generic/toolchain.py b/easybuild/easyblocks/generic/toolchain.py index 326aa35fa9..389182c684 100644 --- a/easybuild/easyblocks/generic/toolchain.py +++ b/easybuild/easyblocks/generic/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/versionindependentpythonpackage.py b/easybuild/easyblocks/generic/versionindependentpythonpackage.py index 0f1da9c1c5..f6cc181b41 100644 --- a/easybuild/easyblocks/generic/versionindependentpythonpackage.py +++ b/easybuild/easyblocks/generic/versionindependentpythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/vscpythonpackage.py b/easybuild/easyblocks/generic/vscpythonpackage.py index 4dee7968c7..772aeb3946 100644 --- a/easybuild/easyblocks/generic/vscpythonpackage.py +++ b/easybuild/easyblocks/generic/vscpythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/waf.py b/easybuild/easyblocks/generic/waf.py index bc0f2885b7..30d36a72dd 100644 --- a/easybuild/easyblocks/generic/waf.py +++ b/easybuild/easyblocks/generic/waf.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/h/hadoop.py b/easybuild/easyblocks/h/hadoop.py index 26805dcd54..aa1ec66619 100644 --- a/easybuild/easyblocks/h/hadoop.py +++ b/easybuild/easyblocks/h/hadoop.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/h/hdf5.py b/easybuild/easyblocks/h/hdf5.py index 06b82c635b..6ee28991e3 100644 --- a/easybuild/easyblocks/h/hdf5.py +++ b/easybuild/easyblocks/h/hdf5.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -77,10 +77,16 @@ def configure_step(self): self.cfg.update('configopts', "--with-pic --with-pthread --enable-shared") self.cfg.update('configopts', "--enable-cxx --enable-fortran %s" % fcomp) + # make C API thread safe (C++ / FORTRAN APIs are unaffected) + self.cfg.update('configopts', "--enable-threadsafe") + + # --enable-unsupported is needed to allow --enable-threadsafe to be used together with --enable-cxx + self.cfg.update('configopts', "--enable-unsupported") + # MPI and C++ support enabled requires --enable-unsupported, because this is untested by HDF5 # also returns False if MPI is not supported by this toolchain if self.toolchain.options.get('usempi', None): - self.cfg.update('configopts', "--enable-unsupported --enable-parallel") + self.cfg.update('configopts', "--enable-parallel") mpich_mpi_families = [toolchain.INTELMPI, toolchain.MPICH, toolchain.MPICH2, toolchain.MVAPICH2] if self.toolchain.mpi_family() in mpich_mpi_families: self.cfg.update('buildopts', 'CXXFLAGS="$CXXFLAGS -DMPICH_IGNORE_CXX_SEEK"') diff --git a/easybuild/easyblocks/h/healpix.py b/easybuild/easyblocks/h/healpix.py index 08170825e5..badc564abb 100644 --- a/easybuild/easyblocks/h/healpix.py +++ b/easybuild/easyblocks/h/healpix.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/h/hpcg.py b/easybuild/easyblocks/h/hpcg.py index be80012ba3..8d68fa34ea 100644 --- a/easybuild/easyblocks/h/hpcg.py +++ b/easybuild/easyblocks/h/hpcg.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/h/hpl.py b/easybuild/easyblocks/h/hpl.py index 21e700141b..d408a7b05e 100644 --- a/easybuild/easyblocks/h/hpl.py +++ b/easybuild/easyblocks/h/hpl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/h/hypre.py b/easybuild/easyblocks/h/hypre.py index 229f323203..556d40ff7b 100644 --- a/easybuild/easyblocks/h/hypre.py +++ b/easybuild/easyblocks/h/hypre.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index 55534a2897..37651f9e30 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/iccifort.py b/easybuild/easyblocks/i/iccifort.py index 3e41077951..b2f0d01492 100644 --- a/easybuild/easyblocks/i/iccifort.py +++ b/easybuild/easyblocks/i/iccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2022 Bart Oldeman, McGill University, Compute Canada +# Copyright 2019-2023 Bart Oldeman, McGill University, Compute Canada # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/easyblocks/i/ifort.py b/easybuild/easyblocks/i/ifort.py index 679d7d410c..d47013a510 100644 --- a/easybuild/easyblocks/i/ifort.py +++ b/easybuild/easyblocks/i/ifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index 487915edda..c2bb129b2e 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/imkl_fftw.py b/easybuild/easyblocks/i/imkl_fftw.py index 80d4c26a0a..e8f8ee4cce 100644 --- a/easybuild/easyblocks/i/imkl_fftw.py +++ b/easybuild/easyblocks/i/imkl_fftw.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/imod.py b/easybuild/easyblocks/i/imod.py index 97393a3a61..75d0760041 100644 --- a/easybuild/easyblocks/i/imod.py +++ b/easybuild/easyblocks/i/imod.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index ee443ff2f2..7eee0df099 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/inspector.py b/easybuild/easyblocks/i/inspector.py index 0db52d56d9..e3ce7653b1 100644 --- a/easybuild/easyblocks/i/inspector.py +++ b/easybuild/easyblocks/i/inspector.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index 28089de5af..3d6d9b030e 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -1,5 +1,5 @@ # # -# Copyright 2021-2022 Ghent University +# Copyright 2021-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/ipp.py b/easybuild/easyblocks/i/ipp.py index 8ff436685d..789a44d71c 100644 --- a/easybuild/easyblocks/i/ipp.py +++ b/easybuild/easyblocks/i/ipp.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/ironpython.py b/easybuild/easyblocks/i/ironpython.py index 07874520fb..9b5eee4111 100644 --- a/easybuild/easyblocks/i/ironpython.py +++ b/easybuild/easyblocks/i/ironpython.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/itac.py b/easybuild/easyblocks/i/itac.py index 197d154fae..1f37b8ab01 100644 --- a/easybuild/easyblocks/i/itac.py +++ b/easybuild/easyblocks/i/itac.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index 2c3535969d..63e0499321 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/j/jaxlib.py b/easybuild/easyblocks/j/jaxlib.py index 0776da73d0..dac39bf16c 100644 --- a/easybuild/easyblocks/j/jaxlib.py +++ b/easybuild/easyblocks/j/jaxlib.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/l/lammps.py b/easybuild/easyblocks/l/lammps.py index e4ad3962db..8b4b4dbe2e 100644 --- a/easybuild/easyblocks/l/lammps.py +++ b/easybuild/easyblocks/l/lammps.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -86,6 +86,7 @@ 'TURING75', # NVIDIA Turing generation CC 7.5 GPU 'AMPERE80', # NVIDIA Ampere generation CC 8.0 GPU 'AMPERE86', # NVIDIA Ampere generation CC 8.6 GPU + 'HOPPER90', # NVIDIA Hopper generation CC 9.0 GPU 'VEGA900', # AMD GPU MI25 GFX900 'VEGA906', # AMD GPU MI50/MI60 GFX906 'VEGA908', # AMD GPU MI100 GFX908 @@ -135,6 +136,7 @@ '7.5': 'TURING75', # NVIDIA Turing generation CC 7.5 '8.0': 'AMPERE80', # NVIDIA Ampere generation CC 8.0 '8.6': 'AMPERE86', # NVIDIA Ampere generation CC 8.6 + '9.0': 'HOPPER90', # NVIDIA Hopper generation CC 9.0 } # lammps version, which caused the most changes. This may not be precise, but it does work with existing easyconfigs @@ -346,7 +348,7 @@ def configure_step(self, **kwargs): # https://lammps.sandia.gov/doc/Build_extras.html # KOKKOS if self.cfg['kokkos']: - print_msg("Using Kokkos arch: CPU - %s, GPU - %s" % (processor_arch, gpu_arch)) + print_msg("Using Kokkos package with arch: CPU - %s, GPU - %s" % (processor_arch, gpu_arch)) self.cfg.update('configopts', '-D%sKOKKOS=on' % self.pkg_prefix) if self.toolchain.options.get('openmp', None): @@ -372,6 +374,7 @@ def configure_step(self, **kwargs): # CUDA only elif self.cuda: + print_msg("Using GPU package (not Kokkos) with arch: CPU - %s, GPU - %s" % (processor_arch, gpu_arch)) self.cfg.update('configopts', '-D%sGPU=on' % self.pkg_prefix) self.cfg.update('configopts', '-DGPU_API=cuda') self.cfg.update('configopts', '-DGPU_ARCH=%s' % get_cuda_gpu_arch(cuda_cc)) diff --git a/easybuild/easyblocks/l/lapack.py b/easybuild/easyblocks/l/lapack.py index b60ecf791d..112875a8fa 100644 --- a/easybuild/easyblocks/l/lapack.py +++ b/easybuild/easyblocks/l/lapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/l/libdrm.py b/easybuild/easyblocks/l/libdrm.py index ece8c023ae..5cc2786f69 100644 --- a/easybuild/easyblocks/l/libdrm.py +++ b/easybuild/easyblocks/l/libdrm.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/l/libint.py b/easybuild/easyblocks/l/libint.py index f2e6fa4ba8..aa76bb6020 100644 --- a/easybuild/easyblocks/l/libint.py +++ b/easybuild/easyblocks/l/libint.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/l/libqglviewer.py b/easybuild/easyblocks/l/libqglviewer.py index ec9dfd0c16..55d6a5d0a6 100644 --- a/easybuild/easyblocks/l/libqglviewer.py +++ b/easybuild/easyblocks/l/libqglviewer.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/l/libsmm.py b/easybuild/easyblocks/l/libsmm.py index 6addd7081e..19a9f6c145 100644 --- a/easybuild/easyblocks/l/libsmm.py +++ b/easybuild/easyblocks/l/libsmm.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/l/libxml2.py b/easybuild/easyblocks/l/libxml2.py index 159f48304a..73e83cc804 100644 --- a/easybuild/easyblocks/l/libxml2.py +++ b/easybuild/easyblocks/l/libxml2.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -92,6 +92,10 @@ def configure_step(self): else: self.cfg.update('configopts', '--without-python') + # building of static libraries is disabled by default for libxml >= 2.10 + if LooseVersion(self.version) >= LooseVersion('2.10'): + self.cfg.update('configopts', '--enable-static') + ConfigureMake.configure_step(self) if self.with_python_bindings: diff --git a/easybuild/easyblocks/l/llvm.py b/easybuild/easyblocks/l/llvm.py index 73703b99d3..c7f06d6b6e 100644 --- a/easybuild/easyblocks/l/llvm.py +++ b/easybuild/easyblocks/l/llvm.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2022 Ghent University +# Copyright 2020-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -26,11 +26,15 @@ EasyBuild support for building and installing LLVM, implemented as an easyblock @author: Simon Branford (University of Birmingham) +@author: Kenneth Hoste (Ghent University) """ +import os + from easybuild.easyblocks.clang import CLANG_TARGETS, DEFAULT_TARGETS_MAP from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import change_dir, symlink from easybuild.tools.modules import get_software_root from easybuild.tools.systemtools import get_cpu_architecture from distutils.version import LooseVersion @@ -98,4 +102,15 @@ def configure_step(self): self.cfg.update('configopts', '-DLLVM_TARGETS_TO_BUILD="%s"' % ';'.join(build_targets)) + if LooseVersion(self.version) >= LooseVersion('15.0'): + # make sure that CMake modules are available in build directory, + # and if so make a 'cmake' symlink so LLVM can find them + cmake_modules_path = os.path.join(self.builddir, 'cmake-%s.src' % self.version) + if os.path.exists(cmake_modules_path): + cwd = change_dir(self.builddir) + symlink('cmake-%s.src' % self.version, 'cmake') + change_dir(cwd) + else: + raise EasyBuildError("Failed to find unpacked CMake modules directory at %s", cmake_modules_path) + super(EB_LLVM, self).configure_step() diff --git a/easybuild/easyblocks/l/lua.py b/easybuild/easyblocks/l/lua.py index fce55919ce..16fc4065e1 100644 --- a/easybuild/easyblocks/l/lua.py +++ b/easybuild/easyblocks/l/lua.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2022 Ghent University +# Copyright 2018-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mamba.py b/easybuild/easyblocks/m/mamba.py new file mode 100644 index 0000000000..c6cfd49e0b --- /dev/null +++ b/easybuild/easyblocks/m/mamba.py @@ -0,0 +1,49 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing Mamba, implemented as an easyblock + +@author: Caspar van Leeuwen (SURF) +@author: Kenneth Hoste (HPC-UGent) +""" + +import os + +from easybuild.easyblocks.a.anaconda import EB_Anaconda + + +class EB_Mamba(EB_Anaconda): + """Support for building/installing Mamba.""" + + def sanity_check_step(self): + """ + Custom sanity check for Mamba + """ + custom_paths = { + 'files': [os.path.join('bin', x) for x in ['2to3', 'conda', 'pydoc', 'python', 'mamba']], + 'dirs': ['etc', 'lib', 'pkgs'], + } + # Directly call EB_Anaconda's super, as this sanity_check_step should _overwrite_ Anaconda's (not call it) + super(EB_Anaconda, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/m/maple.py b/easybuild/easyblocks/m/maple.py index e41e47517f..65868877ff 100644 --- a/easybuild/easyblocks/m/maple.py +++ b/easybuild/easyblocks/m/maple.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,6 +30,7 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) +@author: Sven Hansen (RWTH Aachen University) """ import glob import os @@ -61,6 +62,8 @@ def install_step(self): "Do you accept this license? [y/n]:": 'y', 'ENTER AN ABSOLUTE PATH, OR PRESS TO ACCEPT THE DEFAULT :': self.installdir, 'IS THIS CORRECT? (Y/N):': 'Y', + 'Language Selection\n\nPlease select the installation language\n[1] English - English\n[2] Japanese - \n' + 'Please choose an option [1] : ': '1', 'Do you wish to have a shortcut installed on your desktop? ->1- Yes 2- No ENTER THE NUMBER ' + 'FOR YOUR CHOICE, OR PRESS TO ACCEPT THE DEFAULT::': '2', "Do you wish to have a shortcut installed on your desktop? [Y/n]:": 'n', @@ -69,7 +72,7 @@ def install_step(self): 'PRESS TO EXIT THE INSTALLER:': '', 'License server (DEFAULT: ):': self.cfg['license_server'], "License server []:": self.cfg['license_server'], - 'Port number (optional) (DEFAULT: ):': '', + 'Port number (optional) (DEFAULT: ):': self.cfg['license_server_port'] or '', '->1- Configure toolbox for Matlab 2- Do not configure at this time ENTER THE NUMBER FOR YOUR CHOICE, ' + 'OR PRESS TO ACCEPT THE DEFAULT::': '2', "MATLAB Configuration [y/N]:": 'n', @@ -81,8 +84,9 @@ def install_step(self): r"Choose Install Folder \[.*\]:": self.installdir, r"\[2\] Network License.*\nPlease choose an option \[.\] :": '2', r"\[1\] Single Server.*\n.*\nPlease choose an option \[.\] :": '1', - r"Port number \[[0-9]+\]:": '', + r"Port number \[[0-9]+\]:": self.cfg['license_server_port'] or '', r"Enable periodic checking for Maple .* updates after installation \[Y/n\]:": 'n', + r'Pre-Installation Summary[\s\S]*': '', } no_qa = [ @@ -91,10 +95,10 @@ def install_step(self): 'Launching installer...', "Configuring the installer for this system's environment...", 'Unpacking the JRE...', - r'\[[-|]*', + r'\[[-|#|]*', ] - run_cmd_qa(cmd, qa, std_qa=std_qa, no_qa=no_qa, log_all=True, simple=True) + run_cmd_qa(cmd, qa, std_qa=std_qa, no_qa=no_qa, log_all=True, simple=True, maxhits=150) upgrade_installers = glob.glob(os.path.join(self.builddir, 'Maple*Upgrade*')) if upgrade_installers: diff --git a/easybuild/easyblocks/m/mathematica.py b/easybuild/easyblocks/m/mathematica.py index 288321d063..9f1fb78827 100644 --- a/easybuild/easyblocks/m/mathematica.py +++ b/easybuild/easyblocks/m/mathematica.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -68,7 +68,7 @@ def install_step(self): install_script_glob = '%s_%s_LINUX*.sh' % (self.name, self.version) # Starting at V13, Mathematica have renamed their install file... if LooseVersion(self.version) >= LooseVersion("13"): - install_script_glob = '%s_%s_BNDL_LINUX*.sh' % (self.name, self.version) + install_script_glob = '%s_%s_*LINUX*.sh' % (self.name, self.version) matches = glob.glob(install_script_glob) if len(matches) == 1: diff --git a/easybuild/easyblocks/m/matlab.py b/easybuild/easyblocks/m/matlab.py index bb0487560e..b260e703de 100644 --- a/easybuild/easyblocks/m/matlab.py +++ b/easybuild/easyblocks/m/matlab.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -55,6 +55,7 @@ def __init__(self, *args, **kwargs): super(EB_MATLAB, self).__init__(*args, **kwargs) self.comp_fam = None self.configfile = os.path.join(self.builddir, 'my_installer_input.txt') + self.outputfile = os.path.join(self.builddir, 'my_installer_output.txt') @staticmethod def extra_options(): @@ -98,12 +99,14 @@ def configure_step(self): regagree = re.compile(br"^# agreeToLicense=.*", re.M) regmode = re.compile(br"^# mode=.*", re.M) reglicpath = re.compile(br"^# licensePath=.*", re.M) + regoutfile = re.compile(br"^# outputFile=.*", re.M) # must use byte-strings here when using Python 3, see above config = regdest.sub(b"destinationFolder=%s" % self.installdir.encode('utf-8'), config) config = regagree.sub(b"agreeToLicense=Yes", config) config = regmode.sub(b"mode=silent", config) config = reglicpath.sub(b"licensePath=%s" % licfile.encode('utf-8'), config) + config = regoutfile.sub(b"outputFile=%s" % self.outputfile.encode('utf-8'), config) write_file(self.configfile, config) @@ -159,7 +162,11 @@ def install_step(self): keys = self.cfg['key'] if keys is None: - keys = os.getenv('EB_MATLAB_KEY', '00000-00000-00000-00000-00000-00000-00000-00000-00000-00000') + try: + keys = os.environ['EB_MATLAB_KEY'] + except KeyError: + raise EasyBuildError("The MATLAB install key is not set. This can be set either with the environment " + "variable EB_MATLAB_KEY or by the easyconfig variable 'key'.") if isinstance(keys, string_type): keys = keys.split(',') @@ -184,6 +191,12 @@ def install_step(self): # check installer output for known signs of trouble patterns = [ "Error: You have entered an invalid File Installation Key", + "Not a valid key", + "All selected products are already installed", + "The application encountered an unexpected error and needs to close", + "Error: Unable to write to", + "Exiting with status -\\d", + "End - Unsuccessful", ] for pattern in patterns: @@ -191,6 +204,10 @@ def install_step(self): if regex.search(out): raise EasyBuildError("Found error pattern '%s' in output of installation command '%s': %s", regex.pattern, cmd, out) + with open(self.outputfile) as f: + if regex.search(f.read()): + raise EasyBuildError("Found error pattern '%s' in output file of installer", + regex.pattern) def sanity_check_step(self): """Custom sanity check for MATLAB.""" diff --git a/easybuild/easyblocks/m/mcr.py b/easybuild/easyblocks/m/mcr.py index f3f3830d26..49cb3b36f4 100644 --- a/easybuild/easyblocks/m/mcr.py +++ b/easybuild/easyblocks/m/mcr.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mesa.py b/easybuild/easyblocks/m/mesa.py index 02c6dba274..684f37c2be 100644 --- a/easybuild/easyblocks/m/mesa.py +++ b/easybuild/easyblocks/m/mesa.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/metagenome_atlas.py b/easybuild/easyblocks/m/metagenome_atlas.py index 90536059d6..3658b5cfa6 100644 --- a/easybuild/easyblocks/m/metagenome_atlas.py +++ b/easybuild/easyblocks/m/metagenome_atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2022 Ghent University +# Copyright 2020-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/metavelvet.py b/easybuild/easyblocks/m/metavelvet.py index bcbda38285..57311202e1 100644 --- a/easybuild/easyblocks/m/metavelvet.py +++ b/easybuild/easyblocks/m/metavelvet.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/m/metis.py b/easybuild/easyblocks/m/metis.py index 2e4aa03f1c..b8582b7842 100644 --- a/easybuild/easyblocks/m/metis.py +++ b/easybuild/easyblocks/m/metis.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/modeller.py b/easybuild/easyblocks/m/modeller.py index 57ea0aa3e2..feb1876b16 100755 --- a/easybuild/easyblocks/m/modeller.py +++ b/easybuild/easyblocks/m/modeller.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2022 Ghent University +# Copyright 2014-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). diff --git a/easybuild/easyblocks/m/molpro.py b/easybuild/easyblocks/m/molpro.py index e65bed2401..86358f0dcd 100644 --- a/easybuild/easyblocks/m/molpro.py +++ b/easybuild/easyblocks/m/molpro.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mono.py b/easybuild/easyblocks/m/mono.py index 31e04439c7..b23088ba65 100644 --- a/easybuild/easyblocks/m/mono.py +++ b/easybuild/easyblocks/m/mono.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mothur.py b/easybuild/easyblocks/m/mothur.py index 406294779e..78f5f0f832 100644 --- a/easybuild/easyblocks/m/mothur.py +++ b/easybuild/easyblocks/m/mothur.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/motioncor2.py b/easybuild/easyblocks/m/motioncor2.py index 12201abb0f..9224e89d9b 100644 --- a/easybuild/easyblocks/m/motioncor2.py +++ b/easybuild/easyblocks/m/motioncor2.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2022 Ghent University +# Copyright 2019-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mpich.py b/easybuild/easyblocks/m/mpich.py index 9ab7c65e92..df97d7265b 100644 --- a/easybuild/easyblocks/m/mpich.py +++ b/easybuild/easyblocks/m/mpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University, Forschungszentrum Juelich +# Copyright 2009-2023 Ghent University, Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mrbayes.py b/easybuild/easyblocks/m/mrbayes.py index 097f73839a..8fc399b5d3 100644 --- a/easybuild/easyblocks/m/mrbayes.py +++ b/easybuild/easyblocks/m/mrbayes.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mrtrix.py b/easybuild/easyblocks/m/mrtrix.py index 74b866d4f7..7b148131a8 100644 --- a/easybuild/easyblocks/m/mrtrix.py +++ b/easybuild/easyblocks/m/mrtrix.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -65,9 +65,6 @@ def configure_step(self): env.setvar('QMAKE_CXX', os.getenv('CXX')) cmd = "python configure -verbose" - if LooseVersion(self.version) >= LooseVersion('3.0'): - cmd += " -openmp" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) def build_step(self): diff --git a/easybuild/easyblocks/m/msm.py b/easybuild/easyblocks/m/msm.py index 04f3903831..1269e198ad 100644 --- a/easybuild/easyblocks/m/msm.py +++ b/easybuild/easyblocks/m/msm.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mtl4.py b/easybuild/easyblocks/m/mtl4.py index fc922cb1eb..e788498643 100644 --- a/easybuild/easyblocks/m/mtl4.py +++ b/easybuild/easyblocks/m/mtl4.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mummer.py b/easybuild/easyblocks/m/mummer.py index 244ae489bd..a5038cb895 100644 --- a/easybuild/easyblocks/m/mummer.py +++ b/easybuild/easyblocks/m/mummer.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). diff --git a/easybuild/easyblocks/m/mumps.py b/easybuild/easyblocks/m/mumps.py index fcbf206cd9..26c43adbf9 100644 --- a/easybuild/easyblocks/m/mumps.py +++ b/easybuild/easyblocks/m/mumps.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mutil.py b/easybuild/easyblocks/m/mutil.py index dcb8e3d10e..bb0d8dc11b 100644 --- a/easybuild/easyblocks/m/mutil.py +++ b/easybuild/easyblocks/m/mutil.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2022 Ghent University +# Copyright 2016-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mvapich2.py b/easybuild/easyblocks/m/mvapich2.py index e319aab8b8..3aecd40103 100644 --- a/easybuild/easyblocks/m/mvapich2.py +++ b/easybuild/easyblocks/m/mvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University, Forschungszentrum Juelich +# Copyright 2009-2023 Ghent University, Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mxnet.py b/easybuild/easyblocks/m/mxnet.py index e8a865ef20..7dee5aeed6 100644 --- a/easybuild/easyblocks/m/mxnet.py +++ b/easybuild/easyblocks/m/mxnet.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2022 Free University of Brussels (VUB) +# Copyright 2018-2023 Free University of Brussels (VUB) # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mymedialite.py b/easybuild/easyblocks/m/mymedialite.py index 6b78a342ce..d4876e2d31 100644 --- a/easybuild/easyblocks/m/mymedialite.py +++ b/easybuild/easyblocks/m/mymedialite.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/namd.py b/easybuild/easyblocks/n/namd.py index b48503bf2b..a0f4c82b95 100644 --- a/easybuild/easyblocks/n/namd.py +++ b/easybuild/easyblocks/n/namd.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2013-2022 CaSToRC, The Cyprus Institute +# Copyright:: Copyright 2013-2023 CaSToRC, The Cyprus Institute # Authors:: George Tsouloupas # License:: MIT/GPL # $Id$ @@ -201,10 +201,12 @@ def test_step(self): ppn = '' if self.toolchain.options.get('openmp', False): ppn = '+ppn 2' - cmd = "%(namd)s %(ppn)s %(testdir)s" % { + cmd = "%(pretestopts)s %(namd)s %(ppn)s %(testopts)s %(testdir)s" % { 'namd': namdcmd, 'ppn': ppn, + 'pretestopts': self.cfg['pretestopts'], 'testdir': os.path.join(self.cfg['start_dir'], self.namd_arch, 'src', 'alanin'), + 'testopts': self.cfg['testopts'], } out, ec = run_cmd(cmd, simple=False) if ec == 0: diff --git a/easybuild/easyblocks/n/nccl.py b/easybuild/easyblocks/n/nccl.py index 33ec98b675..e51707013f 100644 --- a/easybuild/easyblocks/n/nccl.py +++ b/easybuild/easyblocks/n/nccl.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2022 Ghent University +# Copyright 2021-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/ncl.py b/easybuild/easyblocks/n/ncl.py index 34cebdd2b6..ddc0b3cc99 100644 --- a/easybuild/easyblocks/n/ncl.py +++ b/easybuild/easyblocks/n/ncl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/ncurses.py b/easybuild/easyblocks/n/ncurses.py index e8a0a75af9..019d31b153 100644 --- a/easybuild/easyblocks/n/ncurses.py +++ b/easybuild/easyblocks/n/ncurses.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/n/nemo.py b/easybuild/easyblocks/n/nemo.py index 3116e4c23c..bb6cfad970 100644 --- a/easybuild/easyblocks/n/nemo.py +++ b/easybuild/easyblocks/n/nemo.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/netcdf.py b/easybuild/easyblocks/n/netcdf.py index 879136495b..45017748bb 100644 --- a/easybuild/easyblocks/n/netcdf.py +++ b/easybuild/easyblocks/n/netcdf.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -141,7 +141,12 @@ def sanity_check_step(self): 'dirs': [] } - super(EB_netCDF, self).sanity_check_step(custom_paths=custom_paths) + custom_commands = [ + "nc-config --help", + "ncgen -h" if LooseVersion(self.version) > LooseVersion("4.6.1") else "ncgen -H", + ] + + super(EB_netCDF, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) def set_netcdf_env_vars(log): diff --git a/easybuild/easyblocks/n/netcdf4_python.py b/easybuild/easyblocks/n/netcdf4_python.py index 4c24e96e0e..a78dab4f4b 100644 --- a/easybuild/easyblocks/n/netcdf4_python.py +++ b/easybuild/easyblocks/n/netcdf4_python.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/netcdf_fortran.py b/easybuild/easyblocks/n/netcdf_fortran.py index 774e63c50c..9de127e0a0 100644 --- a/easybuild/easyblocks/n/netcdf_fortran.py +++ b/easybuild/easyblocks/n/netcdf_fortran.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/neuron.py b/easybuild/easyblocks/n/neuron.py index 6084b87a91..a094670be6 100644 --- a/easybuild/easyblocks/n/neuron.py +++ b/easybuild/easyblocks/n/neuron.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/nim.py b/easybuild/easyblocks/n/nim.py index fa524380fe..3ecc804cff 100644 --- a/easybuild/easyblocks/n/nim.py +++ b/easybuild/easyblocks/n/nim.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2022 Ghent University +# Copyright 2018-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/nose.py b/easybuild/easyblocks/n/nose.py index 7cbc785585..091f6a756d 100644 --- a/easybuild/easyblocks/n/nose.py +++ b/easybuild/easyblocks/n/nose.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index 00fe5c89a7..1c62b4502c 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2022 Ghent University +# Copyright 2019-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -83,7 +83,7 @@ def configure_step(self): os.path.join(self.imkl_root, 'mkl', 'latest', 'lib', 'intel64'), ] mkl_include_dirs = os.path.join(self.imkl_root, 'mkl', 'latest', 'include') - mkl_libs = ['mkl_intel_lp64', 'mkl_intel_thread', 'mkl_core', 'iomp5'] + mkl_libs = ['mkl_rt'] else: mkl_lib_dirs = [ os.path.join(self.imkl_root, 'mkl', 'lib', 'intel64'), @@ -103,15 +103,25 @@ def configure_step(self): else: site_cfg_lines.append("mkl_libs = %s" % ', '.join(mkl_libs)) - write_file('site.cfg', '\n'.join(site_cfg_lines)) + site_cfg_txt = '\n'.join(site_cfg_lines) + write_file('site.cfg', site_cfg_txt) + self.log.info("site.cfg used for numexpr:\n" + site_cfg_txt) def sanity_check_step(self): """Custom sanity check for numexpr.""" custom_commands = [] + # imkl_root may still be None, for example when running with --sanity-check-only + if self.imkl_root is None: + self.imkl_root = get_software_root('imkl') + # if Intel MKL is available, make sure VML is used if self.imkl_root: custom_commands.append("python -c 'import numexpr; assert(numexpr.use_vml)'") + # for sufficiently recent versions of numexpr, also do a more extensive check for VML support + if LooseVersion(self.version) >= LooseVersion('2.7.3'): + custom_commands.append("""python -c "import numexpr; numexpr.set_vml_accuracy_mode('low')" """) + return super(EB_numexpr, self).sanity_check_step(custom_commands=custom_commands) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index 484a8d9f72..f72e2671d4 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/nvhpc.py b/easybuild/easyblocks/n/nvhpc.py index eabb8f0600..b9b27cb009 100644 --- a/easybuild/easyblocks/n/nvhpc.py +++ b/easybuild/easyblocks/n/nvhpc.py @@ -1,6 +1,6 @@ ## -# Copyright 2015-2022 Bart Oldeman -# Copyright 2016-2022 Forschungszentrum Juelich +# Copyright 2015-2023 Bart Oldeman +# Copyright 2016-2023 Forschungszentrum Juelich # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -164,7 +164,10 @@ def install_step(self): line = re.sub(r"^PATH=/", r"#PATH=/", line) sys.stdout.write(line) - cmd = "%s -x %s -g77 /" % (makelocalrc_filename, compilers_subdir) + if LooseVersion(self.version) >= LooseVersion('22.9'): + cmd = "%s -x %s" % (makelocalrc_filename, compilers_subdir) + else: + cmd = "%s -x %s -g77 /" % (makelocalrc_filename, compilers_subdir) run_cmd(cmd, log_all=True, simple=True) # If an OS libnuma is NOT found, makelocalrc creates symbolic links to libpgnuma.so diff --git a/easybuild/easyblocks/n/nwchem.py b/easybuild/easyblocks/n/nwchem.py index b3ba6d8b1a..667b0bbd1e 100644 --- a/easybuild/easyblocks/n/nwchem.py +++ b/easybuild/easyblocks/n/nwchem.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/o/ocaml.py b/easybuild/easyblocks/o/ocaml.py index e514aea86c..fb43648798 100644 --- a/easybuild/easyblocks/o/ocaml.py +++ b/easybuild/easyblocks/o/ocaml.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/o/octave.py b/easybuild/easyblocks/o/octave.py index 06ad78d0bb..d810251351 100644 --- a/easybuild/easyblocks/o/octave.py +++ b/easybuild/easyblocks/o/octave.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/o/openbabel.py b/easybuild/easyblocks/o/openbabel.py index 11858d37e4..b054cc56b0 100644 --- a/easybuild/easyblocks/o/openbabel.py +++ b/easybuild/easyblocks/o/openbabel.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/o/openblas.py b/easybuild/easyblocks/o/openblas.py index 4973fa0df1..996dd280e1 100644 --- a/easybuild/easyblocks/o/openblas.py +++ b/easybuild/easyblocks/o/openblas.py @@ -3,21 +3,39 @@ @author: Andrew Edmondson (University of Birmingham) @author: Alex Domingo (Vrije Universiteit Brussel) +@author: Kenneth Hoste (Ghent University) """ import os +import re from distutils.version import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.systemtools import POWER, get_cpu_architecture, get_shared_lib_ext -from easybuild.tools.build_log import print_warning +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import ERROR from easybuild.tools.run import run_cmd, check_log_for_errors +LAPACK_TEST_TARGET = 'lapack-test' TARGET = 'TARGET' class EB_OpenBLAS(ConfigureMake): """Support for building/installing OpenBLAS.""" + @staticmethod + def extra_options(): + """Custom easyconfig parameters for OpenBLAS easyblock.""" + extra_vars = { + 'max_failing_lapack_tests_num_errors': [0, "Maximum number of LAPACK tests failing " + "due to numerical errors", CUSTOM], + 'max_failing_lapack_tests_other_errors': [0, "Maximum number of LAPACK tests failing " + "due to non-numerical errors", CUSTOM], + 'run_lapack_tests': [False, "Run LAPACK tests during test step, " + "and check whether failing tests exceeds threshold", CUSTOM], + } + + return ConfigureMake.extra_options(extra_vars) + def configure_step(self): """ set up some options - but no configure command to run""" @@ -52,10 +70,13 @@ def build_step(self): """ Custom build step excluding the tests """ # Equivalent to `make all` without the tests - build_parts = ['libs', 'netlib'] - for buildopt in self.cfg['buildopts'].split(): - if 'BUILD_RELAPACK' in buildopt and '1' in buildopt: - build_parts += ['re_lapack'] + build_parts = [] + if LooseVersion(self.version) < LooseVersion('0.3.23'): + build_parts += ['libs', 'netlib'] + for buildopt in self.cfg['buildopts'].split(): + if 'BUILD_RELAPACK' in buildopt and '1' in buildopt: + build_parts += ['re_lapack'] + # just shared is necessary and sufficient with 0.3.23 + xianyi/OpenBLAS#3983 build_parts += ['shared'] # Pass CFLAGS through command line to avoid redefinitions (issue xianyi/OpenBLAS#818) @@ -72,10 +93,47 @@ def build_step(self): cmd = ' '.join([self.cfg['prebuildopts'], makecmd, ' '.join(build_parts), self.cfg['buildopts']]) run_cmd(cmd, log_all=True, simple=True) + def check_lapack_test_results(self, test_output): + """Check output of OpenBLAS' LAPACK test suite ('make lapack-test').""" + + # example: + # --> LAPACK TESTING SUMMARY <-- + # SUMMARY nb test run numerical error other error + # ================ =========== ================= ================ + # ... + # --> ALL PRECISIONS 4116982 4172 (0.101%) 0 (0.000%) + test_summary_pattern = r'\s+'.join([ + r"^--> ALL PRECISIONS", + r"(?P[0-9]+)", + r"(?P[0-9]+)\s+\([0-9.]+\%\)", + r"(?P[0-9]+)\s+\([0-9.]+\%\)", + ]) + regex = re.compile(test_summary_pattern, re.M) + res = regex.search(test_output) + if res: + (tot_cnt, fail_cnt_num_errors, fail_cnt_other_errors) = [int(x) for x in res.groups()] + msg = "%d LAPACK tests run - %d failed due to numerical errors - %d failed due to other errors" + self.log.info(msg, tot_cnt, fail_cnt_num_errors, fail_cnt_other_errors) + + if fail_cnt_other_errors > self.cfg['max_failing_lapack_tests_other_errors']: + raise EasyBuildError("Too many LAPACK tests failed due to non-numerical errors: %d (> %d)", + fail_cnt_other_errors, self.cfg['max_failing_lapack_tests_other_errors']) + + if fail_cnt_num_errors > self.cfg['max_failing_lapack_tests_num_errors']: + raise EasyBuildError("Too many LAPACK tests failed due to numerical errors: %d (> %d)", + fail_cnt_num_errors, self.cfg['max_failing_lapack_tests_num_errors']) + else: + raise EasyBuildError("Failed to find test summary using pattern '%s' in test output: %s", + test_summary_pattern, test_output) + def test_step(self): """ Mandatory test step plus optional runtest""" run_tests = ['tests'] + + if self.cfg['run_lapack_tests']: + run_tests += [LAPACK_TEST_TARGET] + if self.cfg['runtest']: run_tests += [self.cfg['runtest']] @@ -86,6 +144,10 @@ def test_step(self): # Raise an error if any test failed check_log_for_errors(out, [('FATAL ERROR', ERROR)]) + # check number of failing LAPACK tests more closely + if runtest == LAPACK_TEST_TARGET: + self.check_lapack_test_results(out) + def sanity_check_step(self): """ Custom sanity check for OpenBLAS """ custom_paths = { diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index 7d18d0906e..4b1511af1b 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2022 Ghent University +# Copyright 2018-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -165,7 +165,7 @@ def configure_step(self): # see https://github.com/opencv/opencv/wiki/CPU-optimizations-build-options if self.toolchain.options.get('optarch') and 'CPU_BASELINE' not in self.cfg['configopts']: optarch = build_option('optarch') - if optarch is None: + if not optarch: # optimize for host arch (let OpenCV detect it) self.cfg.update('configopts', '-DCPU_BASELINE=DETECT') elif optarch == OPTARCH_GENERIC: @@ -228,6 +228,9 @@ def make_module_extra(self): """Custom extra module file entries for OpenCV.""" txt = super(EB_OpenCV, self).make_module_extra() + if LooseVersion(self.version) >= LooseVersion('4.0'): + txt += self.module_generator.prepend_paths('CPATH', os.path.join('include', 'opencv4')) + txt += self.module_generator.prepend_paths('CLASSPATH', os.path.join('share', 'OpenCV', 'java')) if os.path.exists(os.path.join(self.installdir, self.pylibdir)): diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index bbdfc4e29e..7c8717b455 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -78,7 +78,11 @@ def __init__(self, *args, **kwargs): self.looseversion = LooseVersion(clean_version) - if 'extend' in self.name.lower(): + self.is_extend = 'extend' in self.name.lower() + self.is_dot_com = self.looseversion >= LooseVersion('1606') + self.is_dot_org = self.looseversion <= LooseVersion('100') + + if self.is_extend: if self.looseversion >= LooseVersion('3.0'): self.openfoamdir = 'foam-extend-%s' % self.version else: @@ -153,7 +157,7 @@ def configure_step(self): extra_flags = '-fuse-ld=bfd' # older versions of OpenFOAM-Extend require -fpermissive - if 'extend' in self.name.lower() and self.looseversion < LooseVersion('2.0'): + if self.is_extend and self.looseversion < LooseVersion('2.0'): extra_flags += ' -fpermissive' if self.looseversion < LooseVersion('3.0'): @@ -174,13 +178,13 @@ def configure_step(self): regex_subs = [(r"^(setenv|export) WM_THIRD_PARTY_USE_.*[ =].*$", r"# \g<0>")] # this does not work for OpenFOAM Extend lower than 2.0 - if 'extend' not in self.name.lower() or self.looseversion >= LooseVersion('2.0'): + if not self.is_extend or self.looseversion >= LooseVersion('2.0'): key = "WM_PROJECT_VERSION" regex_subs += [(r"^(setenv|export) %s=.*$" % key, r"export %s=%s #\g<0>" % (key, self.version))] WM_env_var = ['WM_COMPILER', 'WM_COMPILE_OPTION', 'WM_MPLIB', 'WM_THIRD_PARTY_DIR'] # OpenFOAM >= 3.0.0 can use 64 bit integers - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if not self.is_extend and self.looseversion >= LooseVersion('3.0'): WM_env_var.append('WM_LABEL_SIZE') for env_var in WM_env_var: regex_subs.append((r"^(setenv|export) (?P%s)[ =](?P.*)$" % env_var, @@ -257,14 +261,14 @@ def configure_step(self): env.setvar("WM_NCOMPPROCS", str(self.cfg['parallel'])) # OpenFOAM >= 3.0.0 can use 64 bit integers - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if not self.is_extend and self.looseversion >= LooseVersion('3.0'): if self.toolchain.options['i8']: env.setvar("WM_LABEL_SIZE", '64') else: env.setvar("WM_LABEL_SIZE", '32') # make sure lib/include dirs for dependencies are found - openfoam_extend_v3 = 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0') + openfoam_extend_v3 = self.is_extend and self.looseversion >= LooseVersion('3.0') if self.looseversion < LooseVersion("2") or openfoam_extend_v3: self.log.debug("List of deps: %s" % self.cfg.dependencies()) for dep in self.cfg.dependencies(): @@ -297,7 +301,7 @@ def build_step(self): setup_cmake_env(self.toolchain) precmd = "source %s" % os.path.join(self.builddir, self.openfoamdir, "etc", "bashrc") - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('4.0'): + if not self.is_extend and self.looseversion >= LooseVersion('4.0'): if self.looseversion >= LooseVersion('2006'): cleancmd = "cd $WM_PROJECT_DIR && wclean -platform -all && cd -" else: @@ -312,7 +316,7 @@ def build_step(self): 'prebuildopts': self.cfg['prebuildopts'], 'makecmd': os.path.join(self.builddir, self.openfoamdir, '%s'), } - if 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if self.is_extend and self.looseversion >= LooseVersion('3.0'): qa = { "Proceed without compiling ParaView [Y/n]": 'Y', "Proceed without compiling cudaSolvers? [Y/n]": 'Y', @@ -340,7 +344,7 @@ def det_psubdir(self): """Determine the platform-specific installation directory for OpenFOAM.""" # OpenFOAM >= 3.0.0 can use 64 bit integers # same goes for OpenFOAM-Extend >= 4.1 - if 'extend' in self.name.lower(): + if self.is_extend: set_int_size = self.looseversion >= LooseVersion('4.1') else: set_int_size = self.looseversion >= LooseVersion('3.0') @@ -357,7 +361,7 @@ def det_psubdir(self): arch = get_cpu_architecture() if arch == AARCH64: # Variants have different abbreviations for ARM64... - if self.looseversion < LooseVersion("100"): + if self.is_dot_org: archpart = 'Arm64' else: archpart = 'ARM64' @@ -386,7 +390,7 @@ def install_step(self): # to make sure they take precedence over the libraries in the dummy subdirectory shlib_ext = get_shared_lib_ext() psubdir = self.det_psubdir() - openfoam_extend_v3 = 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0') + openfoam_extend_v3 = self.is_extend and self.looseversion >= LooseVersion('3.0') if openfoam_extend_v3 or self.looseversion < LooseVersion("2"): libdir = os.path.join(self.installdir, self.openfoamdir, "lib", psubdir) else: @@ -410,7 +414,7 @@ def sanity_check_step(self): shlib_ext = get_shared_lib_ext() psubdir = self.det_psubdir() - openfoam_extend_v3 = 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0') + openfoam_extend_v3 = self.is_extend and self.looseversion >= LooseVersion('3.0') if openfoam_extend_v3 or self.looseversion < LooseVersion("2"): toolsdir = os.path.join(self.openfoamdir, "applications", "bin", psubdir) libsdir = os.path.join(self.openfoamdir, "lib", psubdir) @@ -422,26 +426,36 @@ def sanity_check_step(self): # some randomly selected binaries # if one of these is missing, it's very likely something went wrong + tools = ["boundaryFoam", "engineFoam", "buoyantSimpleFoam", "buoyantBoussinesqSimpleFoam", "sonicFoam"] + tools += ["surfaceAdd", "surfaceFind", "surfaceSmooth"] + tools += ["blockMesh", "checkMesh", "deformedGeom", "engineSwirl", "modifyMesh", "refineMesh"] + + # surfaceSmooth is replaced by surfaceLambdaMuSmooth is OpenFOAM v2.3.0 + if not self.is_extend and self.looseversion >= LooseVersion("2.3.0"): + tools.remove("surfaceSmooth") + tools.append("surfaceLambdaMuSmooth") + # sonicFoam and buoyantBoussineqSimpleFoam deprecated in version 7+ + if self.is_dot_org and self.looseversion >= LooseVersion('7'): + tools.remove("buoyantBoussinesqSimpleFoam") + tools.remove("sonicFoam") + # buoyantSimpleFoam replaced by buoyantFoam in versions 10+ + if self.is_dot_org and self.looseversion >= LooseVersion("10"): + tools.remove("buoyantSimpleFoam") + tools.append("buoyantFoam") + # engineFoam replaced by reactingFoam in versions 10+ + if self.is_dot_org and self.looseversion >= LooseVersion("10"): + tools.remove("engineFoam") + tools.append("reactingFoam") + bins = [os.path.join(self.openfoamdir, "bin", x) for x in ["paraFoam"]] + \ - [os.path.join(toolsdir, "buoyantSimpleFoam")] + \ - [os.path.join(toolsdir, "%sFoam" % x) for x in ["boundary", "engine"]] + \ - [os.path.join(toolsdir, "surface%s" % x) for x in ["Add", "Find", "Smooth"]] + \ - [os.path.join(toolsdir, x) for x in ['blockMesh', 'checkMesh', 'deformedGeom', 'engineSwirl', - 'modifyMesh', 'refineMesh']] + [os.path.join(toolsdir, x) for x in tools] # test setting up the OpenFOAM environment in bash shell load_openfoam_env = "source $FOAM_BASH" custom_commands = [load_openfoam_env] - # only include Boussinesq and sonic since for OpenFOAM < 7, since those solvers have been deprecated - if self.looseversion < LooseVersion('7'): - bins.extend([ - os.path.join(toolsdir, "buoyantBoussinesqSimpleFoam"), - os.path.join(toolsdir, "sonicFoam") - ]) - # check for the Pstream and scotchDecomp libraries, there must be a dummy one and an mpi one - if 'extend' in self.name.lower(): + if self.is_extend: libs = [os.path.join(libsdir, "libscotchDecomp.%s" % shlib_ext), os.path.join(libsdir, "libmetisDecomp.%s" % shlib_ext)] if self.looseversion < LooseVersion('3.2'): @@ -452,7 +466,7 @@ def sanity_check_step(self): libs.extend([os.path.join(libsdir, "libparMetisDecomp.%s" % shlib_ext)]) else: # OpenFOAM v2012 puts mpi into eb-mpi - if self.looseversion >= LooseVersion("2012"): + if self.is_dot_com and self.looseversion >= LooseVersion("2012"): mpilibssubdir = "eb-mpi" else: mpilibssubdir = "mpi" @@ -463,12 +477,7 @@ def sanity_check_step(self): [os.path.join(libsdir, "libscotchDecomp.%s" % shlib_ext)] + \ [os.path.join(libsdir, "dummy", "libscotchDecomp.%s" % shlib_ext)] - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion("2.3.0"): - # surfaceSmooth is replaced by surfaceLambdaMuSmooth is OpenFOAM v2.3.0 - bins.remove(os.path.join(toolsdir, "surfaceSmooth")) - bins.append(os.path.join(toolsdir, "surfaceLambdaMuSmooth")) - - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion("2.4.0"): + if not self.is_extend and self.looseversion >= LooseVersion("2.4.0"): # also check for foamMonitor for OpenFOAM versions other than OpenFOAM-Extend bins.append(os.path.join(self.openfoamdir, 'bin', 'foamMonitor')) @@ -484,7 +493,7 @@ def sanity_check_step(self): # run motorBike tutorial case to ensure the installation is functional (if it's available); # only for recent (>= v6.0) versions of openfoam.org variant - if self.looseversion >= LooseVersion('6') and self.looseversion < LooseVersion('100'): + if self.is_dot_org and self.looseversion >= LooseVersion('6'): openfoamdir_path = os.path.join(self.installdir, self.openfoamdir) motorbike_path = os.path.join(openfoamdir_path, 'tutorials', 'incompressible', 'simpleFoam', 'motorBike') if os.path.exists(motorbike_path): @@ -537,7 +546,7 @@ def make_module_extra(self, altroot=None, altversion=None): ] # OpenFOAM >= 3.0.0 can use 64 bit integers - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if not self.is_extend and self.looseversion >= LooseVersion('3.0'): if self.toolchain.options['i8']: env_vars += [('WM_LABEL_SIZE', '64')] else: diff --git a/easybuild/easyblocks/o/openifs.py b/easybuild/easyblocks/o/openifs.py index fc06601b0f..2fd8c5a137 100644 --- a/easybuild/easyblocks/o/openifs.py +++ b/easybuild/easyblocks/o/openifs.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/o/openmpi.py b/easybuild/easyblocks/o/openmpi.py index c756d16819..dc510f723f 100644 --- a/easybuild/easyblocks/o/openmpi.py +++ b/easybuild/easyblocks/o/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2022 Ghent University +# Copyright 2019-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -69,7 +69,7 @@ def config_opt_used(key, enable_opt=False): self.cfg.update('configopts', '--enable-%s' % key) # List of EasyBuild dependencies for which OMPI has known options - known_dependencies = ('CUDA', 'hwloc', 'libevent', 'libfabric', 'PMIx', 'UCX') + known_dependencies = ('CUDA', 'hwloc', 'libevent', 'libfabric', 'PMIx', 'UCX', 'UCC') # Value to use for `--with-=` if the dependency is not specified in the easyconfig # No entry is interpreted as no option added at all # This is to make builds reproducible even when the system libraries are changed and avoids failures diff --git a/easybuild/easyblocks/o/openssl.py b/easybuild/easyblocks/o/openssl.py index 2235e4d19c..8238772899 100644 --- a/easybuild/easyblocks/o/openssl.py +++ b/easybuild/easyblocks/o/openssl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 908313214c..f73a5e5295 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -1,5 +1,5 @@ ## -# Copyright 2021 Vrije Universiteit Brussel +# Copyright 2021-2023 Vrije Universiteit Brussel # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -162,7 +162,7 @@ def __init__(self, *args, **kwargs): for solib in solibs: system_solib = find_library_path(solib) if system_solib: - openssl_version = 0 + openssl_version = '0' # get version of system library filename try: openssl_version = full_version_regex.search(os.path.realpath(system_solib)).group(0) diff --git a/easybuild/easyblocks/o/orca.py b/easybuild/easyblocks/o/orca.py index 636544976e..e6066a28eb 100644 --- a/easybuild/easyblocks/o/orca.py +++ b/easybuild/easyblocks/o/orca.py @@ -1,5 +1,5 @@ ## -# Copyright 2021 Vrije Universiteit Brussel +# Copyright 2021-2023 Vrije Universiteit Brussel # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index 2b113a0578..181d660707 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/parmetis.py b/easybuild/easyblocks/p/parmetis.py index 8a73ff3d47..a9885b1c75 100644 --- a/easybuild/easyblocks/p/parmetis.py +++ b/easybuild/easyblocks/p/parmetis.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/pasha.py b/easybuild/easyblocks/p/pasha.py index 0f9e05a21b..7d1c87e81c 100644 --- a/easybuild/easyblocks/p/pasha.py +++ b/easybuild/easyblocks/p/pasha.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/pbdmpi.py b/easybuild/easyblocks/p/pbdmpi.py index b2f83d8146..73e23b0ae5 100644 --- a/easybuild/easyblocks/p/pbdmpi.py +++ b/easybuild/easyblocks/p/pbdmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/pbdslap.py b/easybuild/easyblocks/p/pbdslap.py index a15fc1a7fb..f30d131ff4 100644 --- a/easybuild/easyblocks/p/pbdslap.py +++ b/easybuild/easyblocks/p/pbdslap.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/pdt.py b/easybuild/easyblocks/p/pdt.py index bf9cef0021..253c2f5bbb 100644 --- a/easybuild/easyblocks/p/pdt.py +++ b/easybuild/easyblocks/p/pdt.py @@ -1,7 +1,7 @@ ## # This is an easyblock for EasyBuild, see https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2015-2022 Juelich Supercomputing Centre, Germany +# Copyright:: Copyright 2015-2023 Juelich Supercomputing Centre, Germany # Authors:: Bernd Mohr # Markus Geimer # License:: 3-clause BSD diff --git a/easybuild/easyblocks/p/perl.py b/easybuild/easyblocks/p/perl.py index cc2593c1e7..d96505ce7d 100644 --- a/easybuild/easyblocks/p/perl.py +++ b/easybuild/easyblocks/p/perl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index 0e42a3b60f..a335132c6b 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,7 @@ from easybuild.framework.easyconfig import BUILD, CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import symlink, apply_regex_substitutions -from easybuild.tools.modules import get_software_root +from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.py2vs3 import string_type @@ -210,6 +210,10 @@ def configure_step(self): # filter out deps handled seperately sep_deps = ['BLACS', 'BLAS', 'CMake', 'FFTW', 'LAPACK', 'numpy', 'mpi4py', 'papi', 'ScaLAPACK', 'SciPy-bundle', 'SuiteSparse'] + # SCOTCH has to be treated separately since they add weird postfixes + # to library names from SCOTCH 7.0.1 or PETSc version 3.17. + if (LooseVersion(self.version) >= LooseVersion("3.17")): + sep_deps.append('SCOTCH') depfilter = [d['name'] for d in self.cfg.builddependencies()] + sep_deps deps = [dep['name'] for dep in self.cfg.dependencies() if not dep['name'] in depfilter] @@ -224,6 +228,24 @@ def configure_step(self): withdep = "--with-%s" % dep[1].lower() self.cfg.update('configopts', '%s=1 %s-dir=%s' % (withdep, withdep, deproot)) + # SCOTCH has to be treated separately since they add weird postfixes + # to library names from SCOTCH 7.0.1 or PETSc version 3.17. + scotch = get_software_root('SCOTCH') + scotch_ver = get_software_version('SCOTCH') + if (scotch and LooseVersion(scotch_ver) >= LooseVersion("7.0")): + withdep = "--with-ptscotch" + scotch_inc = [os.path.join(scotch, "include")] + inc_spec = "-include=[%s]" % ','.join(scotch_inc) + + # For some reason there is a v3 suffix added to libptscotchparmetis + # which is the reason for this new code. + req_scotch_libs = ['libesmumps.a', 'libptesmumps.a', 'libptscotch.a', + 'libptscotcherr.a', 'libptscotchparmetisv3.a', 'libscotch.a', + 'libscotcherr.a'] + scotch_libs = [os.path.join(scotch, "lib", x) for x in req_scotch_libs] + lib_spec = "-lib=[%s]" % ','.join(scotch_libs) + self.cfg.update('configopts', ' '.join([withdep + spec for spec in ['=1', inc_spec, lib_spec]])) + # SuiteSparse options changed in PETSc 3.5, suitesparse = get_software_root('SuiteSparse') if suitesparse: @@ -231,13 +253,19 @@ def configure_step(self): withdep = "--with-suitesparse" # specified order of libs matters! ss_libs = ["UMFPACK", "KLU", "CHOLMOD", "BTF", "CCOLAMD", "COLAMD", "CAMD", "AMD"] + # More libraries added after version 3.17 + if LooseVersion(self.version) >= LooseVersion("3.17"): + # specified order of libs matters! + ss_libs = ["UMFPACK", "KLU", "SPQR", "CHOLMOD", "BTF", "CCOLAMD", + "COLAMD", "CSparse", "CXSparse", "LDL", "RBio", + "SLIP_LU", "CAMD", "AMD"] suitesparse_inc = [os.path.join(suitesparse, x, "Include") for x in ss_libs] suitesparse_inc.append(os.path.join(suitesparse, "SuiteSparse_config")) inc_spec = "-include=[%s]" % ','.join(suitesparse_inc) - suitesparse_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.lower()) + suitesparse_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.replace("_", "").lower()) for x in ss_libs] suitesparse_libs.append(os.path.join(suitesparse, "SuiteSparse_config", "libsuitesparseconfig.a")) lib_spec = "-lib=[%s]" % ','.join(suitesparse_libs) diff --git a/easybuild/easyblocks/p/pgi.py b/easybuild/easyblocks/p/pgi.py index 2b033afd12..8d34ae7b06 100644 --- a/easybuild/easyblocks/p/pgi.py +++ b/easybuild/easyblocks/p/pgi.py @@ -1,6 +1,6 @@ ## -# Copyright 2015-2022 Bart Oldeman -# Copyright 2016-2022 Forschungszentrum Juelich +# Copyright 2015-2023 Bart Oldeman +# Copyright 2016-2023 Forschungszentrum Juelich # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/easyblocks/p/picard.py b/easybuild/easyblocks/p/picard.py index 957b7e9cc4..b9f2d1a9b1 100644 --- a/easybuild/easyblocks/p/picard.py +++ b/easybuild/easyblocks/p/picard.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/pplacer.py b/easybuild/easyblocks/p/pplacer.py index feb7d090bc..8e85666dd6 100644 --- a/easybuild/easyblocks/p/pplacer.py +++ b/easybuild/easyblocks/p/pplacer.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2022 Ghent University +# Copyright 2016-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/primer3.py b/easybuild/easyblocks/p/primer3.py index 64acf988bd..31995b2550 100644 --- a/easybuild/easyblocks/p/primer3.py +++ b/easybuild/easyblocks/p/primer3.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/psi.py b/easybuild/easyblocks/p/psi.py index 20c1344f8f..af400c3cec 100644 --- a/easybuild/easyblocks/p/psi.py +++ b/easybuild/easyblocks/p/psi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/psmpi.py b/easybuild/easyblocks/p/psmpi.py index 2ed83d18e8..755cc1f809 100644 --- a/easybuild/easyblocks/p/psmpi.py +++ b/easybuild/easyblocks/p/psmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2022 Ghent University, Forschungszentrum Juelich +# Copyright 2016-2023 Ghent University, Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/pybind11.py b/easybuild/easyblocks/p/pybind11.py index 799a604bba..d964ffb9ac 100644 --- a/easybuild/easyblocks/p/pybind11.py +++ b/easybuild/easyblocks/p/pybind11.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -70,8 +70,9 @@ def configure_step(self): def test_step(self): """Run pybind11 tests""" - # always run tests - self.cfg['runtest'] = 'check' + # run tests unless explicitly disabled + if self.cfg['runtest'] is not False: + self.cfg['runtest'] = 'check' super(EB_pybind11, self).test_step() def install_step(self): diff --git a/easybuild/easyblocks/p/pyquante.py b/easybuild/easyblocks/p/pyquante.py index e2476fb17f..8bfaf5be3f 100644 --- a/easybuild/easyblocks/p/pyquante.py +++ b/easybuild/easyblocks/p/pyquante.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 536aa07aca..975935cdfb 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -444,10 +444,13 @@ def install_step(self): super(EB_Python, self).install_step() - # Create non-versioned, relative symlinks for python and pip + # Create non-versioned, relative symlinks for python, python-config and pip python_binary_path = os.path.join(self.installdir, 'bin', 'python') if not os.path.isfile(python_binary_path): symlink('python' + self.pyshortver, python_binary_path, use_abspath_source=False) + python_config_binary_path = os.path.join(self.installdir, 'bin', 'python-config') + if not os.path.isfile(python_config_binary_path): + symlink('python' + self.pyshortver + '-config', python_config_binary_path, use_abspath_source=False) if self.install_pip: pip_binary_path = os.path.join(self.installdir, 'bin', 'pip') if not os.path.isfile(pip_binary_path): @@ -534,7 +537,13 @@ def sanity_check_step(self): pyver = 'python' + self.pyshortver custom_paths = { - 'files': [os.path.join('bin', pyver), os.path.join('lib', 'lib' + pyver + abiflags + '.' + shlib_ext)], + 'files': [ + os.path.join('bin', pyver), + os.path.join('bin', 'python'), + os.path.join('bin', pyver + '-config'), + os.path.join('bin', 'python-config'), + os.path.join('lib', 'lib' + pyver + abiflags + '.' + shlib_ext), + ], 'dirs': [os.path.join('include', pyver + abiflags), os.path.join('lib', pyver, 'lib-dynload')], } @@ -543,6 +552,7 @@ def sanity_check_step(self): custom_commands = [ "python --version", + "python-config --help", # make sure that symlink was created correctly "python -c 'import _ctypes'", # make sure that foreign function interface (libffi) works "python -c 'import _ssl'", # make sure SSL support is enabled one way or another "python -c 'import readline'", # make sure readline support was built correctly diff --git a/easybuild/easyblocks/p/python_meep.py b/easybuild/easyblocks/p/python_meep.py index 94877e561c..106d31392a 100644 --- a/easybuild/easyblocks/p/python_meep.py +++ b/easybuild/easyblocks/p/python_meep.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 36aa710cb2..2783db4707 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2022 Ghent University +# Copyright 2020-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,14 +37,13 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option -from easybuild.tools.filetools import symlink, apply_regex_substitutions +from easybuild.tools.filetools import apply_regex_substitutions, mkdir, symlink from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.systemtools import POWER, get_cpu_architecture -from easybuild.tools.utilities import nub class EB_PyTorch(PythonPackage): - """Support for building/installing TensorFlow.""" + """Support for building/installing PyTorch.""" @staticmethod def extra_options(): @@ -156,6 +155,9 @@ def configure_step(self): if get_software_root('imkl'): options.append('BLAS=MKL') options.append('INTEL_MKL_DIR=$MKLROOT') + elif pytorch_version >= '1.11.0' and get_software_root('FlexiBLAS'): + options.append('BLAS=FlexiBLAS') + options.append('WITH_BLAS=flexi') elif pytorch_version >= '1.9.0' and get_software_root('BLIS'): options.append('BLAS=BLIS') options.append('BLIS_HOME=' + get_software_root('BLIS')) @@ -240,10 +242,16 @@ def configure_step(self): self.cfg.update('prebuildopts', ' '.join(unique_options) + ' ') self.cfg.update('preinstallopts', ' '.join(unique_options) + ' ') + def _set_cache_dir(self): + """Set $XDG_CACHE_HOME to avoid PyTorch defaulting to $HOME""" + cache_dir = os.path.join(self.tmpdir, '.cache') + # The path must exist! + mkdir(cache_dir, parents=True) + env.setvar('XDG_CACHE_HOME', cache_dir) + def test_step(self): """Run unit tests""" - # Make PyTorch tests not use the user home - env.setvar('XDG_CACHE_HOME', os.path.join(self.tmpdir, '.cache')) + self._set_cache_dir() # Pretend to be on FB CI which disables some tests, especially those which download stuff env.setvar('SANDCASTLE', '1') # Skip this test(s) which is very flaky @@ -261,22 +269,104 @@ def test_step(self): 'excluded_tests': ' '.join(excluded_tests) }) - (tests_out, tests_ec) = super(EB_PyTorch, self).test_step(return_output_ec=True) - - ran_tests_hits = re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M) - test_cnt = sum(int(hit) for hit in ran_tests_hits) - - failed_tests = nub(re.findall(r"^(?P.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M)) - failed_test_cnt = len(failed_tests) - - if failed_test_cnt: + test_result = super(EB_PyTorch, self).test_step(return_output_ec=True) + if test_result is None: + if self.cfg['runtest'] is False: + msg = "Do not set 'runtest' to False, use --skip-test-step instead." + else: + msg = "Tests did not run. Make sure 'runtest' is set to a command." + raise EasyBuildError(msg) + + tests_out, tests_ec = test_result + + def get_count_for_pattern(regex, text): + """Match the regexp containing a single group and return the integer value of the matched group. + Return zero if no or more than 1 match was found and warn for the latter case + """ + match = re.findall(regex, text) + if len(match) == 1: + return int(match[0]) + elif len(match) > 1: + # Shouldn't happen, but means something went wrong with the regular expressions. + # Throw warning, as the build might be fine, no need to error on this. + warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n" + warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable." + print_warning(warn_msg) + return 0 + + # Create clear summary report + failure_report = "" + failure_cnt = 0 + error_cnt = 0 + failed_test_suites = [] + + # Grep for patterns like: + # Ran 219 tests in 67.325s + # + # FAILED (errors=10, skipped=190, expected failures=6) + # test_fx failed! + regex = (r"^Ran (?P[0-9]+) tests.*$\n\n" + r"FAILED \((?P.*)\)$\n" + r"(?:^(?:(?!failed!).)*$\n)*" + r"(?P.*) failed!(?: Received signal: \w+)?\s*$") + + for m in re.finditer(regex, tests_out, re.M): + # E.g. 'failures=3, errors=10, skipped=190, expected failures=6' + failure_summary = m.group('failure_summary') + total, test_suite = m.group('test_cnt', 'failed_test_suite_name') + failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format( + test_suite=test_suite, total=total, failure_summary=failure_summary + ) + failure_cnt += get_count_for_pattern(r"(?.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$" + + for m in re.finditer(regex, tests_out, re.M): + # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings' + failure_summary = m.group('failure_summary') + test_suite = m.group('failed_test_suite_name') + failure_report += "{test_suite} ({failure_summary})\n".format( + test_suite=test_suite, failure_summary=failure_summary + ) + failure_cnt += get_count_for_pattern(r"([0-9]+) failed", failure_summary) + error_cnt += get_count_for_pattern(r"([0-9]+) error", failure_summary) + failed_test_suites.append(test_suite) + + # Make the names unique and sorted + failed_test_suites = sorted(set(failed_test_suites)) + # Gather all failed tests suites in case we missed any (e.g. when it exited due to syntax errors) + # Also unique and sorted to be able to compare the lists below + all_failed_test_suites = sorted(set( + re.findall(r"^(?P.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M) + )) + # If we missed any test suites prepend a list of all failed test suites + if failed_test_suites != all_failed_test_suites: + failure_report_save = failure_report + failure_report = 'Failed tests (suites/files):\n' + failure_report += '\n'.join('* %s' % t for t in all_failed_test_suites) + if failure_report_save: + failure_report += '\n' + failure_report_save + + # Calculate total number of unsuccesful and total tests + failed_test_cnt = failure_cnt + error_cnt + test_cnt = sum(int(hit) for hit in re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M)) + + if failed_test_cnt > 0: max_failed_tests = self.cfg['max_failed_tests'] - test_or_tests = 'tests' if failed_test_cnt > 1 else 'test' - msg = "%d %s (out of %d) failed:\n" % (failed_test_cnt, test_or_tests, test_cnt) - msg += '\n'.join('* %s' % t for t in sorted(failed_tests)) + failure_or_failures = 'failure' if failure_cnt == 1 else 'failures' + error_or_errors = 'error' if error_cnt == 1 else 'errors' + msg = "%d test %s, %d test %s (out of %d):\n" % ( + failure_cnt, failure_or_failures, error_cnt, error_or_errors, test_cnt + ) + msg += failure_report - if max_failed_tests == 0: + # If no tests are supposed to fail or some failed for which we were not able to count errors fail now + if max_failed_tests == 0 or failed_test_suites != all_failed_test_suites: raise EasyBuildError(msg) else: msg += '\n\n' + ' '.join([ @@ -287,17 +377,22 @@ def test_step(self): "are known to be flaky, or do not affect your intended usage of PyTorch.", "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).", ]) + # Print to console, the user should really be aware that we are accepting failing tests here... print_warning(msg) + # Also log this warning in the file log + self.log.warning(msg) + if failed_test_cnt > max_failed_tests: raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", failed_test_cnt, max_failed_tests) + elif failure_report: + raise EasyBuildError("Test command had non-zero exit code (%s)!\n%s", tests_ec, failure_report) elif tests_ec: raise EasyBuildError("Test command had non-zero exit code (%s), but no failed tests found?!", tests_ec) def test_cases_step(self): - # Make PyTorch tests not use the user home - env.setvar('XDG_CACHE_HOME', os.path.join(self.tmpdir, '.cache')) + self._set_cache_dir() super(EB_PyTorch, self).test_cases_step() def sanity_check_step(self, *args, **kwargs): diff --git a/easybuild/easyblocks/p/pyzmq.py b/easybuild/easyblocks/p/pyzmq.py index 2d719efef8..8007d936bb 100644 --- a/easybuild/easyblocks/p/pyzmq.py +++ b/easybuild/easyblocks/p/pyzmq.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/q/qscintilla.py b/easybuild/easyblocks/q/qscintilla.py index 3701fee992..3a64d6c607 100644 --- a/easybuild/easyblocks/q/qscintilla.py +++ b/easybuild/easyblocks/q/qscintilla.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -118,13 +118,23 @@ def install_step(self): pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) sip_incdir = find_glob_pattern(os.path.join(self.pyqt_root, 'include', 'python%s*' % pyshortver), False) - # in case PyQt5's sip was installed in directories that are specific to each version of python - # as could happen with multi_deps - pyqt_sipdir = find_glob_pattern(os.path.join(self.pyqt_root, 'share', 'python%s*' % pyshortver, - 'site-packages', 'sip', self.pyqt_pkg_name), False) - # fall back to a single sipdir + # depending on PyQt5 versions and how it was installed, the sip directory could be in various places + # test them and figure out the first one that matches + pyqt_sip_subdir = [os.path.join('share', 'python%s*' % pyshortver, 'site-packages', 'sip', + self.pyqt_pkg_name), + os.path.join('share', 'sip', self.pyqt_pkg_name), + os.path.join('share', 'sip'), + os.path.join('lib', 'python%s*' % pyshortver, 'site-packages', self.pyqt_pkg_name, + 'bindings') + ] + pyqt_sipdir_options = [os.path.join(self.pyqt_root, subdir) for subdir in pyqt_sip_subdir] + for pyqt_sipdir_option in pyqt_sipdir_options: + pyqt_sipdir = find_glob_pattern(pyqt_sipdir_option, False) + if pyqt_sipdir: + break + if not pyqt_sipdir: - pyqt_sipdir = os.path.join(self.pyqt_root, 'share', 'sip', self.pyqt_pkg_name) + raise EasyBuildError("Failed to find PyQt5 sip directory") cfgopts = [ '--destdir %s' % os.path.join(self.installdir, pylibdir), diff --git a/easybuild/easyblocks/q/qt.py b/easybuild/easyblocks/q/qt.py index 034065f475..cc23f27a99 100644 --- a/easybuild/easyblocks/q/qt.py +++ b/easybuild/easyblocks/q/qt.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/q/quantumespresso.py b/easybuild/easyblocks/q/quantumespresso.py index 89c94b0058..b5a232817b 100644 --- a/easybuild/easyblocks/q/quantumespresso.py +++ b/easybuild/easyblocks/q/quantumespresso.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/__init__.py b/easybuild/easyblocks/r/__init__.py index 119b78d134..d1e94b14c8 100644 --- a/easybuild/easyblocks/r/__init__.py +++ b/easybuild/easyblocks/r/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/r.py b/easybuild/easyblocks/r/r.py index 0e93e9c7a0..5f73952873 100644 --- a/easybuild/easyblocks/r/r.py +++ b/easybuild/easyblocks/r/r.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/repeatmasker.py b/easybuild/easyblocks/r/repeatmasker.py index 5634b12ce1..0a4d10329b 100644 --- a/easybuild/easyblocks/r/repeatmasker.py +++ b/easybuild/easyblocks/r/repeatmasker.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/repeatmodeler.py b/easybuild/easyblocks/r/repeatmodeler.py index 5edce4d0c3..122f64cc4c 100644 --- a/easybuild/easyblocks/r/repeatmodeler.py +++ b/easybuild/easyblocks/r/repeatmodeler.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/reticulate.py b/easybuild/easyblocks/r/reticulate.py index a6850c0552..7d5a7a79d5 100644 --- a/easybuild/easyblocks/r/reticulate.py +++ b/easybuild/easyblocks/r/reticulate.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/rmpi.py b/easybuild/easyblocks/r/rmpi.py index dc58e9d813..c9a1528acd 100644 --- a/easybuild/easyblocks/r/rmpi.py +++ b/easybuild/easyblocks/r/rmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/root.py b/easybuild/easyblocks/r/root.py index 5e80a1ec9c..872671eb4b 100644 --- a/easybuild/easyblocks/r/root.py +++ b/easybuild/easyblocks/r/root.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/rosetta.py b/easybuild/easyblocks/r/rosetta.py index d968d76354..71630c7902 100644 --- a/easybuild/easyblocks/r/rosetta.py +++ b/easybuild/easyblocks/r/rosetta.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/rserve.py b/easybuild/easyblocks/r/rserve.py index 444101f207..e06b7007b1 100644 --- a/easybuild/easyblocks/r/rserve.py +++ b/easybuild/easyblocks/r/rserve.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/ruby.py b/easybuild/easyblocks/r/ruby.py index d811cc2de8..80b66064f3 100644 --- a/easybuild/easyblocks/r/ruby.py +++ b/easybuild/easyblocks/r/ruby.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/r/rust.py b/easybuild/easyblocks/r/rust.py new file mode 100644 index 0000000000..ee66f6854e --- /dev/null +++ b/easybuild/easyblocks/r/rust.py @@ -0,0 +1,124 @@ +## +# Copyright 2023-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing Rust, implemented as an easyblock + +@author: Kenneth Hoste (Ghent University) +""" +import glob +import os +import re + +from easybuild.easyblocks.generic.configuremake import ConfigureMake + +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext + + +class EB_Rust(ConfigureMake): + """Support for building/installing Rust.""" + + def __init__(self, *args, **kwargs): + """Custom easyblock constructor for Rust.""" + super(EB_Rust, self).__init__(*args, **kwargs) + + # see https://rustc-dev-guide.rust-lang.org/building/how-to-build-and-run.html#what-is-xpy + # note: ConfigureMake.build_step automatically adds '-j ' + self.cfg['build_cmd'] = "./x.py build" + self.cfg['install_cmd'] = "./x.py install -j %(parallel)s" + + def configure_step(self): + """Custom configure step for Rust""" + + # perform extended build, which includes cargo, rustfmt, Rust Language Server (RLS), etc. + self.cfg.update('configopts', "--enable-extended") + + self.cfg.update('configopts', "--sysconfdir=%s" % os.path.join(self.installdir, 'etc')) + + # don't use Ninja if it is not listed as a build dependency; + # may be because Ninja requires Python, and Rust is a build dependency for cryptography + # which may be included as an extension with Python + build_dep_names = set(dep['name'] for dep in self.cfg.dependencies(build_only=True)) + if 'Ninja' not in build_dep_names: + self.cfg.update('configopts', "--set=llvm.ninja=false") + + super(EB_Rust, self).configure_step() + + # avoid failure when home directory is an NFS mount, + # see https://github.com/rust-lang/cargo/issues/6652 + cargo_home = "export CARGO_HOME=%s && " % os.path.join(self.builddir, 'cargo') + self.cfg.update('prebuildopts', cargo_home) + self.cfg.update('preinstallopts', cargo_home) + + def install_step(self): + """Custom install step for Rust""" + + super(EB_Rust, self).install_step() + + if build_option('rpath'): + # make sure that all shared libraries use RPATH, not RUNPATH; + # cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/18079 + shlib_ext = get_shared_lib_ext() + shared_libs = glob.glob(os.path.join(self.installdir, 'lib', 'lib*.%s' % shlib_ext)) + + runpath_regex = re.compile(r"\(RUNPATH\)\s+Library runpath") + + for shared_lib in shared_libs: + out, ec = run_cmd("readelf -d %s" % shared_lib, simple=False, trace=False) + if ec: + raise EasyBuildError("Failed to check RPATH section in %s: %s", shared_lib, out) + elif runpath_regex.search(out): + self.log.info("RUNPATH section found in %s - need to change to RPATH", shared_lib) + + # first determine current RUNPATH value + out, ec = run_cmd("patchelf --print-rpath %s" % shared_lib) + if ec: + raise EasyBuildError("Failed to determine current RUNPATH value for %s: %s", shared_lib, out) + else: + runpath = out.strip() + # use RUNPATH value to RPATH value + out, ec = run_cmd("patchelf --set-rpath '%s' --force-rpath %s" % (runpath, shared_lib)) + if ec: + raise EasyBuildError("Failed to set RPATH for %s: %s", shared_lib, out) + else: + self.log.info("RPATH set for %s", shared_lib) + else: + self.log.info("No RUNPATH section found in %s", shared_lib) + + def sanity_check_step(self): + """Custom sanity check for Rust""" + + custom_paths = { + 'files': ['bin/cargo', 'bin/rustc', 'bin/rustdoc'], + 'dirs': ['lib/rustlib', 'share/doc', 'share/man'], + } + + custom_commands = [ + "cargo --version", + "rustc --version", + ] + return super(EB_Rust, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/s/samcef.py b/easybuild/easyblocks/s/samcef.py index 262e6182a2..2f5e7b69d4 100644 --- a/easybuild/easyblocks/s/samcef.py +++ b/easybuild/easyblocks/s/samcef.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/samtools.py b/easybuild/easyblocks/s/samtools.py index f0d88a88e7..6d297c2283 100644 --- a/easybuild/easyblocks/s/samtools.py +++ b/easybuild/easyblocks/s/samtools.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/s/sas.py b/easybuild/easyblocks/s/sas.py index 6321d965d7..31e6c20b50 100644 --- a/easybuild/easyblocks/s/sas.py +++ b/easybuild/easyblocks/s/sas.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/scalapack.py b/easybuild/easyblocks/s/scalapack.py index 7c71c31d4a..466fef57c3 100644 --- a/easybuild/easyblocks/s/scalapack.py +++ b/easybuild/easyblocks/s/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/scalasca1.py b/easybuild/easyblocks/s/scalasca1.py index d91b17f5fc..a78398cc9d 100644 --- a/easybuild/easyblocks/s/scalasca1.py +++ b/easybuild/easyblocks/s/scalasca1.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/scipion.py b/easybuild/easyblocks/s/scipion.py index 0d5bd9e348..86d7460a33 100644 --- a/easybuild/easyblocks/s/scipion.py +++ b/easybuild/easyblocks/s/scipion.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2022 Ghent University +# Copyright 2019-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 9f32eb8466..823116e1e8 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,27 +30,111 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) +@author: Jasper Grimm (University of York) """ +import os +import tempfile from distutils.version import LooseVersion -from easybuild.easyblocks.generic.fortranpythonpackage import FortranPythonPackage -from easybuild.easyblocks.generic.pythonpackage import det_pylibdir +import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain +from easybuild.easyblocks.generic.fortranpythonpackage import FortranPythonPackage +from easybuild.easyblocks.generic.mesonninja import MesonNinja +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.filetools import change_dir, copy_dir, copy_file -class EB_scipy(FortranPythonPackage): +class EB_scipy(FortranPythonPackage, PythonPackage, MesonNinja): """Support for installing the scipy Python package as part of a Python installation.""" + @staticmethod + def extra_options(): + """Easyconfig parameters specific to scipy.""" + extra_vars = ({ + 'enable_slow_tests': [False, "Run scipy test suite, including tests marked as slow", CUSTOM], + 'ignore_test_result': [None, "Run scipy test suite, but ignore test failures (True/False/None). Default " + "(None) implies True for scipy < 1.9, and False for scipy >= 1.9", CUSTOM], + }) + + return PythonPackage.extra_options(extra_vars=extra_vars) + def __init__(self, *args, **kwargs): """Set scipy-specific test command.""" - super(EB_scipy, self).__init__(*args, **kwargs) - + # calling PythonPackage __init__ also lets MesonNinja work in an extension + PythonPackage.__init__(self, *args, **kwargs) self.testinstall = True - self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'" + + if LooseVersion(self.version) >= LooseVersion('1.9'): + self.use_meson = True + + # enforce scipy test suite results if not explicitly disabled for scipy >= 1.9 + # strip inherited PythonPackage installopts + installopts = self.cfg['installopts'] + pythonpackage_installopts = ['--no-deps', '--ignore-installed', '--no-index', '--egg', + '--zip-ok', '--no-index'] + self.log.info("Stripping inherited PythonPackage installopts %s from installopts %s", + pythonpackage_installopts, installopts) + for i in pythonpackage_installopts: + installopts = installopts.replace(i, '') + self.cfg['installopts'] = installopts + + else: + self.use_meson = False + + if self.cfg['ignore_test_result'] is None: + # automatically ignore scipy test suite results for scipy < 1.9, as we did in older easyconfigs + self.cfg['ignore_test_result'] = LooseVersion(self.version) < '1.9' + self.log.info("ignore_test_result not specified, so automatically set to %s for scipy %s", + self.cfg['ignore_test_result'], self.version) + + if self.cfg['ignore_test_result']: + # used to maintain compatibility with easyconfigs predating scipy 1.9; + # runs tests (serially) in a way that exits with code 0 regardless of test results, + # see https://github.com/easybuilders/easybuild-easyblocks/issues/2237 + self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'" + else: + self.testcmd = " && ".join([ + "cd ..", + "touch %(srcdir)s/.coveragerc", + "%(python)s %(srcdir)s/runtests.py -v --no-build --parallel %(parallel)s", + ]) + if self.cfg['enable_slow_tests']: + self.testcmd += " -m full " def configure_step(self): """Custom configure step for scipy: set extra installation options when needed.""" - super(EB_scipy, self).configure_step() + + # scipy >= 1.9.0 uses Meson/Ninja + if self.use_meson: + # configure BLAS/LAPACK library to use with Meson for scipy >= 1.9.0 + lapack_lib = self.toolchain.lapack_family() + if lapack_lib == toolchain.FLEXIBLAS: + blas_lapack = 'flexiblas' + elif lapack_lib == toolchain.INTELMKL: + blas_lapack = 'mkl' + elif lapack_lib == toolchain.OPENBLAS: + blas_lapack = 'openblas' + else: + raise EasyBuildError("Unknown BLAS/LAPACK library used: %s", lapack_lib) + + for opt in ('blas', 'lapack'): + self.cfg.update('configopts', '-D%(opt)s=%(blas_lapack)s' % {'opt': opt, 'blas_lapack': blas_lapack}) + + # need to have already installed extensions in PATH, PYTHONPATH for configure/build/install steps + pythonpath = os.getenv('PYTHONPATH') + pylibdir = det_pylibdir() + env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, pylibdir), pythonpath])) + + path = os.getenv('PATH') + env.setvar('PATH', os.pathsep.join([os.path.join(self.installdir, 'bin'), path])) + + MesonNinja.configure_step(self) + + else: + # scipy < 1.9.0 uses install procedure using setup.py + FortranPythonPackage.configure_step(self) if LooseVersion(self.version) >= LooseVersion('0.13'): # in recent scipy versions, additional compilation is done in the install step, @@ -58,6 +142,77 @@ def configure_step(self): if self.toolchain.comp_family() in [toolchain.GCC, toolchain.CLANGGCC]: # @UndefinedVariable self.cfg.update('preinstallopts', "unset LDFLAGS && ") + def build_step(self): + """Custom build step for scipy: use ninja for scipy >= 1.9.0""" + if self.use_meson: + MesonNinja.build_step(self) + else: + FortranPythonPackage.build_step(self) + + def test_step(self): + """Run available scipy unit tests. Adapted from numpy easyblock""" + + if self.use_meson: + # temporarily install scipy so we can run the test suite + tmpdir = tempfile.mkdtemp() + cwd = os.getcwd() + + tmp_builddir = os.path.join(tmpdir, 'build') + tmp_installdir = os.path.join(tmpdir, 'install') + + # create a copy of the builddir + copy_dir(cwd, tmp_builddir) + change_dir(tmp_builddir) + + # reconfigure (to update prefix), and install to tmpdir + MesonNinja.configure_step(self, cmd_prefix=tmp_installdir) + MesonNinja.install_step(self) + + tmp_pylibdir = [os.path.join(tmp_installdir, det_pylibdir())] + self.prepare_python() + + self.cfg['pretestopts'] = " && ".join([ + # LDFLAGS should not be set when testing numpy/scipy, because it overwrites whatever numpy/scipy sets + # see http://projects.scipy.org/numpy/ticket/182 + "unset LDFLAGS", + "export PYTHONPATH=%s:$PYTHONPATH" % tmp_pylibdir, + "", + ]) + self.cfg['runtest'] = self.testcmd % { + 'python': self.python_cmd, + 'srcdir': self.cfg['start_dir'], + 'parallel': self.cfg['parallel'], + } + + MesonNinja.test_step(self) + + else: + self.testcmd = self.testcmd % { + 'python': '%(python)s', + 'srcdir': self.cfg['start_dir'], + 'parallel': self.cfg['parallel'], + } + FortranPythonPackage.test_step(self) + + def install_step(self): + """Custom install step for scipy: use ninja for scipy >= 1.9.0""" + if self.use_meson: + MesonNinja.install_step(self) + + # copy PKG-INFO file included in scipy source tarball to scipy-.egg-info in installation, + # so pip is aware of the scipy installation (required for 'pip list', 'pip check', etc.); + # see also https://github.com/easybuilders/easybuild-easyblocks/issues/2901 + pkg_info = os.path.join(self.cfg['start_dir'], 'PKG-INFO') + target_egg_info = os.path.join(self.installdir, self.pylibdir, 'scipy-%s.egg-info' % self.version) + if os.path.isfile(pkg_info): + copy_file(pkg_info, target_egg_info) + else: + cwd = os.getcwd() + print_warning("%s not found in %s, so can't use it to create %s!", pkg_info, cwd, target_egg_info, + log=self.log) + else: + FortranPythonPackage.install_step(self) + def sanity_check_step(self, *args, **kwargs): """Custom sanity check for scipy.""" @@ -68,4 +223,9 @@ def sanity_check_step(self, *args, **kwargs): 'dirs': [det_pylibdir()], } - return super(EB_scipy, self).sanity_check_step(custom_paths=custom_paths) + # make sure that scipy is included in output of 'pip list', + # so that 'pip check' passes if scipy is a required dependency for another Python package; + # use case-insensitive match, since name is sometimes reported as 'SciPy' + custom_commands = [r"pip list | grep -iE '^scipy\s+%s\s*$'" % self.version.replace('.', r'\.')] + + return PythonPackage.sanity_check_step(self, custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/s/score_p.py b/easybuild/easyblocks/s/score_p.py index d8cdb386a4..410fea4cb7 100644 --- a/easybuild/easyblocks/s/score_p.py +++ b/easybuild/easyblocks/s/score_p.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/scotch.py b/easybuild/easyblocks/s/scotch.py index 5251848124..89c728f7c9 100644 --- a/easybuild/easyblocks/s/scotch.py +++ b/easybuild/easyblocks/s/scotch.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/sepp.py b/easybuild/easyblocks/s/sepp.py index 78141f5efe..389f591219 100644 --- a/easybuild/easyblocks/s/sepp.py +++ b/easybuild/easyblocks/s/sepp.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/shrimp.py b/easybuild/easyblocks/s/shrimp.py index 4e873ad1af..c4a09fab69 100644 --- a/easybuild/easyblocks/s/shrimp.py +++ b/easybuild/easyblocks/s/shrimp.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/siesta.py b/easybuild/easyblocks/s/siesta.py index a5f250c5ce..81b132236e 100644 --- a/easybuild/easyblocks/s/siesta.py +++ b/easybuild/easyblocks/s/siesta.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/slepc.py b/easybuild/easyblocks/s/slepc.py index 395da82436..0254f195ad 100644 --- a/easybuild/easyblocks/s/slepc.py +++ b/easybuild/easyblocks/s/slepc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/snphylo.py b/easybuild/easyblocks/s/snphylo.py index 26834d2de1..88eb26f7ae 100644 --- a/easybuild/easyblocks/s/snphylo.py +++ b/easybuild/easyblocks/s/snphylo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/soapdenovo.py b/easybuild/easyblocks/s/soapdenovo.py index 0848bb7d0e..f404470ce8 100644 --- a/easybuild/easyblocks/s/soapdenovo.py +++ b/easybuild/easyblocks/s/soapdenovo.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/s/star_ccm.py b/easybuild/easyblocks/s/star_ccm.py index c3b6658585..7f981e9b36 100644 --- a/easybuild/easyblocks/s/star_ccm.py +++ b/easybuild/easyblocks/s/star_ccm.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2019 Ghent University +# Copyright 2018-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/stata.py b/easybuild/easyblocks/s/stata.py index 1cc4703d30..6fd83d850a 100644 --- a/easybuild/easyblocks/s/stata.py +++ b/easybuild/easyblocks/s/stata.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 2cff1575ce..f25fc66785 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/superlu.py b/easybuild/easyblocks/s/superlu.py index 40042409ac..eafa3c1e3e 100644 --- a/easybuild/easyblocks/s/superlu.py +++ b/easybuild/easyblocks/s/superlu.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University, University of Luxembourg +# Copyright 2009-2023 Ghent University, University of Luxembourg # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/s/swig.py b/easybuild/easyblocks/s/swig.py index 2322661b60..d434e9f34f 100644 --- a/easybuild/easyblocks/s/swig.py +++ b/easybuild/easyblocks/s/swig.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/t/tau.py b/easybuild/easyblocks/t/tau.py index 5e4a89cf6c..33ca3ea127 100644 --- a/easybuild/easyblocks/t/tau.py +++ b/easybuild/easyblocks/t/tau.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/t/tbb.py b/easybuild/easyblocks/t/tbb.py index f5bb348226..1ce9ee6750 100644 --- a/easybuild/easyblocks/t/tbb.py +++ b/easybuild/easyblocks/t/tbb.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 897000245f..d63a4bc3f6 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -1,5 +1,5 @@ ## -# Copyright 2017-2022 Ghent University +# Copyright 2017-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,14 +35,13 @@ import re import stat import tempfile -from distutils.version import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_python_version from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools import run +from easybuild.tools import run, LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option, IGNORE from easybuild.tools.filetools import adjust_permissions, apply_regex_substitutions, copy_file, mkdir, resolve_path @@ -211,6 +210,14 @@ def is_version_ok(version_range): return result +def get_bazel_version(): + """Get the Bazel version as a LooseVersion. Error if not found""" + version = get_software_version('Bazel') + if version is None: + raise EasyBuildError('Failed to determine Bazel version - is it listed as a (build) dependency?') + return LooseVersion(version) + + class EB_TensorFlow(PythonPackage): """Support for building/installing TensorFlow.""" @@ -401,6 +408,18 @@ def get_system_libs(self): else: ignored_system_deps.append('%s (Python package %s)' % (tf_name, pkg_name)) + # If we use OpenSSL (potentially as a wrapper) somewhere in the chain we must tell TF to use it too + openssl_root = get_software_root('OpenSSL') + if openssl_root: + if 'boringssl' not in system_libs: + system_libs.append('boringssl') + incpath = os.path.join(openssl_root, 'include') + if os.path.exists(incpath): + cpaths.append(incpath) + libpath = get_software_libdir(dep_name) + if libpath: + libpaths.append(os.path.join(openssl_root, libpath)) + if ignored_system_deps: print_warning('%d TensorFlow dependencies have not been resolved by EasyBuild. Check the log for details.', len(ignored_system_deps)) @@ -418,10 +437,14 @@ def get_system_libs(self): def setup_build_dirs(self): """Setup temporary build directories""" + # This is either the builddir (for standalone builds) or the extension sub folder when TF is an extension + # Either way this folder only contains the folder with the sources and hence we can use fixed names + # for the subfolders + parent_dir = os.path.dirname(self.start_dir) # Path where Bazel will store its output, build artefacts etc. - self.output_user_root_dir = tempfile.mkdtemp(suffix='-bazel-tf', dir=self.builddir) + self.output_user_root_dir = os.path.join(parent_dir, 'bazel-root') # Folder where wrapper binaries can be placed, where required. TODO: Replace by --action_env cmds - self.wrapper_dir = tempfile.mkdtemp(suffix='-wrapper_bin', dir=self.builddir) + self.wrapper_dir = os.path.join(parent_dir, 'wrapper_bin') def configure_step(self): """Custom configuration procedure for TensorFlow.""" @@ -430,7 +453,9 @@ def configure_step(self): # and will hang forever building the TensorFlow package. # So limit to something high but still reasonable while allowing ECs to overwrite it if self.cfg['maxparallel'] is None: - self.cfg['parallel'] = min(self.cfg['parallel'], 64) + # Seemingly Bazel around 3.x got better, so double the max there + bazel_max = 64 if get_bazel_version() < '3.0.0' else 128 + self.cfg['parallel'] = min(self.cfg['parallel'], bazel_max) binutils_root = get_software_root('binutils') if not binutils_root: @@ -733,6 +758,8 @@ def patch_crosstool_files(self): def build_step(self): """Custom build procedure for TensorFlow.""" + bazel_version = get_bazel_version() + # pre-create target installation directory mkdir(os.path.join(self.installdir, self.pylibdir), parents=True) @@ -744,6 +771,15 @@ def build_step(self): self.bazel_opts = [ '--output_user_root=%s' % self.output_user_root_dir, ] + # Increase time to wait for bazel to start, available since 4.0+ + if bazel_version >= '4.0.0': + self.bazel_opts.append('--local_startup_timeout_secs=300') # 5min + + # Environment variables and values needed for Bazel actions. + action_env = {} + # A value of None is interpreted as using the invoking environments value + INHERIT = None # For better readability + jvm_max_memory = self.cfg['jvm_max_memory'] if jvm_max_memory: jvm_startup_memory = min(512, int(jvm_max_memory)) @@ -785,21 +821,17 @@ def build_step(self): # Make TF find our modules. LD_LIBRARY_PATH gets automatically added by configure.py cpaths, libpaths = self.system_libs_info[1:] if cpaths: - self.target_opts.append("--action_env=CPATH='%s'" % ':'.join(cpaths)) + action_env['CPATH'] = ':'.join(cpaths) if libpaths: - self.target_opts.append("--action_env=LIBRARY_PATH='%s'" % ':'.join(libpaths)) - self.target_opts.append('--action_env=PYTHONPATH') + action_env['LIBRARY_PATH'] = ':'.join(libpaths) + action_env['PYTHONPATH'] = INHERIT # Also export $EBPYTHONPREFIXES to handle the multi-deps python setup # See https://github.com/easybuilders/easybuild-easyblocks/pull/1664 if 'EBPYTHONPREFIXES' in os.environ: - self.target_opts.append('--action_env=EBPYTHONPREFIXES') + action_env['EBPYTHONPREFIXES'] = INHERIT # Ignore user environment for Python - self.target_opts.append('--action_env=PYTHONNOUSERSITE=1') - - # Use the same configuration (i.e. environment) for compiling and using host tools - # This means that our action_envs are always passed - self.target_opts.append('--distinct_host_configuration=false') + action_env['PYTHONNOUSERSITE'] = '1' # TF 2 (final) sets this in configure if LooseVersion(self.version) < LooseVersion('2.0'): @@ -835,6 +867,24 @@ def build_step(self): # and download it if needed self.target_opts.append('--config=mkl') + # Use the same configuration (i.e. environment) for compiling and using host tools + # This means that our action_envs are (almost) always passed + # Fully removed in Bazel 6.0 and limited effect after at least 3.7 (see --host_action_env) + if bazel_version < '6.0.0': + self.target_opts.append('--distinct_host_configuration=false') + + for key, value in sorted(action_env.items()): + option = key + if value is not None: + option += "='%s'" % value + + self.target_opts.append('--action_env=' + option) + if bazel_version >= '3.7.0': + # Since Bazel 3.7 action_env only applies to the "target" environment, not the "host" environment + # As we are not cross-compiling we need both be the same -> Duplicate the setting to host_action_env + # See https://github.com/bazelbuild/bazel/commit/a463d9095386b22c121d20957222dbb44caef7d4 + self.target_opts.append('--host_action_env=' + option) + # Compose final command cmd = ( [self.cfg['prebuildopts']] @@ -1001,6 +1051,7 @@ def test_step(self): self.log.warning('Test %s failed with output\n%s', test_name, read_file(log_path, log_error=False)) if failed_tests: + failed_tests = sorted(set(failed_tests)) # Make unique to not count retries fail_msg = 'At least %s %s tests failed:\n%s' % ( len(failed_tests), device, ', '.join(failed_tests)) self.report_test_failure(fail_msg) diff --git a/easybuild/easyblocks/t/tensorrt.py b/easybuild/easyblocks/t/tensorrt.py index 3d5489b13c..204302e7d6 100644 --- a/easybuild/easyblocks/t/tensorrt.py +++ b/easybuild/easyblocks/t/tensorrt.py @@ -1,5 +1,5 @@ ## -# Copyright 2017-2022 Ghent University +# Copyright 2017-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -108,6 +108,7 @@ def extensions_step(self): 'installopts': self.cfg['installopts'], 'loc': whl_paths[0], 'prefix': self.installdir, + 'python': self.python_cmd, } # Use --no-deps to prevent pip from downloading & installing diff --git a/easybuild/easyblocks/t/tinker.py b/easybuild/easyblocks/t/tinker.py index 0eac1f609b..4892873b0d 100644 --- a/easybuild/easyblocks/t/tinker.py +++ b/easybuild/easyblocks/t/tinker.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/t/tkinter.py b/easybuild/easyblocks/t/tkinter.py index ec853d89a9..8f4312f4b5 100644 --- a/easybuild/easyblocks/t/tkinter.py +++ b/easybuild/easyblocks/t/tkinter.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/t/torchvision.py b/easybuild/easyblocks/t/torchvision.py index 1613a79be2..5536b89139 100644 --- a/easybuild/easyblocks/t/torchvision.py +++ b/easybuild/easyblocks/t/torchvision.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2022 Ghent University +# Copyright 2021-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -26,12 +26,12 @@ EasyBuild support for building and installing torchvision, implemented as an easyblock @author: Alexander Grund (TU Dresden) +@author: Kenneth Hoste (HPC-UGent) """ - -from easybuild.easyblocks.generic.pythonpackage import PythonPackage +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.modules import get_software_root, get_software_version +from easybuild.tools.modules import get_software_version import easybuild.tools.environment as env @@ -40,24 +40,66 @@ class EB_torchvision(PythonPackage): @staticmethod def extra_options(): - """Change some defaults.""" + """Change some defaults for easyconfig parameters.""" extra_vars = PythonPackage.extra_options() extra_vars['use_pip'][0] = True extra_vars['download_dep_fail'][0] = True extra_vars['sanity_pip_check'][0] = True return extra_vars + def __init__(self, *args, **kwargs): + """Initialize torchvision easyblock.""" + super(EB_torchvision, self).__init__(*args, **kwargs) + + dep_names = set(dep['name'] for dep in self.cfg.dependencies()) + + # require that PyTorch is listed as dependency + if 'PyTorch' not in dep_names: + raise EasyBuildError('PyTorch not found as a dependency') + + # enable building with GPU support if CUDA is included as dependency + if 'CUDA' in dep_names: + self.with_cuda = True + else: + self.with_cuda = False + def configure_step(self): """Set up torchvision config""" - if not get_software_root('PyTorch'): - raise EasyBuildError('PyTorch not found as a dependency') # Note: Those can be overwritten by e.g. preinstallopts env.setvar('BUILD_VERSION', self.version) env.setvar('PYTORCH_VERSION', get_software_version('PyTorch')) - if get_software_root('CUDA'): + + if self.with_cuda: + # make sure that torchvision is installed with CUDA support by setting $FORCE_CUDA + env.setvar('FORCE_CUDA', '1') + # specify CUDA compute capabilities via $TORCH_CUDA_ARCH_LIST cuda_cc = self.cfg['cuda_compute_capabilities'] or build_option('cuda_compute_capabilities') if cuda_cc: env.setvar('TORCH_CUDA_ARCH_LIST', ';'.join(cuda_cc)) super(EB_torchvision, self).configure_step() + + def sanity_check_step(self): + """Custom sanity check for torchvision.""" + custom_commands = None + custom_paths = None + + # check whether torchvision was indeed built with CUDA support, + # cfr. https://discuss.pytorch.org/t/notimplementederror-could-not-run-torchvision-nms-with-arguments-from-\ + # the-cuda-backend-this-could-be-because-the-operator-doesnt-exist-for-this-backend/132352/4 + if self.with_cuda: + custom_commands = [] + python_code = '; '.join([ + "import torch, torchvision", + "boxes = torch.tensor([[0., 1., 2., 3.]]).to('cuda')", + "scores = torch.randn(1).to('cuda')", + "print(torchvision.ops.nms(boxes, scores, 0.5))", + ]) + custom_commands.append('python -c "%s"' % python_code) + custom_paths = { + 'files': [], + 'dirs': [det_pylibdir()], + } + + return super(EB_torchvision, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) diff --git a/easybuild/easyblocks/t/tornado.py b/easybuild/easyblocks/t/tornado.py index 86c4f222ef..98a1c59705 100644 --- a/easybuild/easyblocks/t/tornado.py +++ b/easybuild/easyblocks/t/tornado.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/t/totalview.py b/easybuild/easyblocks/t/totalview.py index 3e5de8fb82..2cd8234d03 100644 --- a/easybuild/easyblocks/t/totalview.py +++ b/easybuild/easyblocks/t/totalview.py @@ -1,8 +1,8 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA -# Copyright:: Copyright 2016-2022 Forschungszentrum Juelich +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2016-2023 Forschungszentrum Juelich # Authors:: Fotis Georgatos # Authors:: Damian Alvarez # License:: MIT/GPL diff --git a/easybuild/easyblocks/t/trilinos.py b/easybuild/easyblocks/t/trilinos.py index 6261e821f5..aa4b911473 100644 --- a/easybuild/easyblocks/t/trilinos.py +++ b/easybuild/easyblocks/t/trilinos.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -54,6 +54,8 @@ def extra_options(): extra_vars = { 'shared_libs': [None, "Deprecated. Use build_shared_libs", CUSTOM], 'openmp': [True, "Enable OpenMP support", CUSTOM], + 'forward_deps': [True, "Enable all forward dependencies", CUSTOM], + 'build_tests': [True, "Enable building tests/examples", CUSTOM], 'all_exts': [True, "Enable all Trilinos packages", CUSTOM], 'skip_exts': [[], "List of Trilinos packages to skip", CUSTOM], 'verbose': [False, "Configure for verbose output", CUSTOM], @@ -106,9 +108,11 @@ def configure_step(self): if self.toolchain.options.get('usempi', None): self.cfg.update('configopts', "-DTPL_ENABLE_MPI:BOOL=ON") - # enable full testing - self.cfg.update('configopts', "-DTrilinos_ENABLE_TESTS:BOOL=ON") - self.cfg.update('configopts', "-DTrilinos_ENABLE_ALL_FORWARD_DEP_PACKAGES:BOOL=ON") + if self.cfg['build_tests']: + # enable full testing + self.cfg.update('configopts', "-DTrilinos_ENABLE_TESTS:BOOL=ON") + if self.cfg['forward_deps']: + self.cfg.update('configopts', "-DTrilinos_ENABLE_ALL_FORWARD_DEP_PACKAGES:BOOL=ON") lib_re = re.compile("^lib(.*).a$") @@ -285,6 +289,15 @@ def sanity_check_step(self): libs.remove('Galeri') libs.extend(['galeri-epetra', 'galeri-xpetra']) + # Mesquite and MOOCHO packages gone in 12.18: + if LooseVersion(self.version) >= LooseVersion('12.18'): + libs.remove('Mesquite') + libs.remove('MOOCHO') + + # GlobiPack package gone in 13.0: + if LooseVersion(self.version) >= LooseVersion('13.0'): + libs.remove('GlobiPack') + # Get the library extension if self.cfg['build_shared_libs']: lib_ext = get_shared_lib_ext() diff --git a/easybuild/easyblocks/t/trinity.py b/easybuild/easyblocks/t/trinity.py index a69d1229df..7eeedf6e09 100644 --- a/easybuild/easyblocks/t/trinity.py +++ b/easybuild/easyblocks/t/trinity.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/u/ucx_plugins.py b/easybuild/easyblocks/u/ucx_plugins.py index e74758f063..10e5a97e13 100644 --- a/easybuild/easyblocks/u/ucx_plugins.py +++ b/easybuild/easyblocks/u/ucx_plugins.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/u/ufc.py b/easybuild/easyblocks/u/ufc.py index 1bb7caaac3..f77cf7ea24 100644 --- a/easybuild/easyblocks/u/ufc.py +++ b/easybuild/easyblocks/u/ufc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/v/velvet.py b/easybuild/easyblocks/v/velvet.py index 7f410c6657..c79f6ff995 100644 --- a/easybuild/easyblocks/v/velvet.py +++ b/easybuild/easyblocks/v/velvet.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2022 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2023 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/v/vep.py b/easybuild/easyblocks/v/vep.py index c9f5b9233c..c339abbe0a 100644 --- a/easybuild/easyblocks/v/vep.py +++ b/easybuild/easyblocks/v/vep.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/v/vmd.py b/easybuild/easyblocks/v/vmd.py index 43c32354a4..c7273ab5c6 100644 --- a/easybuild/easyblocks/v/vmd.py +++ b/easybuild/easyblocks/v/vmd.py @@ -1,6 +1,6 @@ ## -# Copyright 2009-2022 Ghent University -# Copyright 2015-2022 Stanford University +# Copyright 2009-2023 Ghent University +# Copyright 2015-2023 Stanford University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/v/vsc_tools.py b/easybuild/easyblocks/v/vsc_tools.py index ae520eb1ec..83611fdf73 100644 --- a/easybuild/easyblocks/v/vsc_tools.py +++ b/easybuild/easyblocks/v/vsc_tools.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/v/vtune.py b/easybuild/easyblocks/v/vtune.py index 00724666ec..40253f57af 100644 --- a/easybuild/easyblocks/v/vtune.py +++ b/easybuild/easyblocks/v/vtune.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/w/wien2k.py b/easybuild/easyblocks/w/wien2k.py index 1eb6671b8c..07c32ad40c 100644 --- a/easybuild/easyblocks/w/wien2k.py +++ b/easybuild/easyblocks/w/wien2k.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/w/wps.py b/easybuild/easyblocks/w/wps.py index 9536e24e29..d6c6cca849 100644 --- a/easybuild/easyblocks/w/wps.py +++ b/easybuild/easyblocks/w/wps.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index dc520e08bc..b232400724 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/w/wrf_fire.py b/easybuild/easyblocks/w/wrf_fire.py index d3e0017de7..ce5ea7d82a 100644 --- a/easybuild/easyblocks/w/wrf_fire.py +++ b/easybuild/easyblocks/w/wrf_fire.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/w/wxpython.py b/easybuild/easyblocks/w/wxpython.py index b777a669ba..35b22b1867 100644 --- a/easybuild/easyblocks/w/wxpython.py +++ b/easybuild/easyblocks/w/wxpython.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/x/xalt.py b/easybuild/easyblocks/x/xalt.py index 269ad01df2..e5e4f07f45 100644 --- a/easybuild/easyblocks/x/xalt.py +++ b/easybuild/easyblocks/x/xalt.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2022 NVIDIA +# Copyright 2020-2023 NVIDIA # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/easyblocks/x/xcrysden.py b/easybuild/easyblocks/x/xcrysden.py index ad2cfc5d37..2a2c9891f9 100644 --- a/easybuild/easyblocks/x/xcrysden.py +++ b/easybuild/easyblocks/x/xcrysden.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/x/xmipp.py b/easybuild/easyblocks/x/xmipp.py index c84a43a50f..95230cd03e 100644 --- a/easybuild/easyblocks/x/xmipp.py +++ b/easybuild/easyblocks/x/xmipp.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -101,10 +101,15 @@ def configure_step(self): # Tell xmipp config that there is no Scipion. env.setvar('XMIPP_NOSCIPION', 'True') # Initialize the config file and then patch it with the correct values + if LooseVersion(self.version) >= LooseVersion('3.20.07'): + noask = 'noAsk' + else: + noask = '' cmd = ' '.join([ self.cfg['preconfigopts'], self.xmipp_exe, 'config', + noask, self.cfg['configopts'], ]) run_cmd(cmd, log_all=True, simple=True) @@ -145,9 +150,13 @@ def configure_step(self): if cuda_root: params.update({'CUDA_BIN': os.path.join(cuda_root, 'bin')}) params.update({'CUDA_LIB': os.path.join(cuda_root, 'lib64')}) - params.update({'NVCC': os.environ['CUDA_CXX']}) + params.update({'NVCC': os.environ.get('CUDA_CXX', 'nvcc')}) # Their default for NVCC is to use g++-5, fix that - nvcc_flags = '-v --x cu -D_FORCE_INLINES -Xcompiler -fPIC -Wno-deprecated-gpu-targets -std=c++11' + nvcc_flags = '-v --x cu -D_FORCE_INLINES -Xcompiler -fPIC -Wno-deprecated-gpu-targets' + if LooseVersion(self.version) < LooseVersion('3.22'): + nvcc_flags += ' --std=c++11' + else: + nvcc_flags += ' --std=c++17' if LooseVersion(self.version) >= LooseVersion('3.20.07'): nvcc_flags += ' --extended-lambda' params.update({'NVCC_CXXFLAGS': nvcc_flags}) diff --git a/easybuild/easyblocks/x/xml.py b/easybuild/easyblocks/x/xml.py index befe9144fa..14216fa364 100644 --- a/easybuild/easyblocks/x/xml.py +++ b/easybuild/easyblocks/x/xml.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/setup.py b/setup.py index 768d47e80f..5162b2fe3c 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -79,6 +79,8 @@ def read(fname): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Build Tools", ], platforms="Linux", diff --git a/test/__init__.py b/test/__init__.py index 461a6de808..01ba03af45 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2022 Ghent University +# Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/easyblocks/easyblock_specific.py b/test/easyblocks/easyblock_specific.py index 1db4277b72..4e4d10b4d2 100644 --- a/test/easyblocks/easyblock_specific.py +++ b/test/easyblocks/easyblock_specific.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2022 Ghent University +# Copyright 2019-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/easyblocks/general.py b/test/easyblocks/general.py index 824d4b215d..25b3ccba40 100644 --- a/test/easyblocks/general.py +++ b/test/easyblocks/general.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/easyblocks/init_easyblocks.py b/test/easyblocks/init_easyblocks.py index 650257f738..9a53f4b898 100644 --- a/test/easyblocks/init_easyblocks.py +++ b/test/easyblocks/init_easyblocks.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2022 Ghent University +# Copyright 2013-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -96,7 +96,7 @@ def tearDown(self): self.log.error("Failed to remove %s: %s" % (self.eb_file, err)) -def template_init_test(self, easyblock, name='foo', version='1.3.2', toolchain=None): +def template_init_test(self, easyblock, name='foo', version='1.3.2', toolchain=None, deps=None): """Test whether all easyblocks can be initialized.""" def check_extra_options_format(extra_options): @@ -164,6 +164,9 @@ def check_extra_options_format(extra_options): test_param = 'foo' extra_txt += '%s = "%s"\n' % (key, test_param) + if deps: + extra_txt += 'dependencies = %s' % str(deps) + # write easyconfig file self.write_ec(ebname, name=name, version=version, toolchain=toolchain, extratxt=extra_txt) @@ -224,6 +227,9 @@ def innertest(self): elif easyblock_fn == 'openssl_wrapper.py': # easyblock to create OpenSSL wrapper expects an OpenSSL version innertest = make_inner_test(easyblock, version='1.1') + elif easyblock_fn == 'torchvision.py': + # torchvision easyblock requires that PyTorch is listed as dependency + innertest = make_inner_test(easyblock, name='torchvision', deps=[('PyTorch', '1.12.1')]) else: innertest = make_inner_test(easyblock) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index 034dc65961..db44834031 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -42,6 +42,8 @@ from easybuild.base import fancylogger from easybuild.base.testing import TestCase from easybuild.easyblocks.generic.gopackage import GoPackage +from easybuild.easyblocks.generic.juliabundle import JuliaBundle +from easybuild.easyblocks.generic.juliapackage import JuliaPackage from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.easyblocks.generic.pythonbundle import PythonBundle from easybuild.easyblocks.gcc import EB_GCC @@ -116,7 +118,7 @@ class ModuleOnlyTest(TestCase): def writeEC(self, easyblock, name='foo', version='1.3.2', extratxt='', toolchain=None): """ create temporary easyconfig file """ if toolchain is None: - toolchain = {'name': 'dummy', 'version': 'dummy'} + toolchain = {'name': 'system', 'version': 'system'} txt = '\n'.join([ 'easyblock = "%s"', @@ -286,6 +288,11 @@ def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt= os.environ['EBROOTGO'] = '/fake/install/prefix/Go/1.14' os.environ['EBVERSIONGO'] = '1.14' + elif app_class in (JuliaPackage, JuliaBundle): + # $EBROOTJULIA must be set for JuliaPackage/JuliaBundle easyblock + os.environ['EBROOTJULIA'] = '/fake/install/prefix/Julia/1.6.7' + os.environ['EBVERSIONJULIA'] = '1.6.7' + elif app_class == EB_OpenFOAM: # proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER write_file(os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([ @@ -326,7 +333,7 @@ def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt= self.writeEC(ebname, name=name, version=version, extratxt=extra_txt, toolchain=toolchain) # take into account that for some easyblock, particular dependencies are hard required early on - # (in prepare_step for exampel); + # (in prepare_step for example); # we just set the corresponding $EBROOT* environment variables here to fool it... req_deps = { # QScintilla easyblock requires that either PyQt or PyQt5 are available as dependency @@ -427,8 +434,9 @@ def innertest(self): for prgenv in ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi']: write_file(os.path.join(TMPDIR, 'modules', 'all', prgenv, '1.2.3'), "#%Module") - # add foo/1.3.2.1.1 module, required for testing ModuleAlias easyblock - write_file(os.path.join(TMPDIR, 'modules', 'all', 'foo', '1.2.3.4.5'), "#%Module") + # add empty module files for dependencies that are required for testing easyblocks + for dep_mod_name in ('foo/1.2.3.4.5', 'PyTorch/1.12.1'): + write_file(os.path.join(TMPDIR, 'modules', 'all', dep_mod_name), "#%Module") for easyblock in easyblocks: eb_fn = os.path.basename(easyblock) @@ -453,6 +461,10 @@ def innertest(self): elif eb_fn == 'openssl_wrapper.py': # easyblock to create OpenSSL wrapper expects an OpenSSL version innertest = make_inner_test(easyblock, name='OpenSSL-wrapper', version='1.1') + elif eb_fn == 'torchvision.py': + # torchvision easyblock requires that PyTorch is listed as dependency + extra_txt = "dependencies = [('PyTorch', '1.12.1')]" + innertest = make_inner_test(easyblock, name='torchvision', extra_txt=extra_txt) elif eb_fn == 'ucx_plugins.py': # install fake ucx_info command (used in make_module_extra) tmpdir = tempfile.mkdtemp() diff --git a/test/easyblocks/suite.py b/test/easyblocks/suite.py index c3f3613b4d..3af803f312 100644 --- a/test/easyblocks/suite.py +++ b/test/easyblocks/suite.py @@ -1,6 +1,6 @@ #!/usr/bin/python ## -# Copyright 2012-2022 Ghent University +# Copyright 2012-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),