diff --git a/easybuild/easyblocks/generic/binary.py b/easybuild/easyblocks/generic/binary.py index d9ff3a5bb7..62380d49cd 100644 --- a/easybuild/easyblocks/generic/binary.py +++ b/easybuild/easyblocks/generic/binary.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, remove_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd PREPEND_TO_PATH_DEFAULT = [''] @@ -126,7 +126,7 @@ def install_step(self): for install_cmd in install_cmds: cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']]) self.log.info("Running install command for %s: '%s'..." % (self.name, cmd)) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) else: raise EasyBuildError("Incorrect value type for install_cmds, should be list or tuple: ", install_cmds) diff --git a/easybuild/easyblocks/generic/cargo.py b/easybuild/easyblocks/generic/cargo.py index c47f2e9f05..3559da7eaf 100644 --- a/easybuild/easyblocks/generic/cargo.py +++ b/easybuild/easyblocks/generic/cargo.py @@ -36,7 +36,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.filetools import extract_file, change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.config import build_option from easybuild.tools.filetools import write_file, compute_checksum from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC @@ -199,7 +199,7 @@ def build_step(self): if self.cfg['lto'] is not None: lto = '--config profile.%s.lto=\\"%s\\"' % (self.profile, self.cfg['lto']) - run_cmd('rustc --print cfg', log_all=True, simple=True) # for tracking in log file + run_shell_cmd('rustc --print cfg') # for tracking in log file cmd = ' '.join([ self.cfg['prebuildopts'], 'cargo build', @@ -209,7 +209,7 @@ def build_step(self): parallel, self.cfg['buildopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def test_step(self): """Test with cargo""" @@ -220,7 +220,7 @@ def test_step(self): '--profile=' + self.profile, self.cfg['testopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def install_step(self): """Install with cargo""" @@ -232,7 +232,7 @@ def install_step(self): '--path=.', self.cfg['installopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def generate_crate_list(sourcedir): diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 0a326a8e5a..6838a68168 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -45,7 +45,7 @@ from easybuild.tools.filetools import change_dir, create_unused_dir, mkdir, which from easybuild.tools.environment import setvar from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import nub @@ -63,7 +63,8 @@ def det_cmake_version(): regex = re.compile(r"^[cC][mM]ake version (?P[0-9]\.[0-9a-zA-Z.-]+)$", re.M) cmd = "cmake --version" - (out, _) = run_cmd(cmd, simple=False, log_ok=False, log_all=False, trace=False) + cmd_res = run_shell_cmd(cmd, hidden=True, fail_on_error=False) + out = cmd_res.output res = regex.search(out) if res: cmake_version = res.group('version') @@ -311,9 +312,9 @@ def configure_step(self, srcdir=None, builddir=None): self.cfg.get('configure_cmd'), self.cfg['configopts']]) - (out, _) = run_cmd(command, log_all=True, simple=False) + res = run_shell_cmd(command) - return out + return res.output def test_step(self): """CMake specific test setup""" diff --git a/easybuild/easyblocks/generic/cmdcp.py b/easybuild/easyblocks/generic/cmdcp.py index 5a91112a4e..12c34486d2 100644 --- a/easybuild/easyblocks/generic/cmdcp.py +++ b/easybuild/easyblocks/generic/cmdcp.py @@ -32,7 +32,7 @@ from easybuild.easyblocks.generic.makecp import MakeCp from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class CmdCp(MakeCp): @@ -80,4 +80,4 @@ def build_step(self): raise EasyBuildError("No match for %s in %s, don't know which command to use.", src, self.cfg['cmds_map']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) diff --git a/easybuild/easyblocks/generic/conda.py b/easybuild/easyblocks/generic/conda.py index 03e78ab42e..3ad69f1f6e 100644 --- a/easybuild/easyblocks/generic/conda.py +++ b/easybuild/easyblocks/generic/conda.py @@ -33,7 +33,7 @@ from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.modules import get_software_root from easybuild.tools.build_log import EasyBuildError @@ -74,7 +74,7 @@ def install_step(self): # initialize conda environment # setuptools is just a choice, but *something* needs to be there cmd = "%s config --add create_default_packages setuptools" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if self.cfg['environment_file'] or self.cfg['remote_environment']: @@ -86,7 +86,7 @@ def install_step(self): # use --force to ignore existing installation directory cmd = "%s %s env create --force %s -p %s" % (self.cfg['preinstallopts'], conda_cmd, env_spec, self.installdir) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) else: @@ -100,11 +100,11 @@ def install_step(self): cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd, self.installdir, install_args) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # clean up cmd = "%s clean -ya" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def make_module_extra(self): """Add the install directory to the PATH.""" diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index 9e4066b60b..39e8825b65 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -49,7 +49,7 @@ from easybuild.tools.config import source_paths, build_option from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file from easybuild.tools.filetools import read_file, remove_file -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd # string that indicates that a configure script was generated by Autoconf # note: bytes string since this constant is used to check the contents of 'configure' which is read as bytes @@ -247,8 +247,8 @@ def determine_build_and_host_type(self): "EasyBuild attempts to download a recent config.guess but seems to have failed!") else: self.check_config_guess() - system_type, _ = run_cmd(self.config_guess, log_all=True) - system_type = system_type.strip() + res = run_shell_cmd(self.config_guess) + system_type = res.output.strip() self.log.info("%s returned a system type '%s'", self.config_guess, system_type) if build_type is None: @@ -323,16 +323,19 @@ def configure_step(self, cmd_prefix=''): ] + build_and_host_options + [self.cfg['configopts']] ) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output - def build_step(self, verbose=False, path=None): + def build_step(self, verbose=None, path=None): """ Start the actual build - typical: make -j X """ + if verbose is not None: + self.log.deprecated("The 'verbose' parameter to build_step is deprecated and unneeded.", '6.0') + paracmd = '' if self.cfg['parallel']: paracmd = "-j %s" % self.cfg['parallel'] @@ -351,7 +354,8 @@ def build_step(self, verbose=False, path=None): ]) self.log.info("Building target '%s'", target) - (out, _) = run_cmd(cmd, path=path, log_all=True, simple=False, log_output=verbose) + res = run_shell_cmd(cmd, work_dir=path) + out = res.output return out @@ -369,9 +373,9 @@ def test_step(self): runtest = '' # Compose command filtering out empty values cmd = ' '.join([x for x in (self.cfg['pretestopts'], test_cmd, runtest, self.cfg['testopts']) if x]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def install_step(self): """ @@ -385,6 +389,6 @@ def install_step(self): self.cfg['installopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output diff --git a/easybuild/easyblocks/generic/configuremakepythonpackage.py b/easybuild/easyblocks/generic/configuremakepythonpackage.py index 8da38021a2..d9ebf8dc1c 100644 --- a/easybuild/easyblocks/generic/configuremakepythonpackage.py +++ b/easybuild/easyblocks/generic/configuremakepythonpackage.py @@ -31,7 +31,7 @@ """ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.pythonpackage import PythonPackage -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class ConfigureMakePythonPackage(ConfigureMake, PythonPackage): @@ -58,7 +58,7 @@ def configure_step(self, *args, **kwargs): """Configure build using ``python configure``.""" PythonPackage.configure_step(self, *args, **kwargs) cmd = ' '.join([self.cfg['preconfigopts'], self.python_cmd, self.cfg['configopts']]) - run_cmd(cmd, log_all=True) + run_shell_cmd(cmd) def build_step(self, *args, **kwargs): """Build Python package with ``make``.""" diff --git a/easybuild/easyblocks/generic/fortranpythonpackage.py b/easybuild/easyblocks/generic/fortranpythonpackage.py index 849dacf54b..1e434c50ca 100644 --- a/easybuild/easyblocks/generic/fortranpythonpackage.py +++ b/easybuild/easyblocks/generic/fortranpythonpackage.py @@ -36,7 +36,7 @@ import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class FortranPythonPackage(PythonPackage): @@ -70,4 +70,4 @@ def build_step(self): raise EasyBuildError("Unknown family of compilers being used: %s", comp_fam) cmd = "%s %s setup.py build %s" % (self.cfg['prebuildopts'], self.python_cmd, self.cfg['buildopts']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index 8a3b591402..b829159349 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -35,7 +35,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class GoPackage(EasyBlock): @@ -71,8 +71,8 @@ def configure_step(self): env.setvar('GOBIN', os.path.join(self.installdir, 'bin'), verbose=False) # creates log entries for go being used, for debugging - run_cmd("go version", verbose=False, trace=False) - run_cmd("go env", verbose=False, trace=False) + run_shell_cmd("go version", hidden=True) + run_shell_cmd("go env", hidden=True) def build_step(self): """If Go package is not native go module, lets try to make the module.""" @@ -91,13 +91,13 @@ def build_step(self): # go mod init cmd = ' '.join(['go', 'mod', 'init', self.cfg['modulename']]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if self.cfg['forced_deps']: for dep in self.cfg['forced_deps']: # go get specific dependencies which locks them in go.mod cmd = ' '.join(['go', 'get', '%s@%s' % dep]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # note: ... (tripledot) used below is not a typo, but go wildcard pattern # which means: anything you can find in this directory, including all subdirectories @@ -105,20 +105,20 @@ def build_step(self): # see: https://stackoverflow.com/a/28031651/2047157 # building and testing will add packages to go.mod - run_cmd('go build ./...', log_all=True, simple=True) - run_cmd('go test ./...', log_all=True, simple=True) + run_shell_cmd('go build ./...') + run_shell_cmd('go test ./...') # tidy up go.mod - run_cmd('go mod tidy', log_all=True, simple=True) + run_shell_cmd('go mod tidy') # build and test again, to ensure go mod tidy didn't removed anything needed - run_cmd('go build ./...', log_all=True, simple=True) - run_cmd('go test ./...', log_all=True, simple=True) + run_shell_cmd('go build ./...') + run_shell_cmd('go test ./...') self.log.warn('Include generated go.mod and go.sum via patch to ensure locked dependencies ' 'and run this easyconfig again.') - run_cmd('cat go.mod', log_all=True, simple=True) - run_cmd('cat go.sum', log_all=True, simple=True) + run_shell_cmd('cat go.mod') + run_shell_cmd('cat go.sum') if not os.path.exists(go_sum_file) or not os.path.isfile(go_sum_file): raise EasyBuildError("go.sum not found! This module has no locked dependency versions.") @@ -136,7 +136,7 @@ def install_step(self): '-x', self.cfg['installopts'], ]) - run_cmd(cmd, log_all=True, log_ok=True, simple=True) + run_shell_cmd(cmd) def sanity_check_step(self): """Custom sanity check for Go package.""" diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index 1e918605dc..665cca479d 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -49,7 +49,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, find_flexlm_license from easybuild.tools.filetools import mkdir, read_file, remove_file, write_file -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd # different supported activation types (cfr. Intel documentation) @@ -428,7 +428,7 @@ def install_step_classic(self, silent_cfg_names_map=None, silent_cfg_extras=None self.cfg['installopts'], ]) - return run_cmd(cmd, log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd) def install_step_oneapi(self, *args, **kwargs): """ @@ -468,16 +468,16 @@ def install_step_oneapi(self, *args, **kwargs): cmd.append(self.cfg['installopts']) - return run_cmd(' '.join(cmd), log_all=True, simple=True, log_output=True) + run_shell_cmd(' '.join(cmd)) def install_step(self, *args, **kwargs): """ Install Intel software """ if LooseVersion(self.version) >= LooseVersion('2021'): - return self.install_step_oneapi(*args, **kwargs) + self.install_step_oneapi(*args, **kwargs) else: - return self.install_step_classic(*args, **kwargs) + self.install_step_classic(*args, **kwargs) def move_after_install(self): """Move installed files to correct location after installation.""" diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 2ff1e49e89..90a2c6f139 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -38,7 +38,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.filetools import copy_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "") USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?$") @@ -147,9 +147,9 @@ def install_step(self): "julia -e '%s'" % julia_pkg_cmd, self.cfg['installopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def run(self): """Install Julia package as an extension.""" diff --git a/easybuild/easyblocks/generic/mesonninja.py b/easybuild/easyblocks/generic/mesonninja.py index 1fafaed53d..7297b84d4d 100644 --- a/easybuild/easyblocks/generic/mesonninja.py +++ b/easybuild/easyblocks/generic/mesonninja.py @@ -34,7 +34,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, create_unused_dir, which from easybuild.tools.modules import get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd DEFAULT_CONFIGURE_CMD = 'meson' DEFAULT_BUILD_CMD = 'ninja' @@ -99,8 +99,8 @@ def configure_step(self, cmd_prefix=''): 'preconfigopts': self.cfg['preconfigopts'], 'source_dir': build_dir, } - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output def build_step(self, verbose=False, path=None): """ @@ -118,8 +118,8 @@ def build_step(self, verbose=False, path=None): 'parallel': parallel, 'prebuildopts': self.cfg['prebuildopts'], } - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output def test_step(self): """ @@ -127,8 +127,8 @@ def test_step(self): """ if self.cfg['runtest']: cmd = "%s %s %s" % (self.cfg['pretestopts'], self.cfg['runtest'], self.cfg['testopts']) - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output def install_step(self): """ @@ -146,5 +146,5 @@ def install_step(self): 'install_cmd': install_cmd, 'preinstallopts': self.cfg['preinstallopts'], } - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output diff --git a/easybuild/easyblocks/generic/ocamlpackage.py b/easybuild/easyblocks/generic/ocamlpackage.py index d0b460d2ca..4010006c7e 100644 --- a/easybuild/easyblocks/generic/ocamlpackage.py +++ b/easybuild/easyblocks/generic/ocamlpackage.py @@ -29,7 +29,7 @@ """ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class OCamlPackage(ExtensionEasyBlock): @@ -42,8 +42,8 @@ def configure_step(self): def run(self): """Perform OCaml package installation (as extension).""" # install using 'opam install' - run_cmd("eval `opam config env` && opam install -yv %s.%s" % (self.name, self.version)) + run_shell_cmd("eval `opam config env` && opam install -yv %s.%s" % (self.name, self.version)) # 'opam pin add' fixes the version of the package # see https://opam.ocaml.org/doc/Usage.html#opampin - run_cmd("eval `opam config env` && opam pin -yv add %s %s" % (self.name, self.version)) + run_shell_cmd("eval `opam config env` && opam pin -yv add %s %s" % (self.name, self.version)) diff --git a/easybuild/easyblocks/generic/octavepackage.py b/easybuild/easyblocks/generic/octavepackage.py index cff63cc9c6..cbe26f3287 100644 --- a/easybuild/easyblocks/generic/octavepackage.py +++ b/easybuild/easyblocks/generic/octavepackage.py @@ -34,7 +34,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class OctavePackage(ExtensionEasyBlock): @@ -56,7 +56,7 @@ def run(self): # create temporary tarball from unpacked & patched source src = os.path.join(tempfile.gettempdir(), '%s-%s-patched.tar.gz' % (self.name, self.version)) cwd = change_dir(os.path.dirname(self.ext_dir)) - run_cmd("tar cfvz %s %s" % (src, os.path.basename(self.ext_dir))) + run_shell_cmd("tar cfvz %s %s" % (src, os.path.basename(self.ext_dir))) change_dir(cwd) else: src = self.src @@ -69,4 +69,4 @@ def run(self): octave_cmd += "pkg install -global %s" % src - run_cmd("octave --eval '%s'" % octave_cmd) + run_shell_cmd("octave --eval '%s'" % octave_cmd) diff --git a/easybuild/easyblocks/generic/perlmodule.py b/easybuild/easyblocks/generic/perlmodule.py index e10b6d5b1c..ff3c1bbf50 100644 --- a/easybuild/easyblocks/generic/perlmodule.py +++ b/easybuild/easyblocks/generic/perlmodule.py @@ -35,7 +35,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.environment import unset_env_vars @@ -79,7 +79,7 @@ def install_perl_module(self): '%s=%s' % (prefix_opt, self.installdir), self.cfg['configopts'], ]) - run_cmd(install_cmd) + run_shell_cmd(install_cmd) ConfigureMake.build_step(self) ConfigureMake.test_step(self) @@ -98,13 +98,13 @@ def install_perl_module(self): self.installdir, self.cfg['configopts'], ]) - run_cmd(install_cmd) + run_shell_cmd(install_cmd) - run_cmd("%s perl Build build %s" % (self.cfg['prebuildopts'], self.cfg['buildopts'])) + run_shell_cmd("%s perl Build build %s" % (self.cfg['prebuildopts'], self.cfg['buildopts'])) if self.cfg['runtest']: - run_cmd('perl Build %s' % self.cfg['runtest']) - run_cmd('%s perl Build install %s' % (self.cfg['preinstallopts'], self.cfg['installopts'])) + run_shell_cmd('perl Build %s' % self.cfg['runtest']) + run_shell_cmd('%s perl Build install %s' % (self.cfg['preinstallopts'], self.cfg['installopts'])) def run(self): """Perform the actual Perl module build/installation procedure""" diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index b6f63dbf5b..1a9cec9ab3 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -51,7 +51,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import change_dir, mkdir, remove_dir, symlink, which from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd, subprocess_popen_text +from easybuild.tools.run import run_shell_cmd, subprocess_popen_text from easybuild.tools.utilities import nub from easybuild.tools.hooks import CONFIGURE_STEP, BUILD_STEP, TEST_STEP, INSTALL_STEP @@ -78,8 +78,8 @@ def det_python_version(python_cmd): """Determine version of specified 'python' command.""" pycode = 'import sys; print("%s.%s.%s" % sys.version_info[:3])' - out, _ = run_cmd("%s -c '%s'" % (python_cmd, pycode), simple=False, trace=False) - return out.strip() + res = run_shell_cmd("%s -c '%s'" % (python_cmd, pycode), hidden=True) + return res.output.strip() def pick_python_cmd(req_maj_ver=None, req_min_ver=None): @@ -169,7 +169,7 @@ def det_pylibdir(plat_specific=False, python_cmd=None): python_cmd = 'python' # determine Python lib dir via distutils - # use run_cmd, we can to talk to the active Python, not the system Python running EasyBuild + # use run_shell_cmd, we can to talk to the active Python, not the system Python running EasyBuild prefix = '/tmp/' if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'): # Python 3.12 removed distutils but has a core sysconfig module which is similar @@ -183,13 +183,13 @@ def det_pylibdir(plat_specific=False, python_cmd=None): log.debug("Determining Python library directory using command '%s'", cmd) - out, ec = run_cmd(cmd, simple=False, force_in_dry_run=True, trace=False) - txt = out.strip().split('\n')[-1] + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True) + txt = res.output.strip().split('\n')[-1] # value obtained should start with specified prefix, otherwise something is very wrong if not txt.startswith(prefix): raise EasyBuildError("Last line of output of %s does not start with specified prefix %s: %s (exit code %s)", - cmd, prefix, out, ec) + cmd, prefix, res.output, res.exit_code) pylibdir = txt[len(prefix):] log.debug("Determined pylibdir using '%s': %s", cmd, pylibdir) @@ -227,7 +227,8 @@ def det_pip_version(python_cmd='python'): log = fancylogger.getLogger('det_pip_version', fname=False) log.info("Determining pip version...") - out, _ = run_cmd("%s -m pip --version" % python_cmd, verbose=False, simple=False, trace=False) + res = run_shell_cmd("%s -m pip --version" % python_cmd, hidden=True) + out = res.output pip_version_regex = re.compile('^pip ([0-9.]+)') res = pip_version_regex.search(out) @@ -261,8 +262,8 @@ def det_py_install_scheme(python_cmd='python'): cmd = "%s -c 'import sysconfig; print(sysconfig.%s())'" % (python_cmd, get_default_scheme) log.debug("Determining active Python installation scheme with: %s", cmd) - out, _ = run_cmd(cmd, verbose=False, simple=False, trace=False) - py_install_scheme = out.strip() + res = run_shell_cmd(cmd, hidden=True) + py_install_scheme = res.output.strip() if py_install_scheme in PY_INSTALL_SCHEMES: log.info("Active Python installation scheme: %s", py_install_scheme) @@ -754,7 +755,7 @@ def configure_step(self): # creates log entries for python being used, for debugging cmd = "%(python)s -V; %(python)s -c 'import sys; print(sys.executable, sys.path)'" - run_cmd(cmd % {'python': self.python_cmd}, verbose=False, trace=False) + run_shell_cmd(cmd % {'python': self.python_cmd}, hidden=True) def build_step(self): """Build Python package using setup.py""" @@ -774,12 +775,12 @@ def build_step(self): if build_cmd: build_cmd = build_cmd % {'python': self.python_cmd} cmd = ' '.join([self.cfg['prebuildopts'], build_cmd, self.cfg['buildopts']]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) # keep track of all output, so we can check for auto-downloaded dependencies; # take into account that build/install steps may be run multiple times # We consider the build and install output together as downloads likely happen here if this is run - self.install_cmd_output += out + self.install_cmd_output += res.output def test_step(self, return_output_ec=False): """ @@ -817,13 +818,13 @@ def test_step(self, return_output_ec=False): raise EasyBuildError("Failed to create test install dir: %s", err) # print Python search path (just debugging purposes) - run_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, verbose=False, trace=False) + run_shell_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, hidden=True) abs_pylibdirs = [os.path.join(actual_installdir, pylibdir) for pylibdir in self.all_pylibdirs] extrapath = "export PYTHONPATH=%s &&" % os.pathsep.join(abs_pylibdirs + ['$PYTHONPATH']) cmd = self.compose_install_command(test_installdir, extrapath=extrapath) - run_cmd(cmd, log_all=True, simple=True, verbose=False) + run_shell_cmd(cmd, hidden=True) self.py_post_install_shenanigans(test_installdir) @@ -837,11 +838,12 @@ def test_step(self, return_output_ec=False): ]) if return_output_ec: - (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False) - # need to log seperately, since log_all and log_ok need to be false to retrieve out and ec + res = run_shell_cmd(cmd, fail_on_error=False) + # need to retrieve ec by not failing on error + (out, ec) = (res.output, res.exit_code) self.log.info("cmd '%s' exited with exit code %s and output:\n%s", cmd, ec, out) else: - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if test_installdir: remove_dir(test_installdir) @@ -879,12 +881,12 @@ def install_step(self): # actually install Python package cmd = self.compose_install_command(self.installdir) - (out, _) = run_cmd(cmd, log_all=True, log_ok=True, simple=False) + res = run_shell_cmd(cmd) # keep track of all output from install command, so we can check for auto-downloaded dependencies; # take into account that install step may be run multiple times # (for iterated installations over multiply Python versions) - self.install_cmd_output += out + self.install_cmd_output += res.output self.py_post_install_shenanigans(self.installdir) @@ -1029,8 +1031,9 @@ def sanity_check_step(self, *args, **kwargs): pip_check_errors = [] - pip_check_msg, ec = run_cmd(pip_check_command, log_ok=False) - if ec: + res = run_shell_cmd(pip_check_command, fail_on_error=False) + pip_check_msg = res.output + if res.exit_code: pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg)) else: self.log.info('`%s` completed successfully' % pip_check_command) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 897408503c..bc43ce6970 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -41,7 +41,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import mkdir, copy_file -from easybuild.tools.run import run_cmd, parse_log_for_error +from easybuild.tools.run import run_shell_cmd, parse_log_for_error def make_R_install_option(opt, values, cmdline=False): @@ -168,8 +168,8 @@ def build_step(self): def install_R_package(self, cmd, inp=None): """Install R package as specified, and check for errors.""" - output, _ = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False) - self.check_install_output(output) + res = run_shell_cmd(cmd, stdin=inp) + self.check_install_output(res.output) def check_install_output(self, output): """ @@ -184,7 +184,7 @@ def check_install_output(self, output): """ % self.name # remove package if errors were detected # it's possible that some of the dependencies failed, but the package itself was installed - run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) + run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) raise EasyBuildError("Errors detected during installation of R package %s!", self.name) else: self.log.debug("R package %s installed succesfully", self.name) @@ -216,11 +216,11 @@ def required_deps(self): if self._required_deps is None: if self.src: cmd = "tar --wildcards --extract --file %s --to-stdout '*/DESCRIPTION'" % self.src - out, _ = run_cmd(cmd, simple=False, trace=False) + res = run_shell_cmd(cmd, hidden=True) # lines that start with whitespace are merged with line above lines = [] - for line in out.splitlines(): + for line in res.output.splitlines(): if line and line[0] in (' ', '\t'): lines[-1] = lines[-1] + line else: @@ -271,8 +271,8 @@ def prepare_r_ext_install(self): # determine location if isinstance(self.master, EB_R): # extension is being installed as part of an R installation/module - (out, _) = run_cmd("R RHOME", log_all=True, simple=False, trace=False) - rhome = out.strip() + res = run_shell_cmd("R RHOME", hidden=True) + rhome = res.output.strip() lib_install_prefix = os.path.join(rhome, 'library') else: # extension is being installed in a separate installation prefix diff --git a/easybuild/easyblocks/generic/rpm.py b/easybuild/easyblocks/generic/rpm.py index 1ab63bfff1..5156f716ea 100644 --- a/easybuild/easyblocks/generic/rpm.py +++ b/easybuild/easyblocks/generic/rpm.py @@ -46,7 +46,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, mkdir, symlink, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd _log = fancylogger.getLogger('easyblocks.generic.rpm') @@ -88,7 +88,7 @@ def rebuild_rpm(rpm_path, targetdir): targetdir, rpm_path, ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) class Rpm(Binary): @@ -142,10 +142,10 @@ def configure_step(self): # determine whether RPMs need to be rebuilt to make relocation work cmd = "rpm --version" - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) rpmver_re = re.compile(r"^RPM\s+version\s+(?P[0-9.]+).*") - res = rpmver_re.match(out) + res = rpmver_re.match(res.output) self.log.debug("RPM version found: %s" % res.group()) if res: @@ -185,7 +185,7 @@ def install_step(self): cmd = "rpm --initdb --dbpath /rpm --root %s" % self.installdir - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) force = '' if self.cfg['force']: @@ -219,7 +219,7 @@ def install_step(self): 'post': postinstall, 'installopts': self.cfg['installopts'], } - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) for path in self.cfg['makesymlinks']: # allow globs, always use first hit. diff --git a/easybuild/easyblocks/generic/rubygem.py b/easybuild/easyblocks/generic/rubygem.py index 58ecfa05c4..a989b91bf7 100644 --- a/easybuild/easyblocks/generic/rubygem.py +++ b/easybuild/easyblocks/generic/rubygem.py @@ -36,7 +36,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class RubyGem(ExtensionEasyBlock): @@ -113,7 +113,7 @@ def install_step(self): env.setvar('GEM_HOME', self.installdir) bindir = os.path.join(self.installdir, 'bin') - run_cmd("gem install --bindir %s --local %s" % (bindir, self.ext_src)) + run_shell_cmd("gem install --bindir %s --local %s" % (bindir, self.ext_src)) def make_module_extra(self): """Extend $GEM_PATH in module file.""" diff --git a/easybuild/easyblocks/generic/scons.py b/easybuild/easyblocks/generic/scons.py index 9820cb5855..b581bf0152 100644 --- a/easybuild/easyblocks/generic/scons.py +++ b/easybuild/easyblocks/generic/scons.py @@ -29,7 +29,7 @@ """ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class SCons(EasyBlock): @@ -66,9 +66,9 @@ def build_step(self, verbose=False): 'prefix': self.prefix, 'par': par, } - (out, _) = run_cmd(cmd, log_all=True, log_output=verbose) + res = run_shell_cmd(cmd) - return out + return res.output def test_step(self): """ @@ -76,7 +76,7 @@ def test_step(self): """ if self.cfg['runtest']: cmd = "%s scons %s %s" % (self.cfg['pretestopts'], self.cfg['runtest'], self.cfg['testopts']) - run_cmd(cmd, log_all=True) + run_shell_cmd(cmd) def install_step(self): """ @@ -87,6 +87,6 @@ def install_step(self): 'preinstallopts': self.cfg['preinstallopts'], 'prefix': self.prefix, } - (out, _) = run_cmd(cmd, log_all=True) + res = run_shell_cmd(cmd) - return out + return res.output diff --git a/easybuild/easyblocks/generic/systemcompiler.py b/easybuild/easyblocks/generic/systemcompiler.py index 0b01c30036..46d800731f 100644 --- a/easybuild/easyblocks/generic/systemcompiler.py +++ b/easybuild/easyblocks/generic/systemcompiler.py @@ -41,7 +41,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.filetools import read_file, resolve_path, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd _log = fancylogger.getLogger('easyblocks.generic.systemcompiler') @@ -54,7 +54,8 @@ def extract_compiler_version(compiler_name): # Intel(R) C Intel(R) 64 Compiler XE for applications running on Intel(R) 64, Version 15.0.1.133 Build 20141023 version_regex = re.compile(r'\s([0-9]+(?:\.[0-9]+){1,3})\s', re.M) if compiler_name == 'gcc': - out, _ = run_cmd("gcc --version", simple=False) + res = run_shell_cmd("gcc --version") + out = res.output res = version_regex.search(out) if res is None: raise EasyBuildError("Could not extract GCC version from %s", out) diff --git a/easybuild/easyblocks/generic/systemmpi.py b/easybuild/easyblocks/generic/systemmpi.py index 5ba5a3f812..5a71d058a1 100644 --- a/easybuild/easyblocks/generic/systemmpi.py +++ b/easybuild/easyblocks/generic/systemmpi.py @@ -38,7 +38,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.filetools import read_file, resolve_path, which from easybuild.tools.modules import get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class SystemMPI(Bundle, ConfigureMake, EB_impi): @@ -124,7 +124,8 @@ def prepare_step(self, *args, **kwargs): # Determine MPI version, installation prefix and underlying compiler if mpi_name in ('openmpi', 'spectrummpi'): # Spectrum MPI is based on Open MPI so is also covered by this logic - output_of_ompi_info, _ = run_cmd("ompi_info", simple=False) + res = run_shell_cmd("ompi_info") + output_of_ompi_info = res.output # Extract the version of the MPI implementation if mpi_name == 'spectrummpi': @@ -183,8 +184,9 @@ def prepare_step(self, *args, **kwargs): self.mpi_env_vars[key] = value # Extract the C compiler used underneath Intel MPI - compile_info, exit_code = run_cmd("%s -compile-info" % mpi_c_wrapper, simple=False) - if exit_code == 0: + res == run_shell_cmd("%s -compile-info" % mpi_c_wrapper) + compile_info = res.output + if res.exit_code == 0: self.mpi_c_compiler = compile_info.split(' ', 1)[0] else: raise EasyBuildError("Could not determine C compiler underneath Intel MPI, '%s -compiler-info' " diff --git a/easybuild/easyblocks/generic/tarball.py b/easybuild/easyblocks/generic/tarball.py index 5ad887cc81..0d656cc2e8 100644 --- a/easybuild/easyblocks/generic/tarball.py +++ b/easybuild/easyblocks/generic/tarball.py @@ -41,7 +41,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_dir, extract_file, remove_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class Tarball(ExtensionEasyBlock): @@ -94,7 +94,7 @@ def install_step(self, src=None): preinstall_cmd = '&& '.join([cmd for cmd in [preinstall_cmd, self.cfg['preinstall_cmd']] if cmd]) if preinstall_cmd: self.log.info("Preparing installation of %s using command '%s'..." % (self.name, preinstall_cmd)) - run_cmd(preinstall_cmd, log_all=True, simple=True) + run_shell_cmd(preinstall_cmd) # Copy source directory source_path = src or self.cfg['start_dir'] diff --git a/easybuild/easyblocks/generic/versionindependentpythonpackage.py b/easybuild/easyblocks/generic/versionindependentpythonpackage.py index f6cc181b41..6c45efe7a8 100644 --- a/easybuild/easyblocks/generic/versionindependentpythonpackage.py +++ b/easybuild/easyblocks/generic/versionindependentpythonpackage.py @@ -37,7 +37,7 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class VersionIndependentPythonPackage(PythonPackage): @@ -79,7 +79,7 @@ def install_step(self): raise EasyBuildError("%s easyblock is not compatible with using easy_install or pip", eb_name) cmd = self.compose_install_command(self.installdir) - run_cmd(cmd, log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd) # setuptools stubbornly replaces the shebang line in scripts with # the full path to the Python interpreter used to install; diff --git a/easybuild/easyblocks/generic/waf.py b/easybuild/easyblocks/generic/waf.py index 30d36a72dd..6ce9541adc 100644 --- a/easybuild/easyblocks/generic/waf.py +++ b/easybuild/easyblocks/generic/waf.py @@ -29,7 +29,7 @@ """ from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class Waf(EasyBlock): @@ -48,9 +48,9 @@ def configure_step(self, cmd_prefix=''): '--prefix=%s' % self.installdir, self.cfg['configopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def build_step(self, verbose=False, path=None): """ @@ -62,9 +62,9 @@ def build_step(self, verbose=False, path=None): 'build', self.cfg['buildopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def install_step(self, verbose=False, path=None): """ @@ -76,6 +76,6 @@ def install_step(self, verbose=False, path=None): 'install', self.cfg['installopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output