Merge pull request #3046 from bartoldeman/generic-run-cmd

Convert generic easyblocks to `run_shell_cmd`

easybuild/easyblocks/generic/binary.py

@@ -40,7 +40,7 @@
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, remove_dir
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 
 
 PREPEND_TO_PATH_DEFAULT = ['']
@@ -126,7 +126,7 @@ def install_step(self):
                 for install_cmd in install_cmds:
                     cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']])
                     self.log.info("Running install command for %s: '%s'..." % (self.name, cmd))
-                    run_cmd(cmd, log_all=True, simple=True)
+                    run_shell_cmd(cmd)
             else:
                 raise EasyBuildError("Incorrect value type for install_cmds, should be list or tuple: ",
                                      install_cmds)

easybuild/easyblocks/generic/cargo.py

@@ -36,7 +36,7 @@
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
 from easybuild.tools.filetools import extract_file, change_dir
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import write_file, compute_checksum
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
@@ -199,7 +199,7 @@ def build_step(self):
         if self.cfg['lto'] is not None:
             lto = '--config profile.%s.lto=\\"%s\\"' % (self.profile, self.cfg['lto'])
 
-        run_cmd('rustc --print cfg', log_all=True, simple=True)  # for tracking in log file
+        run_shell_cmd('rustc --print cfg')  # for tracking in log file
         cmd = ' '.join([
             self.cfg['prebuildopts'],
             'cargo build',
@@ -209,7 +209,7 @@ def build_step(self):
             parallel,
             self.cfg['buildopts'],
         ])
-        run_cmd(cmd, log_all=True, simple=True)
+        run_shell_cmd(cmd)
 
     def test_step(self):
         """Test with cargo"""
@@ -220,7 +220,7 @@ def test_step(self):
                 '--profile=' + self.profile,
                 self.cfg['testopts'],
             ])
-            run_cmd(cmd, log_all=True, simple=True)
+            run_shell_cmd(cmd)
 
     def install_step(self):
         """Install with cargo"""
@@ -232,7 +232,7 @@ def install_step(self):
             '--path=.',
             self.cfg['installopts'],
         ])
-        run_cmd(cmd, log_all=True, simple=True)
+        run_shell_cmd(cmd)
 
 
 def generate_crate_list(sourcedir):

easybuild/easyblocks/generic/cmakemake.py

@@ -45,7 +45,7 @@
 from easybuild.tools.filetools import change_dir, create_unused_dir, mkdir, which
 from easybuild.tools.environment import setvar
 from easybuild.tools.modules import get_software_root, get_software_version
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.systemtools import get_shared_lib_ext
 from easybuild.tools.utilities import nub
 
@@ -63,7 +63,8 @@ def det_cmake_version():
         regex = re.compile(r"^[cC][mM]ake version (?P<version>[0-9]\.[0-9a-zA-Z.-]+)$", re.M)
 
         cmd = "cmake --version"
-        (out, _) = run_cmd(cmd, simple=False, log_ok=False, log_all=False, trace=False)
+        cmd_res = run_shell_cmd(cmd, hidden=True, fail_on_error=False)
+        out = cmd_res.output
         res = regex.search(out)
         if res:
             cmake_version = res.group('version')
@@ -311,9 +312,9 @@ def configure_step(self, srcdir=None, builddir=None):
                 self.cfg.get('configure_cmd'),
                 self.cfg['configopts']])
 
-        (out, _) = run_cmd(command, log_all=True, simple=False)
+        res = run_shell_cmd(command)
 
-        return out
+        return res.output
 
     def test_step(self):
         """CMake specific test setup"""

easybuild/easyblocks/generic/cmdcp.py

@@ -32,7 +32,7 @@
 from easybuild.easyblocks.generic.makecp import MakeCp
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 
 
 class CmdCp(MakeCp):
@@ -80,4 +80,4 @@ def build_step(self):
                 raise EasyBuildError("No match for %s in %s, don't know which command to use.",
                                      src, self.cfg['cmds_map'])
 
-            run_cmd(cmd, log_all=True, simple=True)
+            run_shell_cmd(cmd)

easybuild/easyblocks/generic/conda.py

@@ -33,7 +33,7 @@
 
 from easybuild.easyblocks.generic.binary import Binary
 from easybuild.framework.easyconfig import CUSTOM
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.modules import get_software_root
 from easybuild.tools.build_log import EasyBuildError
 
@@ -74,7 +74,7 @@ def install_step(self):
         # initialize conda environment
         # setuptools is just a choice, but *something* needs to be there
         cmd = "%s config --add create_default_packages setuptools" % conda_cmd
-        run_cmd(cmd, log_all=True, simple=True)
+        run_shell_cmd(cmd)
 
         if self.cfg['environment_file'] or self.cfg['remote_environment']:
 
@@ -86,7 +86,7 @@ def install_step(self):
             # use --force to ignore existing installation directory
             cmd = "%s %s env create --force %s -p %s" % (self.cfg['preinstallopts'], conda_cmd,
                                                          env_spec, self.installdir)
-            run_cmd(cmd, log_all=True, simple=True)
+            run_shell_cmd(cmd)
 
         else:
 
@@ -100,11 +100,11 @@ def install_step(self):
 
             cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd,
                                                         self.installdir, install_args)
-            run_cmd(cmd, log_all=True, simple=True)
+            run_shell_cmd(cmd)
 
         # clean up
         cmd = "%s clean -ya" % conda_cmd
-        run_cmd(cmd, log_all=True, simple=True)
+        run_shell_cmd(cmd)
 
     def make_module_extra(self):
         """Add the install directory to the PATH."""

easybuild/easyblocks/generic/configuremake.py

@@ -49,7 +49,7 @@
 from easybuild.tools.config import source_paths, build_option
 from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file
 from easybuild.tools.filetools import read_file, remove_file
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 
 # string that indicates that a configure script was generated by Autoconf
 # note: bytes string since this constant is used to check the contents of 'configure' which is read as bytes
@@ -247,8 +247,8 @@ def determine_build_and_host_type(self):
                               "EasyBuild attempts to download a recent config.guess but seems to have failed!")
             else:
                 self.check_config_guess()
-                system_type, _ = run_cmd(self.config_guess, log_all=True)
-                system_type = system_type.strip()
+                res = run_shell_cmd(self.config_guess)
+                system_type = res.output.strip()
                 self.log.info("%s returned a system type '%s'", self.config_guess, system_type)
 
                 if build_type is None:
@@ -323,16 +323,19 @@ def configure_step(self, cmd_prefix=''):
             ] + build_and_host_options + [self.cfg['configopts']]
         )
 
-        (out, _) = run_cmd(cmd, log_all=True, simple=False)
+        res = run_shell_cmd(cmd)
 
-        return out
+        return res.output
 
-    def build_step(self, verbose=False, path=None):
+    def build_step(self, verbose=None, path=None):
         """
         Start the actual build
         - typical: make -j X
         """
 
+        if verbose is not None:
+            self.log.deprecated("The 'verbose' parameter to build_step is deprecated and unneeded.", '6.0')
+
         paracmd = ''
         if self.cfg['parallel']:
             paracmd = "-j %s" % self.cfg['parallel']
@@ -351,7 +354,8 @@ def build_step(self, verbose=False, path=None):
             ])
             self.log.info("Building target '%s'", target)
 
-            (out, _) = run_cmd(cmd, path=path, log_all=True, simple=False, log_output=verbose)
+            res = run_shell_cmd(cmd, work_dir=path)
+            out = res.output
 
         return out
 
@@ -369,9 +373,9 @@ def test_step(self):
                 runtest = ''
             # Compose command filtering out empty values
             cmd = ' '.join([x for x in (self.cfg['pretestopts'], test_cmd, runtest, self.cfg['testopts']) if x])
-            (out, _) = run_cmd(cmd, log_all=True, simple=False)
+            res = run_shell_cmd(cmd)
 
-            return out
+            return res.output
 
     def install_step(self):
         """
@@ -385,6 +389,6 @@ def install_step(self):
             self.cfg['installopts'],
         ])
 
-        (out, _) = run_cmd(cmd, log_all=True, simple=False)
+        res = run_shell_cmd(cmd)
 
-        return out
+        return res.output

easybuild/easyblocks/generic/configuremakepythonpackage.py

@@ -31,7 +31,7 @@
 """
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
 from easybuild.easyblocks.generic.pythonpackage import PythonPackage
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 
 
 class ConfigureMakePythonPackage(ConfigureMake, PythonPackage):
@@ -58,7 +58,7 @@ def configure_step(self, *args, **kwargs):
         """Configure build using ``python configure``."""
         PythonPackage.configure_step(self, *args, **kwargs)
         cmd = ' '.join([self.cfg['preconfigopts'], self.python_cmd, self.cfg['configopts']])
-        run_cmd(cmd, log_all=True)
+        run_shell_cmd(cmd)
 
     def build_step(self, *args, **kwargs):
         """Build Python package with ``make``."""

easybuild/easyblocks/generic/fortranpythonpackage.py

@@ -36,7 +36,7 @@
 import easybuild.tools.toolchain as toolchain
 from easybuild.easyblocks.generic.pythonpackage import PythonPackage
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 
 
 class FortranPythonPackage(PythonPackage):
@@ -70,4 +70,4 @@ def build_step(self):
             raise EasyBuildError("Unknown family of compilers being used: %s", comp_fam)
 
         cmd = "%s %s setup.py build %s" % (self.cfg['prebuildopts'], self.python_cmd, self.cfg['buildopts'])
-        run_cmd(cmd, log_all=True, simple=True)
+        run_shell_cmd(cmd)

easybuild/easyblocks/generic/gopackage.py

@@ -35,7 +35,7 @@
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.modules import get_software_root, get_software_version
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 
 
 class GoPackage(EasyBlock):
@@ -71,8 +71,8 @@ def configure_step(self):
         env.setvar('GOBIN', os.path.join(self.installdir, 'bin'), verbose=False)
 
         # creates log entries for go being used, for debugging
-        run_cmd("go version", verbose=False, trace=False)
-        run_cmd("go env", verbose=False, trace=False)
+        run_shell_cmd("go version", hidden=True)
+        run_shell_cmd("go env", hidden=True)
 
     def build_step(self):
         """If Go package is not native go module, lets try to make the module."""
@@ -91,34 +91,34 @@ def build_step(self):
 
             # go mod init
             cmd = ' '.join(['go', 'mod', 'init', self.cfg['modulename']])
-            run_cmd(cmd, log_all=True, simple=True)
+            run_shell_cmd(cmd)
 
             if self.cfg['forced_deps']:
                 for dep in self.cfg['forced_deps']:
                     # go get specific dependencies which locks them in go.mod
                     cmd = ' '.join(['go', 'get', '%s@%s' % dep])
-                    run_cmd(cmd, log_all=True, simple=True)
+                    run_shell_cmd(cmd)
 
             # note: ... (tripledot) used below is not a typo, but go wildcard pattern
             # which means: anything you can find in this directory, including all subdirectories
             # see: 'go help packages' or https://golang.org/pkg/cmd/go/internal/help/
             # see: https://stackoverflow.com/a/28031651/2047157
 
             # building and testing will add packages to go.mod
-            run_cmd('go build ./...', log_all=True, simple=True)
-            run_cmd('go test ./...', log_all=True, simple=True)
+            run_shell_cmd('go build ./...')
+            run_shell_cmd('go test ./...')
 
             # tidy up go.mod
-            run_cmd('go mod tidy', log_all=True, simple=True)
+            run_shell_cmd('go mod tidy')
 
             # build and test again, to ensure go mod tidy didn't removed anything needed
-            run_cmd('go build ./...', log_all=True, simple=True)
-            run_cmd('go test ./...', log_all=True, simple=True)
+            run_shell_cmd('go build ./...')
+            run_shell_cmd('go test ./...')
 
             self.log.warn('Include generated go.mod and go.sum via patch to ensure locked dependencies '
                           'and run this easyconfig again.')
-            run_cmd('cat go.mod', log_all=True, simple=True)
-            run_cmd('cat go.sum', log_all=True, simple=True)
+            run_shell_cmd('cat go.mod')
+            run_shell_cmd('cat go.sum')
 
         if not os.path.exists(go_sum_file) or not os.path.isfile(go_sum_file):
             raise EasyBuildError("go.sum not found! This module has no locked dependency versions.")
@@ -136,7 +136,7 @@ def install_step(self):
             '-x',
             self.cfg['installopts'],
         ])
-        run_cmd(cmd, log_all=True, log_ok=True, simple=True)
+        run_shell_cmd(cmd)
 
     def sanity_check_step(self):
         """Custom sanity check for Go package."""

easybuild/easyblocks/generic/intelbase.py

@@ -49,7 +49,7 @@
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import adjust_permissions, find_flexlm_license
 from easybuild.tools.filetools import mkdir, read_file, remove_file, write_file
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 
 
 # different supported activation types (cfr. Intel documentation)
@@ -428,7 +428,7 @@ def install_step_classic(self, silent_cfg_names_map=None, silent_cfg_extras=None
             self.cfg['installopts'],
         ])
 
-        return run_cmd(cmd, log_all=True, simple=True, log_output=True)
+        run_shell_cmd(cmd)
 
     def install_step_oneapi(self, *args, **kwargs):
         """
@@ -468,16 +468,16 @@ def install_step_oneapi(self, *args, **kwargs):
 
         cmd.append(self.cfg['installopts'])
 
-        return run_cmd(' '.join(cmd), log_all=True, simple=True, log_output=True)
+        run_shell_cmd(' '.join(cmd))
 
     def install_step(self, *args, **kwargs):
         """
         Install Intel software
         """
         if LooseVersion(self.version) >= LooseVersion('2021'):
-            return self.install_step_oneapi(*args, **kwargs)
+            self.install_step_oneapi(*args, **kwargs)
         else:
-            return self.install_step_classic(*args, **kwargs)
+            self.install_step_classic(*args, **kwargs)
 
     def move_after_install(self):
         """Move installed files to correct location after installation."""

easybuild/easyblocks/generic/juliapackage.py

@@ -38,7 +38,7 @@
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.modules import get_software_root, get_software_version
 from easybuild.tools.filetools import copy_dir
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 
 EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "")
 USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?$")
@@ -147,9 +147,9 @@ def install_step(self):
             "julia -e '%s'" % julia_pkg_cmd,
             self.cfg['installopts'],
         ])
-        (out, _) = run_cmd(cmd, log_all=True, simple=False)
+        res = run_shell_cmd(cmd)
 
-        return out
+        return res.output
 
     def run(self):
         """Install Julia package as an extension."""