From f47ea2732fe8cd6dfcac6dad1a6e0f077193c237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Thu, 23 May 2024 15:17:11 +0200 Subject: [PATCH 1/9] Use new prefer_ebpythonprefixes --- easybuild/easyblocks/generic/pythonbundle.py | 5 ++++- easybuild/easyblocks/generic/pythonpackage.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index f987cce830..e25010caf9 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -34,6 +34,7 @@ from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option from easybuild.tools.filetools import which from easybuild.tools.modules import get_software_root import easybuild.tools.environment as env @@ -147,7 +148,9 @@ def make_module_extra(self, *args, **kwargs): # update $EBPYTHONPREFIXES rather than $PYTHONPATH # if this Python package was installed for multiple Python versions - if self.multi_python: + # or if we prefer it + prefer_ebpythonprefixes = build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes'] + if self.multi_python or prefer_ebpythonprefixes: txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') else: diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 96e6e0c68e..3101049c11 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -1163,7 +1163,9 @@ def make_module_extra(self, *args, **kwargs): # update $EBPYTHONPREFIXES rather than $PYTHONPATH # if this Python package was installed for multiple Python versions - if self.multi_python: + # or if we prefer it + prefer_ebpythonprefixes = build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes'] + if self.multi_python or prefer_ebpythonprefixes: txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') elif self.require_python: self.set_pylibdirs() From 51b6b96398e23a975b376cac8d6726acdc542129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sat, 6 Jul 2024 22:01:40 +0200 Subject: [PATCH 2/9] Remove superfluous (and incorrect) pythonpath code Some of them looked incorrect as they never actually support EBPYTHONPREFIXES and multi_deps. Relying on the default framework implement instead. --- easybuild/easyblocks/a/amber.py | 2 -- easybuild/easyblocks/c/cplex.py | 8 -------- easybuild/easyblocks/e/esmf.py | 9 --------- easybuild/easyblocks/g/gurobi.py | 4 ---- easybuild/easyblocks/l/lammps.py | 13 ------------- easybuild/easyblocks/m/mxnet.py | 6 ------ easybuild/easyblocks/n/neuron.py | 7 ------- easybuild/easyblocks/o/openbabel.py | 8 -------- easybuild/easyblocks/o/opencv.py | 3 --- easybuild/easyblocks/q/qscintilla.py | 11 ----------- easybuild/easyblocks/t/tkinter.py | 7 ------- 11 files changed, 78 deletions(-) diff --git a/easybuild/easyblocks/a/amber.py b/easybuild/easyblocks/a/amber.py index 1f7a6b98fa..d3fce50661 100644 --- a/easybuild/easyblocks/a/amber.py +++ b/easybuild/easyblocks/a/amber.py @@ -381,7 +381,5 @@ def make_module_extra(self): txt = super(EB_Amber, self).make_module_extra() txt += self.module_generator.set_environment('AMBERHOME', self.installdir) - if self.pylibdir: - txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir) return txt diff --git a/easybuild/easyblocks/c/cplex.py b/easybuild/easyblocks/c/cplex.py index 162207499f..8db513a65c 100644 --- a/easybuild/easyblocks/c/cplex.py +++ b/easybuild/easyblocks/c/cplex.py @@ -38,8 +38,6 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.binary import Binary -from easybuild.easyblocks.generic.pythonpackage import det_pylibdir -from easybuild.easyblocks.python import EBPYTHONPREFIXES from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir from easybuild.tools.modules import get_software_root @@ -150,12 +148,6 @@ def make_module_extra(self): txt += self.module_generator.set_environment('CPLEX_HOME', os.path.join(self.installdir, 'cplex')) txt += self.module_generator.set_environment('CPLEXDIR', os.path.join(self.installdir, 'cplex')) - if self.with_python: - if self.multi_python: - txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') - else: - txt += self.module_generator.prepend_paths('PYTHONPATH', [det_pylibdir()]) - self.log.debug("make_module_extra added %s" % txt) return txt diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index fc73c86d8e..564a39addc 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -156,15 +156,6 @@ def make_module_extra(self): esmf_mkfile_path = os.path.join(self.installdir, "lib", "esmf.mk") txt += self.module_generator.set_environment('ESMFMKFILE', esmf_mkfile_path) - if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: - txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '') - else: - python = get_software_version('Python') - if python: - pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) - pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages') - txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath]) - return txt def sanity_check_step(self): diff --git a/easybuild/easyblocks/g/gurobi.py b/easybuild/easyblocks/g/gurobi.py index b5d0988e6b..a95c68eff6 100644 --- a/easybuild/easyblocks/g/gurobi.py +++ b/easybuild/easyblocks/g/gurobi.py @@ -101,10 +101,6 @@ def make_module_extra(self): txt = super(EB_Gurobi, self).make_module_extra() txt += self.module_generator.set_environment('GUROBI_HOME', self.installdir) txt += self.module_generator.set_environment('GRB_LICENSE_FILE', self.license_file) - - if get_software_root('Python'): - txt += self.module_generator.prepend_paths('PYTHONPATH', det_pylibdir()) - txt += self.module_generator.prepend_paths('MATLABPATH', 'matlab') return txt diff --git a/easybuild/easyblocks/l/lammps.py b/easybuild/easyblocks/l/lammps.py index e80f2259cd..36b5539d73 100644 --- a/easybuild/easyblocks/l/lammps.py +++ b/easybuild/easyblocks/l/lammps.py @@ -548,19 +548,6 @@ def sanity_check_step(self, *args, **kwargs): return super(EB_LAMMPS, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) - def make_module_extra(self): - """Add install path to PYTHONPATH""" - - txt = super(EB_LAMMPS, self).make_module_extra() - - python = get_software_version('Python') - if python: - pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) - pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages') - txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath]) - - return txt - def get_cuda_gpu_arch(cuda_cc): """Return CUDA gpu ARCH in LAMMPS required format. Example: 'sm_32' """ diff --git a/easybuild/easyblocks/m/mxnet.py b/easybuild/easyblocks/m/mxnet.py index 5b78c1e05b..8ccee2553c 100644 --- a/easybuild/easyblocks/m/mxnet.py +++ b/easybuild/easyblocks/m/mxnet.py @@ -252,12 +252,6 @@ def make_module_extra(self, *args, **kwargs): """Custom variables for MXNet module.""" txt = super(EB_MXNet, self).make_module_extra(*args, **kwargs) - for path in self.py_ext.all_pylibdirs: - fullpath = os.path.join(self.installdir, path) - # only extend $PYTHONPATH with existing, non-empty directories - if os.path.exists(fullpath) and os.listdir(fullpath): - txt += self.module_generator.prepend_paths('PYTHONPATH', path) - txt += self.module_generator.prepend_paths("R_LIBS", ['']) # prepend R_LIBS with install path return txt diff --git a/easybuild/easyblocks/n/neuron.py b/easybuild/easyblocks/n/neuron.py index 03bbc2f24d..fcdd0aeb68 100644 --- a/easybuild/easyblocks/n/neuron.py +++ b/easybuild/easyblocks/n/neuron.py @@ -283,11 +283,4 @@ def make_module_extra(self): else: self.log.debug("%s not set: %s" % (var, os.environ.get(var, None))) - if self.with_python: - if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: - txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '') - else: - txt += self.module_generator.prepend_paths('PYTHONPATH', [self.pylibdir]) - # also adds lib/python to PYTHONPATH - txt += self.module_generator.prepend_paths('PYTHONPATH', ['lib/python']) return txt diff --git a/easybuild/easyblocks/o/openbabel.py b/easybuild/easyblocks/o/openbabel.py index 420d54c9ce..22f795f77d 100644 --- a/easybuild/easyblocks/o/openbabel.py +++ b/easybuild/easyblocks/o/openbabel.py @@ -108,14 +108,6 @@ def sanity_check_step(self): def make_module_extra(self): """Custom variables for OpenBabel module.""" txt = super(EB_OpenBabel, self).make_module_extra() - if self.with_python: - if LooseVersion(self.version) >= LooseVersion('2.4'): - # since OpenBabel 2.4.0 the Python bindings under - # ${PREFIX}/lib/python2.7/site-packages rather than ${PREFIX}/lib - ob_pythonpath = det_pylibdir() - else: - ob_pythonpath = 'lib' - txt += self.module_generator.prepend_paths('PYTHONPATH', [ob_pythonpath]) babel_libdir = os.path.join(self.installdir, 'lib', 'openbabel', self.version) txt += self.module_generator.set_environment('BABEL_LIBDIR', babel_libdir) babel_datadir = os.path.join(self.installdir, 'share', 'openbabel', self.version) diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index b777f3bec3..7568497311 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -248,7 +248,4 @@ def make_module_extra(self): txt += self.module_generator.prepend_paths('CLASSPATH', os.path.join('share', 'OpenCV', 'java')) - if os.path.exists(os.path.join(self.installdir, self.pylibdir)): - txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir) - return txt diff --git a/easybuild/easyblocks/q/qscintilla.py b/easybuild/easyblocks/q/qscintilla.py index 9ac8c1c481..6c97ae8670 100644 --- a/easybuild/easyblocks/q/qscintilla.py +++ b/easybuild/easyblocks/q/qscintilla.py @@ -200,14 +200,3 @@ def sanity_check_step(self): custom_commands.append("python -c 'import %s.Qsci'" % self.pyqt_pkg_name) super(EB_QScintilla, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - - def make_module_extra(self): - """Custom extra module file entries for QScintilla.""" - txt = super(EB_QScintilla, self).make_module_extra() - python = get_software_root('Python') - if python: - if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: - txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '') - else: - txt += self.module_generator.prepend_paths('PYTHONPATH', [det_pylibdir()]) - return txt diff --git a/easybuild/easyblocks/t/tkinter.py b/easybuild/easyblocks/t/tkinter.py index 3588836d5d..bc4f50d3ac 100644 --- a/easybuild/easyblocks/t/tkinter.py +++ b/easybuild/easyblocks/t/tkinter.py @@ -134,10 +134,3 @@ def sanity_check_step(self): 'dirs': ['lib'] } super(EB_Python, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) - - def make_module_extra(self): - """Set PYTHONPATH""" - txt = super(EB_Tkinter, self).make_module_extra() - txt += self.module_generator.prepend_paths('PYTHONPATH', det_pylibdir()) - - return txt From c76921dc4a01cf5001004a11c329bf99afc9500a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 7 Jul 2024 02:40:40 +0200 Subject: [PATCH 3/9] Add checks for existing python paths in PythonBundle --- easybuild/easyblocks/generic/pythonbundle.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index e25010caf9..838e977143 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -151,7 +151,8 @@ def make_module_extra(self, *args, **kwargs): # or if we prefer it prefer_ebpythonprefixes = build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes'] if self.multi_python or prefer_ebpythonprefixes: - txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') + if EBPYTHONPREFIXES not in self.module_generator.added_paths_per_key: + txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') else: # the temporary module file that is generated before installing extensions @@ -166,7 +167,8 @@ def make_module_extra(self, *args, **kwargs): ] for pylibdir in new_pylibdirs: - txt += self.module_generator.prepend_paths('PYTHONPATH', pylibdir) + if pylibdir not in self.module_generator.added_paths_per_key['PYTHONPATH']: + txt += self.module_generator.prepend_paths('PYTHONPATH', pylibdir) return txt From 962e341fc19f7e483e72f2204f8962de1ececc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Thu, 15 Aug 2024 19:14:58 +0200 Subject: [PATCH 4/9] Only use EBPYTHONPREFIXES when Python is used. --- easybuild/easyblocks/generic/pythonbundle.py | 5 +++-- easybuild/easyblocks/generic/pythonpackage.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 838e977143..20a6226a78 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -149,8 +149,9 @@ def make_module_extra(self, *args, **kwargs): # update $EBPYTHONPREFIXES rather than $PYTHONPATH # if this Python package was installed for multiple Python versions # or if we prefer it - prefer_ebpythonprefixes = build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes'] - if self.multi_python or prefer_ebpythonprefixes: + use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \ + self.cfg['prefer_ebpythonprefixes'] + if self.multi_python or use_ebpythonprefixes: if EBPYTHONPREFIXES not in self.module_generator.added_paths_per_key: txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') else: diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 3101049c11..854b719f79 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -1164,8 +1164,10 @@ def make_module_extra(self, *args, **kwargs): # update $EBPYTHONPREFIXES rather than $PYTHONPATH # if this Python package was installed for multiple Python versions # or if we prefer it - prefer_ebpythonprefixes = build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes'] - if self.multi_python or prefer_ebpythonprefixes: + + use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \ + self.cfg['prefer_ebpythonprefixes'] + if self.multi_python or use_ebpythonprefixes: txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') elif self.require_python: self.set_pylibdirs() From 85d5d6bc2b8ff95966513a48103a36dee2c250fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Thu, 15 Aug 2024 20:32:45 +0200 Subject: [PATCH 5/9] Add more reliable way to detect Python dep --- easybuild/easyblocks/generic/pythonbundle.py | 5 +++-- easybuild/easyblocks/generic/pythonpackage.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 20a6226a78..d6fc5958b1 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -149,8 +149,9 @@ def make_module_extra(self, *args, **kwargs): # update $EBPYTHONPREFIXES rather than $PYTHONPATH # if this Python package was installed for multiple Python versions # or if we prefer it - use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \ - self.cfg['prefer_ebpythonprefixes'] + runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] + use_ebpythonprefixes = 'Python' in runtime_deps and \ + build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes'] if self.multi_python or use_ebpythonprefixes: if EBPYTHONPREFIXES not in self.module_generator.added_paths_per_key: txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 854b719f79..20bbaa21ed 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -1164,9 +1164,9 @@ def make_module_extra(self, *args, **kwargs): # update $EBPYTHONPREFIXES rather than $PYTHONPATH # if this Python package was installed for multiple Python versions # or if we prefer it - - use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \ - self.cfg['prefer_ebpythonprefixes'] + runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] + use_ebpythonprefixes = 'Python' in runtime_deps and \ + build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes'] if self.multi_python or use_ebpythonprefixes: txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') elif self.require_python: From 9ff6585bc55d87c284770b4b639f2ee016045c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Fri, 20 Sep 2024 16:37:37 +0200 Subject: [PATCH 6/9] Use new --prefer-python-search-path --- easybuild/easyblocks/generic/pythonbundle.py | 15 ++++++++------- easybuild/easyblocks/generic/pythonpackage.py | 11 ++++++----- easybuild/easyblocks/p/python.py | 6 ++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index d6fc5958b1..18a0fb7081 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -31,10 +31,10 @@ import sys from easybuild.easyblocks.generic.bundle import Bundle -from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES +from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_PYTHON_PACKAGES from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import build_option +from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES from easybuild.tools.filetools import which from easybuild.tools.modules import get_software_root import easybuild.tools.environment as env @@ -151,10 +151,11 @@ def make_module_extra(self, *args, **kwargs): # or if we prefer it runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] use_ebpythonprefixes = 'Python' in runtime_deps and \ - build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes'] + build_option('prefer_python_search_path') == EBPYTHONPREFIXES and not self.cfg['force_pythonpath'] if self.multi_python or use_ebpythonprefixes: - if EBPYTHONPREFIXES not in self.module_generator.added_paths_per_key: - txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') + path = '' # EBPYTHONPREFIXES are relative to the install dir + if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]: + txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, path) else: # the temporary module file that is generated before installing extensions @@ -169,8 +170,8 @@ def make_module_extra(self, *args, **kwargs): ] for pylibdir in new_pylibdirs: - if pylibdir not in self.module_generator.added_paths_per_key['PYTHONPATH']: - txt += self.module_generator.prepend_paths('PYTHONPATH', pylibdir) + if pylibdir not in self.module_generator.added_paths_per_key[PYTHONPATH]: + txt += self.module_generator.prepend_paths(PYTHONPATH, pylibdir) return txt diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 20bbaa21ed..916f8545f8 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -42,13 +42,13 @@ import easybuild.tools.environment as env from easybuild.base import fancylogger -from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES +from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.default import DEFAULT_CONFIG from easybuild.framework.easyconfig.templates import PYPI_SOURCE from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.config import build_option +from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES from easybuild.tools.filetools import change_dir, mkdir, remove_dir, symlink, which from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_shell_cmd, subprocess_popen_text @@ -1166,15 +1166,16 @@ def make_module_extra(self, *args, **kwargs): # or if we prefer it runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] use_ebpythonprefixes = 'Python' in runtime_deps and \ - build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes'] + build_option('prefer_python_search_path') == EBPYTHONPREFIXES and not self.cfg['force_pythonpath'] if self.multi_python or use_ebpythonprefixes: - txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') + path = '' # EBPYTHONPREFIXES are relative to the install dir + txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, path) elif self.require_python: self.set_pylibdirs() for path in self.all_pylibdirs: fullpath = os.path.join(self.installdir, path) # only extend $PYTHONPATH with existing, non-empty directories if os.path.exists(fullpath) and os.listdir(fullpath): - txt += self.module_generator.prepend_paths('PYTHONPATH', path) + txt += self.module_generator.prepend_paths(PYTHONPATH, path) return super(PythonPackage, self).make_module_extra(txt, *args, **kwargs) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index b450d74c79..01355b23e5 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -45,7 +45,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.templates import PYPI_SOURCE from easybuild.tools.build_log import EasyBuildError, print_warning -from easybuild.tools.config import build_option, ERROR, log_path +from easybuild.tools.config import build_option, ERROR, log_path, PYTHONPATH, EBPYTHONPREFIXES from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, symlink, write_file @@ -59,8 +59,6 @@ # magic value for unlimited stack size UNLIMITED = 'unlimited' -EBPYTHONPREFIXES = 'EBPYTHONPREFIXES' - # We want the following import order: # 1. Packages installed into VirtualEnv # 2. Packages installed into $EBPYTHONPREFIXES (e.g. our modules) @@ -643,6 +641,6 @@ def make_module_extra(self, *args, **kwargs): txt = super(EB_Python, self).make_module_extra() if self.cfg.get('ebpythonprefixes'): - txt += self.module_generator.prepend_paths('PYTHONPATH', self.pythonpath) + txt += self.module_generator.prepend_paths(PYTHONPATH, self.pythonpath) return txt From e4f8a31377d74aa7c8bc1dec72e566051d6ab794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Fri, 20 Sep 2024 17:28:27 +0200 Subject: [PATCH 7/9] Drop now unused imports --- easybuild/easyblocks/g/gurobi.py | 1 - easybuild/easyblocks/o/openbabel.py | 1 - 2 files changed, 2 deletions(-) diff --git a/easybuild/easyblocks/g/gurobi.py b/easybuild/easyblocks/g/gurobi.py index a95c68eff6..05a80ca0fb 100644 --- a/easybuild/easyblocks/g/gurobi.py +++ b/easybuild/easyblocks/g/gurobi.py @@ -31,7 +31,6 @@ """ import os -from easybuild.easyblocks.generic.pythonpackage import det_pylibdir from easybuild.easyblocks.generic.tarball import Tarball from easybuild.framework.easyconfig import CUSTOM from easybuild.tools import LooseVersion diff --git a/easybuild/easyblocks/o/openbabel.py b/easybuild/easyblocks/o/openbabel.py index 22f795f77d..a03d8c2a66 100644 --- a/easybuild/easyblocks/o/openbabel.py +++ b/easybuild/easyblocks/o/openbabel.py @@ -31,7 +31,6 @@ import glob import os from easybuild.easyblocks.generic.cmakemake import CMakeMake -from easybuild.easyblocks.generic.pythonpackage import det_pylibdir from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version From bc1d2b13fbc316be6f60d39855cf4010f5b024ba Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Sep 2024 16:19:28 +0200 Subject: [PATCH 8/9] don't use force_pythonpath + clarify why we're keeping logic on $PYTHONPATH vs $EBPYTHONPREFIXES in PythonBundle + PythonPackage easyblocks --- easybuild/easyblocks/generic/pythonbundle.py | 13 +++++++++---- easybuild/easyblocks/generic/pythonpackage.py | 15 +++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 18a0fb7081..76729c4105 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -147,11 +147,16 @@ def make_module_extra(self, *args, **kwargs): txt = super(Bundle, self).make_module_extra(*args, **kwargs) # update $EBPYTHONPREFIXES rather than $PYTHONPATH - # if this Python package was installed for multiple Python versions - # or if we prefer it + # if this Python package was installed for multiple Python versions, or if we prefer it + use_ebpythonprefixes = False runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] - use_ebpythonprefixes = 'Python' in runtime_deps and \ - build_option('prefer_python_search_path') == EBPYTHONPREFIXES and not self.cfg['force_pythonpath'] + + if 'Python' in runtime_deps: + self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...") + if build_option('prefer_python_search_path') == EBPYTHONPREFIXES: + self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that") + use_ebpythonprefixes = True + if self.multi_python or use_ebpythonprefixes: path = '' # EBPYTHONPREFIXES are relative to the install dir if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]: diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 916f8545f8..1b9a8ea517 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -1162,11 +1162,18 @@ def make_module_extra(self, *args, **kwargs): txt = '' # update $EBPYTHONPREFIXES rather than $PYTHONPATH - # if this Python package was installed for multiple Python versions - # or if we prefer it + # if this Python package was installed for multiple Python versions, or if we prefer it; + # note: although EasyBuild framework also has logic for this in EasyBlock.make_module_extra, + # we retain full control here, since the logic is slightly different + use_ebpythonprefixes = False runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] - use_ebpythonprefixes = 'Python' in runtime_deps and \ - build_option('prefer_python_search_path') == EBPYTHONPREFIXES and not self.cfg['force_pythonpath'] + + if 'Python' in runtime_deps: + self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...") + if build_option('prefer_python_search_path') == EBPYTHONPREFIXES: + self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that") + use_ebpythonprefixes = True + if self.multi_python or use_ebpythonprefixes: path = '' # EBPYTHONPREFIXES are relative to the install dir txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, path) From cb728d4eeb85d9c431248813fa46d4e50e34392c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Oct 2024 19:24:41 +0200 Subject: [PATCH 9/9] fix docstring for make_module_extra method in ESMF easyblock --- easybuild/easyblocks/e/esmf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index 564a39addc..a02e69bb99 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -148,7 +148,7 @@ def install_step(self): run_shell_cmd(cmd) def make_module_extra(self): - """Add install path to PYTHONPATH or EBPYTHONPREFIXES""" + """Set $ESMFMKFILE environment variable""" txt = super(EB_ESMF, self).make_module_extra() # set environment variable ESMFMKFILE