add option search-path-cpp-headers to control how EasyBuild sets paths to headers at build time

easybuild/toolchains/fcc.py

@@ -40,39 +40,10 @@ class FCC(FujitsuCompiler):
     OPTIONAL = False
 
     # override in order to add an exception for the Fujitsu lang/tcsds module
-    def _add_dependency_variables(self, names=None, cpp=None, ld=None):
-        """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies
-            names should be a list of strings containing the name of the dependency
+    def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
         """
-        cpp_paths = ['include']
-        ld_paths = ['lib64', 'lib']
-
-        if cpp is not None:
-            for p in cpp:
-                if p not in cpp_paths:
-                    cpp_paths.append(p)
-        if ld is not None:
-            for p in ld:
-                if p not in ld_paths:
-                    ld_paths.append(p)
-
-        if not names:
-            deps = self.dependencies
-        else:
-            deps = [{'name': name} for name in names if name is not None]
-
-        # collect software install prefixes for dependencies
-        roots = []
-        for dep in deps:
-            if dep.get('external_module', False):
-                # for software names provided via external modules, install prefix may be unknown
-                names = dep['external_module_metadata'].get('name', [])
-                roots.extend([root for root in self.get_software_root(names) if root is not None])
-            else:
-                roots.extend(self.get_software_root(dep['name']))
-
-        for root in roots:
-            # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode
-            if 'tcsds' not in root:
-                self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths)
-            self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths)
+        Append prepocessor paths for given dependency root directory
+        """
+        # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode
+        if "tcsds" not in dep_root:
+            super()._add_dependency_cpp_headers(dep_root, extra_dirs=extra_dirs)

easybuild/toolchains/linalg/acml.py

@@ -77,9 +77,9 @@ def _set_blas_variables(self):
             for root in self.get_software_root(self.BLAS_MODULE_NAME):
                 subdirs = self.ACML_SUBDIRS_MAP[self.COMPILER_FAMILY]
                 self.BLAS_LIB_DIR = [os.path.join(x, 'lib') for x in subdirs]
-                self.variables.append_exists('LDFLAGS', root, self.BLAS_LIB_DIR, append_all=True)
+                self._add_dependency_linker_paths(root, extra_dirs=self.BLAS_LIB_DIR)
                 incdirs = [os.path.join(x, 'include') for x in subdirs]
-                self.variables.append_exists('CPPFLAGS', root, incdirs, append_all=True)
+                self._add_dependency_cpp_headers(root, extra_dirs=incdirs)
         except Exception:
             raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP "
                                  "with compiler family %s", self.COMPILER_FAMILY)

easybuild/tools/config.py

@@ -206,6 +206,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'backup_modules',
         'banned_linked_shared_libs',
         'checksum_priority',
+        'cpp_headers_search_path',
         'container_config',
         'container_image_format',
         'container_image_name',

easybuild/tools/options.py

@@ -101,6 +101,7 @@
 from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.package.utilities import avail_package_naming_schemes
 from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler
+from easybuild.tools.toolchain.toolchain import CPP_HEADERS_SEARCH_PATHS, DEFAULT_CPP_HEADERS_SEARCH_PATH
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
 from easybuild.tools.repository.repository import avail_repositories
 from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
@@ -375,6 +376,8 @@ def override_options(self):
                                   'choice', 'store_or_None', DEFAULT_CHECKSUM_PRIORITY, CHECKSUM_PRIORITY_CHOICES),
             'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True),
             'cleanup-tmpdir': ("Cleanup tmp dir after successful run.", None, 'store_true', True),
+            'cpp-headers-search-path': ("Search path used by EasyBuild to inject include directories", 'choice',
+                                        'store', DEFAULT_CPP_HEADERS_SEARCH_PATH, [*CPP_HEADERS_SEARCH_PATHS]),
             'color': ("Colorize output", 'choice', 'store', fancylogger.Colorize.AUTO, fancylogger.Colorize,
                       {'metavar': 'WHEN'}),
             'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False),

easybuild/tools/toolchain/compiler.py

@@ -93,6 +93,7 @@ class Compiler(Toolchain):
         'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"),
         'packed-linker-options': (False, "Pack the linker options as comma separated list"),  # ScaLAPACK mainly
         'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"),
+        'cpp-headers-search-path': (None, "Search path used by EasyBuild to inject include directories"),
         'extra_cflags': (None, "Specify extra CFLAGS options."),
         'extra_cxxflags': (None, "Specify extra CXXFLAGS options."),
         'extra_fflags': (None, "Specify extra FFLAGS options."),

easybuild/tools/toolchain/constants.py

@@ -31,9 +31,9 @@
 * Kenneth Hoste (Ghent University)
 """
 
+from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs, FlagList
+from easybuild.tools.toolchain.variables import IncludePaths, LibraryList, LinkLibraryPaths, SearchPaths
 from easybuild.tools.variables import AbsPathList
-from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs
-from easybuild.tools.toolchain.variables import FlagList, IncludePaths, LibraryList, LinkLibraryPaths
 
 
 COMPILER_VARIABLES = [
@@ -65,7 +65,13 @@
         ('LDFLAGS', 'Flags passed to linker'),  # TODO: overridden by command line?
     ],
     IncludePaths: [
-        ('CPPFLAGS', 'Precompiler flags'),
+        ('CPPFLAGS', 'Preprocessor flags'),
+    ],
+    SearchPaths: [
+        ('CPATH', 'Location of C/C++ header files'),
+        ('C_INCLUDE_PATH', 'Location of C header files'),
+        ('CPLUS_INCLUDE_PATH', 'Location of C++ header files'),
+        ('OBJC_INCLUDE_PATH', 'Location of Objective C header files'),
     ],
     CommandFlagList: COMPILER_VARIABLES,
 }

easybuild/tools/toolchain/toolchain.py

@@ -95,6 +95,17 @@
     TOOLCHAIN_CAPABILITY_LAPACK_FAMILY,
     TOOLCHAIN_CAPABILITY_MPI_FAMILY,
 ]
+# modes to handle CPP header search paths
+# see: https://gcc.gnu.org/onlinedocs/cpp/Environment-Variables.html
+CPP_HEADERS_SEARCH_PATH_FLAGS = "CPPFLAGS"
+CPP_HEADERS_SEARCH_PATH_CPATH = "CPATH"
+CPP_HEADERS_SEARCH_PATH_INCLUDE = "INCLUDE_PATHS"
+CPP_HEADERS_SEARCH_PATHS = {
+    CPP_HEADERS_SEARCH_PATH_FLAGS: ["CPPFLAGS"],
+    CPP_HEADERS_SEARCH_PATH_CPATH: ["CPATH"],
+    CPP_HEADERS_SEARCH_PATH_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
+}
+DEFAULT_CPP_HEADERS_SEARCH_PATH = CPP_HEADERS_SEARCH_PATH_FLAGS
 
 
 def is_system_toolchain(tc_name):
@@ -850,7 +861,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
             else:
                 self.log.debug("prepare: set additional variables onlymod=%s", onlymod)
 
-                # add LDFLAGS and CPPFLAGS from dependencies to self.vars
+                # add linker and preprocessor paths to dependencies to self.vars
                 self._add_dependency_variables()
                 self.generate_vars()
                 self._setenv_variables(onlymod, verbose=not silent)
@@ -1049,39 +1060,71 @@ def handle_sysroot(self):
                 setvar('PKG_CONFIG_PATH', os.pathsep.join(pkg_config_path))
 
     def _add_dependency_variables(self, names=None, cpp=None, ld=None):
-        """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies
-            names should be a list of strings containing the name of the dependency
         """
-        cpp_paths = ['include']
-        ld_paths = ['lib64', 'lib']
-
-        if cpp is not None:
-            for p in cpp:
-                if p not in cpp_paths:
-                    cpp_paths.append(p)
-        if ld is not None:
-            for p in ld:
-                if p not in ld_paths:
-                    ld_paths.append(p)
-
-        if not names:
-            deps = self.dependencies
-        else:
-            deps = [{'name': name} for name in names if name is not None]
+        Add linker and preprocessor paths to dependencies to self.variables
+        :names: list of strings containing the name of the dependency
+        """
+        # collect dependencies
+        dependencies = self.dependencies if names is None else [{"name": name} for name in names if name]
 
         # collect software install prefixes for dependencies
-        roots = []
-        for dep in deps:
-            if dep.get('external_module', False):
+        dependency_roots = []
+        for dep in dependencies:
+            if dep.get("external_module", False):
                 # for software names provided via external modules, install prefix may be unknown
-                names = dep['external_module_metadata'].get('name', [])
-                roots.extend([root for root in self.get_software_root(names) if root is not None])
+                names = dep["external_module_metadata"].get("name", [])
+                dependency_roots.extend([root for root in self.get_software_root(names) if root is not None])
             else:
-                roots.extend(self.get_software_root(dep['name']))
+                dependency_roots.extend(self.get_software_root(dep["name"]))
 
-        for root in roots:
-            self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths)
-            self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths)
+        for root in dependency_roots:
+            self._add_dependency_cpp_headers(root, extra_dirs=cpp)
+            self._add_dependency_linker_paths(root, extra_dirs=ld)
+
+    def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
+        """
+        Append prepocessor paths for given dependency root directory
+        """
+        header_dirs = ["include"]
+        if isinstance(extra_dirs, (tuple, list)):
+            header_dirs.extend(extra_dirs)
+        header_dirs = set(header_dirs)  # remove duplicates
+
+        # mode of operation is defined by cpp-headers-search-path option
+        # toolchain option has precedence over build option
+        cpp_headers_mode = DEFAULT_CPP_HEADERS_SEARCH_PATH
+        tc_opt = self.options.option("cpp-headers-search-path")
+        if tc_opt is not None:
+            self.log.debug("cpp-headers-search-path set by toolchain option: %s", cpp_headers_mode)
+            cpp_headers_mode = tc_opt
+        else:
+            build_opt = build_option("cpp_headers_search_path")
+            if build_opt is not None:
+                self.log.debug("cpp-headers-search-path set by build option: %s", cpp_headers_mode)
+                cpp_headers_mode = build_opt
+
+        if cpp_headers_mode not in CPP_HEADERS_SEARCH_PATHS:
+            raise EasyBuildError(
+                "Unknown value selected for option cpp-headers-search-path. Choose one of: %s",
+                ", ".join(CPP_HEADERS_SEARCH_PATHS)
+            )
+
+        for env_var in CPP_HEADERS_SEARCH_PATHS[cpp_headers_mode]:
+            self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root)
+            self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs)
+
+    def _add_dependency_linker_paths(self, dep_root, extra_dirs=None):
+        """
+        Append linker paths for given dependency root directory
+        """
+        lib_dirs = ["lib64", "lib"]
+        if isinstance(extra_dirs, (tuple, list)):
+            lib_dirs.extend(extra_dirs)
+        lib_dirs = set(lib_dirs)  # remove duplicates
+
+        env_var = "LDFLAGS"
+        self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root)
+        self.variables.append_subdirs(env_var, dep_root, subdirs=lib_dirs)
 
     def _setenv_variables(self, donotset=None, verbose=True):
         """Actually set the environment variables"""

easybuild/tools/toolchain/variables.py

@@ -46,6 +46,11 @@ class LinkLibraryPaths(AbsPathList):
     PREFIX = '-L'
 
 
+class SearchPaths(AbsPathList):
+    """Colon-separated list of absolute paths"""
+    SEPARATOR = ':'
+
+
 class FlagList(StrList):
     """Flag list"""
     PREFIX = "-"

test/framework/toolchain.py

@@ -955,6 +955,49 @@ def test_precision_flags(self):
 
                 self.modtool.purge()
 
+    def test_cpp_headers_search_path(self):
+        """Test functionality behind cpp-headers-search-path option"""
+        cpp_headers_mode = {
+            "CPPFLAGS": ["CPPFLAGS"],
+            "CPATH": ["CPATH"],
+            "INCLUDE_PATHS": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
+        }
+        # test without toolchain option
+        for build_opt in cpp_headers_mode:
+            init_config(build_options={"cpp_headers_search_path": build_opt, "silent": True})
+            tc = self.get_toolchain("foss", version="2018a")
+            with self.mocked_stdout_stderr():
+                tc.prepare()
+                for env_var in cpp_headers_mode[build_opt]:
+                    assert_fail_msg = (
+                        f"Variable {env_var} required by cpp-headers-search-path build option '{build_opt}' "
+                        "not found in toolchain environment"
+                    )
+                    self.assertIn(env_var, [*tc.variables], assert_fail_msg)
+                self.modtool.purge()
+        # test with toolchain option
+        for build_opt in cpp_headers_mode:
+            init_config(build_options={"cpp_headers_search_path": build_opt, "silent": True})
+            for tc_opt in cpp_headers_mode:
+                tc = self.get_toolchain("foss", version="2018a")
+                tc.set_options({"cpp-headers-search-path": tc_opt})
+                with self.mocked_stdout_stderr():
+                    tc.prepare()
+                    for env_var in cpp_headers_mode[tc_opt]:
+                        assert_fail_msg = (
+                            f"Variable {env_var} required by cpp-headers-search-path toolchain option '{tc_opt}' "
+                            "not found in toolchain environment"
+                        )
+                        self.assertIn(env_var, [*tc.variables], assert_fail_msg)
+                self.modtool.purge()
+        # test wrong toolchain option
+        tc = self.get_toolchain("foss", version="2018a")
+        tc.set_options({"cpp-headers-search-path": "WRONG_MODE"})
+        with self.mocked_stdout_stderr():
+            error_pattern = "Unknown value selected for option cpp-headers-search-path"
+            self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+        self.modtool.purge()
+
     def test_cgoolf_toolchain(self):
         """Test for cgoolf toolchain."""
         tc = self.get_toolchain("cgoolf", version="1.1.6")