Merge pull request #4595 from Flamefire/template-constant-dict

convert template constant lists to dicts and export the constants by name

easybuild/framework/easyblock.py

@@ -3922,7 +3922,7 @@ def update_config_template_run_step(self):
         """Update the the easyconfig template dictionary with easyconfig.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP names"""
 
         for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
-            self.cfg.template_values[name[0]] = str(getattr(self, name[0], None))
+            self.cfg.template_values[name] = str(getattr(self, name, None))
         self.cfg.generate_template_values()
 
     def skip_step(self, step, skippable):

easybuild/framework/easyconfig/easyconfig.py

@@ -1217,7 +1217,7 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals
             default_values.update({key: value[0] for key, value in self.extra_options.items()})
 
             self.generate_template_values()
-            templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS}
+            templ_const = {quote_py_str(value): name for name, (value, _) in TEMPLATE_CONSTANTS.items()}
 
             # create reverse map of templates, to inject template values where possible
             # longer template values are considered first, shorter template keys get preference over longer ones
@@ -1842,7 +1842,7 @@ def get_cuda_cc_template_value(self, key):
         Returns user-friendly error message in case neither are defined,
         or if an unknown key is used.
         """
-        if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC):
+        if key.startswith('cuda_') and any(x == key for x in TEMPLATE_NAMES_DYNAMIC):
             try:
                 return self.template_values[key]
             except KeyError:

easybuild/framework/easyconfig/format/pyheaderconfigobj.py

@@ -52,8 +52,8 @@
 def build_easyconfig_constants_dict():
     """Make a dictionary with all constants that can be used"""
     all_consts = [
-        ('TEMPLATE_CONSTANTS', {x[0]: x[1] for x in TEMPLATE_CONSTANTS}),
-        ('EASYCONFIG_CONSTANTS', {key: val[0] for key, val in EASYCONFIG_CONSTANTS.items()}),
+        ('TEMPLATE_CONSTANTS', {name: value for name, (value, _) in TEMPLATE_CONSTANTS.items()}),
+        ('EASYCONFIG_CONSTANTS', {name: value for name, (value, _) in EASYCONFIG_CONSTANTS.items()}),
         ('EASYCONFIG_LICENSES', {klass().name: name for name, klass in EASYCONFIG_LICENSES_DICT.items()}),
     ]
     err = []

easybuild/framework/extension.py

@@ -116,7 +116,7 @@ def __init__(self, mself, ext, extra_params=None):
 
         # Add install/builddir templates with values from master.
         for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
-            self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None))
+            self.cfg.template_values[key] = str(getattr(self.master, key, None))
 
         # We can't inherit the 'start_dir' value from the parent (which will be set, and will most likely be wrong).
         # It should be specified for the extension specifically, or be empty (so it is auto-derived).

easybuild/tools/docs.py

@@ -478,16 +478,16 @@ def avail_easyconfig_templates_txt():
 
     # step 1: add TEMPLATE_NAMES_EASYCONFIG
     doc.append('Template names/values derived from easyconfig instance')
-    for name in TEMPLATE_NAMES_EASYCONFIG:
-        doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+    for name, curDoc in TEMPLATE_NAMES_EASYCONFIG.items():
+        doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, curDoc))
     doc.append('')
 
     # step 2: add SOFTWARE_VERSIONS
     doc.append('Template names/values for (short) software versions')
-    for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
-        doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name))
-        doc.append("%s%%(%sshortver)s: short version for %s (<major>.<minor>)" % (INDENT_4SPACES, pref, name))
-        doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name))
+    for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+        doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, prefix, name))
+        doc.append("%s%%(%sshortver)s: short version for %s (<major>.<minor>)" % (INDENT_4SPACES, prefix, name))
+        doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, prefix, name))
     doc.append('')
 
     # step 3: add remaining config
@@ -506,20 +506,20 @@ def avail_easyconfig_templates_txt():
     # step 5: template_values can/should be updated from outside easyconfig
     # (eg the run_step code in EasyBlock)
     doc.append('Template values set outside EasyBlock runstep')
-    for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
-        doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+    for name, cur_doc in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.items():
+        doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc))
     doc.append('')
 
     # some template values are only defined dynamically,
     # see template_constant_dict function in easybuild.framework.easyconfigs.templates
     doc.append('Template values which are defined dynamically')
-    for name in TEMPLATE_NAMES_DYNAMIC:
-        doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+    for name, cur_doc in TEMPLATE_NAMES_DYNAMIC.items():
+        doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc))
     doc.append('')
 
     doc.append('Template constants that can be used in easyconfigs')
-    for cst in TEMPLATE_CONSTANTS:
-        doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, cst[0], cst[2], cst[1]))
+    for name, (value, cur_doc) in TEMPLATE_CONSTANTS.items():
+        doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, name, cur_doc, value))
 
     return '\n'.join(doc)
 
@@ -530,19 +530,19 @@ def avail_easyconfig_templates_rst():
 
     title = 'Template names/values derived from easyconfig instance'
     table_values = [
-        ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG],
-        [name[1] for name in TEMPLATE_NAMES_EASYCONFIG],
+        ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG],
+        list(TEMPLATE_NAMES_EASYCONFIG.values()),
     ]
     doc = rst_title_and_table(title, table_titles, table_values)
     doc.append('')
 
     title = 'Template names/values for (short) software versions'
     ver = []
     ver_desc = []
-    for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
-        ver.append('``%%(%smajver)s``' % pref)
-        ver.append('``%%(%sshortver)s``' % pref)
-        ver.append('``%%(%sver)s``' % pref)
+    for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+        ver.append('``%%(%smajver)s``' % prefix)
+        ver.append('``%%(%sshortver)s``' % prefix)
+        ver.append('``%%(%sver)s``' % prefix)
         ver_desc.append('major version for %s' % name)
         ver_desc.append('short version for %s (<major>.<minor>)' % name)
         ver_desc.append('full version for %s' % name)
@@ -565,24 +565,24 @@ def avail_easyconfig_templates_rst():
 
     title = 'Template values set outside EasyBlock runstep'
     table_values = [
-        ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
-        [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+        ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+        list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()),
     ]
     doc.extend(rst_title_and_table(title, table_titles, table_values))
 
     title = 'Template values which are defined dynamically'
     table_values = [
-        ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC],
-        [name[1] for name in TEMPLATE_NAMES_DYNAMIC],
+        ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC],
+        list(TEMPLATE_NAMES_DYNAMIC.values()),
     ]
     doc.extend(rst_title_and_table(title, table_titles, table_values))
 
     title = 'Template constants that can be used in easyconfigs'
-    titles = ['Constant', 'Template value', 'Template name']
+    titles = ['Constant', 'Template description', 'Template value']
     table_values = [
-        ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS],
-        [cst[2] for cst in TEMPLATE_CONSTANTS],
-        ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS],
+        ['``%s``' % name for name in TEMPLATE_CONSTANTS],
+        [doc for _, doc in TEMPLATE_CONSTANTS.values()],
+        ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()],
     ]
     doc.extend(rst_title_and_table(title, titles, table_values))
 
@@ -595,19 +595,19 @@ def avail_easyconfig_templates_md():
 
     title = 'Template names/values derived from easyconfig instance'
     table_values = [
-        ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG],
-        [name[1] for name in TEMPLATE_NAMES_EASYCONFIG],
+        ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG],
+        list(TEMPLATE_NAMES_EASYCONFIG.values()),
     ]
     doc = md_title_and_table(title, table_titles, table_values, title_level=2)
     doc.append('')
 
     title = 'Template names/values for (short) software versions'
     ver = []
     ver_desc = []
-    for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
-        ver.append('``%%(%smajver)s``' % pref)
-        ver.append('``%%(%sshortver)s``' % pref)
-        ver.append('``%%(%sver)s``' % pref)
+    for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+        ver.append('``%%(%smajver)s``' % prefix)
+        ver.append('``%%(%sshortver)s``' % prefix)
+        ver.append('``%%(%sver)s``' % prefix)
         ver_desc.append('major version for %s' % name)
         ver_desc.append('short version for %s (``<major>.<minor>``)' % name)
         ver_desc.append('full version for %s' % name)
@@ -631,27 +631,28 @@ def avail_easyconfig_templates_md():
 
     title = 'Template values set outside EasyBlock runstep'
     table_values = [
-        ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
-        [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+        ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+        list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()),
     ]
     doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2))
     doc.append('')
 
     title = 'Template values which are defined dynamically'
     table_values = [
-        ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC],
-        [name[1] for name in TEMPLATE_NAMES_DYNAMIC],
+        ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC],
+        list(TEMPLATE_NAMES_DYNAMIC.values()),
     ]
     doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2))
     doc.append('')
 
     title = 'Template constants that can be used in easyconfigs'
-    titles = ['Constant', 'Template value', 'Template name']
+    titles = ['Constant', 'Template description', 'Template value']
     table_values = [
-        ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS],
-        [cst[2] for cst in TEMPLATE_CONSTANTS],
-        ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS],
+        ['``%s``' % name for name in TEMPLATE_CONSTANTS],
+        [doc for _, doc in TEMPLATE_CONSTANTS.values()],
+        ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()],
     ]
+
     doc.extend(md_title_and_table(title, titles, table_values, title_level=2))
 
     return '\n'.join(doc)

test/framework/easyconfig.py

@@ -1272,6 +1272,14 @@ def test_templating_constants(self):
         ec = EasyConfig(test_ec)
         self.assertEqual(ec['sanity_check_commands'], ['mpiexec -np 1 -- toy'])
 
+    def test_template_constant_import(self):
+        """Test importing template constants works"""
+        from easybuild.framework.easyconfig.templates import GITHUB_SOURCE, GNU_SOURCE, SHLIB_EXT
+        from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
+        self.assertEqual(GITHUB_SOURCE, TEMPLATE_CONSTANTS['GITHUB_SOURCE'][0])
+        self.assertEqual(GNU_SOURCE, TEMPLATE_CONSTANTS['GNU_SOURCE'][0])
+        self.assertEqual(SHLIB_EXT, get_shared_lib_ext())
+
     def test_templating_cuda_toolchain(self):
         """Test templates via toolchain component, like setting %(cudaver)s with fosscuda toolchain."""
 
@@ -1379,7 +1387,7 @@ def test_templating_doc(self):
         # expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between)
         temps = [
             easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG,
-            easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3,
+            list(easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS.keys()) * 3,
             easyconfig.templates.TEMPLATE_NAMES_CONFIG,
             easyconfig.templates.TEMPLATE_NAMES_LOWER,
             easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP,