Overview page modules

.github/workflows/script_module_list.yml

@@ -0,0 +1,41 @@
+name: Run Linter and tests
+on: [push, pull_request]
+# Declare default permissions as read only.
+permissions: read-all
+jobs:
+
+  flake8-lint:
+    runs-on: ubuntu-20.04
+    name: Lint
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v3
+      - name: Set up Python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: flake8 Lint
+        uses: py-actions/flake8@v2
+        with:
+          max-line-length: "120"
+          path: "scripts/module_list"
+
+  pytest-tests:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          cd scripts/module_list
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Test with pytest
+        run: |
+          cd scripts/module_list
+          ./test.sh
+#         pip install pytest pytest-cov
+#         pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html

scripts/README.md

@@ -0,0 +1 @@
+scripts that can be used to automatically generate markdown files, can be found here.

scripts/module_list/README.md

@@ -0,0 +1,2 @@
+# Module list
+A script that generates a list (in markdown) of all available lmod modules.

scripts/module_list/module_overview.py

@@ -0,0 +1,175 @@
+import os
+import subprocess
+from typing import Union, Tuple
+from mdutils.mdutils import MdUtils
+import numpy as np
+
+
+# --------------------------------------------------------------------------------------------------------
+# Module bash API
+# --------------------------------------------------------------------------------------------------------
+
+def module(*args, filter_fn=lambda x: x) -> np.ndarray:
+    """
+    API to call the module command.
+
+    @param args: Extra arguments for the module command.
+    @param filter_fn: Filter function on the ouput.
+    @return:
+    """
+    lmod = os.getenv('LMOD_CMD')
+    proc = subprocess.run(
+        [lmod, "python", "-t"] + list(args),
+        encoding="utf-8",
+        stderr=subprocess.PIPE,
+        stdout=subprocess.PIPE
+    )
+    exec(proc.stdout)
+    return filter_fn(np.array(proc.stderr.split()))
+
+
+def module_avail(name: str = "", filter_fn=lambda x: x) -> np.ndarray:
+    """
+    API to call the module avail command of Lmod.
+
+    @param name: Possible module name.
+    @param filter_fn: Filter on the output.
+    @return: List of all available modules of name, or all if name is not given.
+    """
+    # return module(f"avail {name if name else ''}", filter_fn=filter_fn)
+    return module("avail", name, filter_fn=filter_fn)
+
+
+def module_swap(name: str) -> None:
+    """
+    API to call swap command of module.
+
+    @param name: Module you want to swap to.
+    """
+    module("swap", name)
+
+
+# --------------------------------------------------------------------------------------------------------
+# Fetch data
+# --------------------------------------------------------------------------------------------------------
+
+def filter_fn_gent_cluster(data: np.ndarray) -> np.ndarray:
+    """
+    Filter function for the output of clusters.
+    @param data: Output
+    @return: Filtered output
+    """
+    return data[~np.char.endswith(data, ":") &
+                ~np.char.startswith(data, "env/") &
+                ~np.char.startswith(data, "cluster/default")
+                ]
+
+
+def filter_fn_gent_modules(data: np.ndarray) -> np.ndarray:
+    """
+    Filter function for the output of all modules.
+    @param data: Output
+    @return: Filtered output
+    """
+    return data[~np.char.endswith(data, ":") &
+                ~np.char.startswith(data, "env/") &
+                ~np.char.startswith(data, "cluster/")
+                ]
+
+
+def clusters_ugent() -> np.ndarray:
+    """
+    Returns all the cluster names of the HPC at UGent.
+    @return: cluster names
+    """
+
+    return module_avail(name="cluster/", filter_fn=filter_fn_gent_cluster)
+
+
+def modules_ugent() -> dict:
+    """
+    Returns all the module names that are installed on the HPC on UGent.
+    They are grouped by cluster.
+    @return: Dictionary with all the modules per cluster
+    """
+
+    data = {}
+    for cluster in clusters_ugent():
+        module_swap(cluster)
+        data[cluster] = module_avail(filter_fn=filter_fn_gent_modules)
+    return data
+
+
+# --------------------------------------------------------------------------------------------------------
+# Util functions
+# --------------------------------------------------------------------------------------------------------
+
+def simplify_modules(data: Union[dict, list, np.ndarray]) -> Union[dict, list, np.ndarray]:
+    """
+    It removes the version of the modules and the duplicates.
+
+    @param data: List of modules
+    @return: List of programs.
+    """
+
+    if isinstance(data, dict):
+        simplified_data = {}
+        for cluster in data:
+            simplified_data[cluster] = np.unique([entry.split("/")[0] for entry in data[cluster]])
+    else:
+        simplified_data = np.unique([entry.split("/")[0] for entry in data])
+
+    return simplified_data
+
+
+# --------------------------------------------------------------------------------------------------------
+# Generate markdown
+# --------------------------------------------------------------------------------------------------------
+
+def generate_table_data(data: dict) -> Tuple[np.ndarray, int, int]:
+    """
+    Generate the data for the markdown table.
+
+    @param data: Available data
+    @return: Returns tuple (Table data, #col, #row)
+    """
+    data = simplify_modules(data)
+    all_modules = simplify_modules(np.concatenate(list(data.values())))
+
+    final = np.array([" "])
+    final = np.append(final, list(data.keys()))
+
+    for package in all_modules:
+        final = np.append(final, package)
+
+        for cluster in data:
+            final = np.append(final, "X" if package in data[cluster] else " ")
+
+    return final, len(data.keys()) + 1, len(all_modules) + 1
+
+
+def generate_module_table(data: dict, md_file: MdUtils) -> None:
+    """
+    Generate the general table of the overview.
+
+    @param data: Dict with all the data. Keys are the cluster names.
+    @param md_file: MdUtils object.
+    """
+    structured, col, row = generate_table_data(data)
+    md_file.new_table(columns=col, rows=row, text=list(structured), text_align='center')
+
+
+def generate_general_overview() -> None:
+    """
+    Generate the general overview in a markdown file.
+    It generates a list of all the available software and indicates on which cluster it is available.
+    """
+    md_file = MdUtils(file_name='Example_Markdown', title='Overview Moduls')
+    data = modules_ugent()
+    generate_module_table(data, md_file)
+    md_file.create_md_file()
+
+
+if __name__ == '__main__':
+    # Generate the overview
+    generate_general_overview()

scripts/module_list/requirements.txt

@@ -0,0 +1,5 @@
+flake8
+pytest
+mdutils
+numpy
+setuptools

scripts/module_list/test.sh

@@ -0,0 +1 @@
+PYTHONPATH=$PWD:$PYTHONPATH pytest -v -s

scripts/module_list/tests/data/data_avail_cluster_simple.txt

@@ -0,0 +1,3 @@
+/etc/modulefiles/vsc:
+cluster/accelgor
+cluster/victini

scripts/module_list/tests/data/data_avail_simple.txt

@@ -0,0 +1,29 @@
+/apps/gent/RHEL8/zen2-ib/modules/all:
+ABAQUS/
+ABAQUS/2021-hotfix-2132
+ABAQUS/2022-hotfix-2214
+ABAQUS/2022
+ABAQUS/2023
+ABINIT/
+ABINIT/9.2.1-intel-2020a
+ABINIT/9.4.1-intel-2020b
+zstd/
+zstd/1.4.4-GCCcore-9.3.0
+zstd/1.4.5-GCCcore-10.2.0
+zstd/1.4.9-GCCcore-10.3.0
+zstd/1.5.0-GCCcore-11.2.0
+zstd/1.5.2-GCCcore-11.3.0
+zstd/1.5.2-GCCcore-12.2.0
+/etc/modulefiles/vsc:
+cluster/
+cluster/accelgor
+cluster/victini
+env/slurm/
+env/slurm/accelgor
+env/slurm/victini
+env/software/
+env/software/accelgor
+env/software/victini
+env/vsc/
+env/vsc/accelgor
+env/vsc/victini