Fix formatting and bugfix in spread (itvs_cts_slots2)

add workflow again

.github/workflows/run-tests.yml

@@ -0,0 +1,75 @@
+name: OAR3 - run tests
+on:
+  push:
+    branches:
+      - '*'
+  workflow_call:
+    secrets:
+      DOCKERTOKEN:
+        required: true
+      DOCKERUSER:
+        required: true
+
+jobs:
+    Check-Code-Formatting:
+      runs-on: ubuntu-latest
+      steps:
+        - name: Check out repository code
+          uses: actions/checkout@v2
+        - name: Install dev requirements
+          run: pip install -r requirements/dev.txt
+        - name: Check code formatting
+          run: ./scripts/ci/check-formatting.sh
+    Run-Tests-On-Docker:
+      runs-on: ubuntu-latest
+      strategy:
+          matrix:
+            images:
+            #  - { PYTHON_IMAGE: "python:3.6", POSTGRES_IMAGE: "postgres:11" }
+            #  - { PYTHON_IMAGE: "python:3.6", POSTGRES_IMAGE: "postgres:12" }
+              - { PYTHON_IMAGE: "python:3.7", POSTGRES_IMAGE: "postgres:11" }
+              - { PYTHON_IMAGE: "python:3.7", POSTGRES_IMAGE: "postgres:12" }
+              - { PYTHON_IMAGE: "python:3.8", POSTGRES_IMAGE: "postgres:11" }
+              - { PYTHON_IMAGE: "python:3.8", POSTGRES_IMAGE: "postgres:12" }
+              # Configuration for bullseye
+              - { PYTHON_IMAGE: "python:3.9", POSTGRES_IMAGE: "postgres:13" }
+      steps:
+        - name: Check out repository code
+          uses: actions/checkout@v2
+          # Connect to docker to have user-base docker rate limit instead of by ip
+        - name: Connect to dockerhub
+          run: echo $DOCKERTOKEN | docker login --username ${DOCKERUSER} --password-stdin
+          env:
+            DOCKERUSER: ${{ secrets.DOCKERUSER }}
+            DOCKERTOKEN: ${{ secrets.DOCKERTOKEN }}
+        - name: Build dockers
+          run: ./scripts/ci/build-docker-env.sh
+          env:
+            # Images configurations
+            PYTHON_IMAGE: ${{ matrix.images.PYTHON_IMAGE }}
+            POSTGRES_IMAGE: ${{ matrix.images.POSTGRES_IMAGE }}
+        - name: Run tests
+          run:  ./scripts/ci/run-tests-with-docker.sh
+          env:
+            PYTHON_IMAGE: ${{ matrix.images.PYTHON_IMAGE }}
+            POSTGRES_IMAGE: ${{ matrix.images.POSTGRES_IMAGE }}
+        - name: Upload coverage
+          uses: actions/upload-artifact@v2
+          with:
+            name: coverage-report
+            path: coverage.xml
+    Code-Coverage:
+      runs-on: ubuntu-latest
+      needs: Run-Tests-On-Docker
+      if: success()
+      steps:
+        # Check out repository so codecov isn't lost
+        - uses: actions/checkout@master
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            path: artifacts
+        - uses: codecov/codecov-action@v1
+          with:
+            files: ./artifacts/coverage-report/coverage.xml
+            fail_ci_if_error: true # optional (default = false)

etc/oar/admission_rules.d/22_compact.py

@@ -6,4 +6,6 @@
         for prop_res in resource_desc:
             for resource_value in prop_res["resources"]:
                 if not resource_value["resource"] == "core":
-                    raise Exception("# ADMISSION RULE> Error: If type compact is given, only core can be used as resource type.")
+                    raise Exception(
+                        "# ADMISSION RULE> Error: If type compact is given, only core can be used as resource type."
+                    )

etc/oar/admission_rules.d/23_spread.py

@@ -115,7 +115,9 @@ def estimate_job_nb_resources_in_spread(
         for prop_res in resource_desc:
             for resource_value in prop_res["resources"]:
                 if not resource_value["resource"] == "core":
-                    raise Exception("# ADMISSION RULE> Error: If type spread is given, only core can be used as resource type.")
+                    raise Exception(
+                        "# ADMISSION RULE> Error: If type spread is given, only core can be used as resource type."
+                    )
 
     if (
         estimate_job_nb_resources_in_spread(

etc/oar/admission_rules.d/24_no_pref.py

@@ -6,4 +6,6 @@
         for prop_res in resource_desc:
             for resource_value in prop_res["resources"]:
                 if not resource_value["resource"] == "core":
-                    raise Exception("# ADMISSION RULE> Error: If type no_pref is given, only core can be used as resource type.")
+                    raise Exception(
+                        "# ADMISSION RULE> Error: If type no_pref is given, only core can be used as resource type."
+                    )

etc/oar/admission_rules.d/26_co_loc.py

@@ -115,7 +115,9 @@ def estimate_job_nb_resources_in_spread(
         for prop_res in resource_desc:
             for resource_value in prop_res["resources"]:
                 if not resource_value["resource"] == "core":
-                    raise Exception("# ADMISSION RULE> Error: If type co_loc is given, only core can be used as resource type.")
+                    raise Exception(
+                        "# ADMISSION RULE> Error: If type co_loc is given, only core can be used as resource type."
+                    )
 
     if (
         estimate_job_nb_resources_in_spread(

etc/oar/admission_rules.d/27_f_compact.py

@@ -6,4 +6,6 @@
         for prop_res in resource_desc:
             for resource_value in prop_res["resources"]:
                 if not resource_value["resource"] == "core":
-                    raise Exception("# ADMISSION RULE> Error: If type f_compact is given, only core can be used as resource type.")
+                    raise Exception(
+                        "# ADMISSION RULE> Error: If type f_compact is given, only core can be used as resource type."
+                    )

etc/oar/admission_rules.d/28_f_spread.py

@@ -115,7 +115,9 @@ def estimate_job_nb_resources_in_spread(
         for prop_res in resource_desc:
             for resource_value in prop_res["resources"]:
                 if not resource_value["resource"] == "core":
-                    raise Exception("# ADMISSION RULE> Error: If type f_spread is given, only core can be used as resource type.")
+                    raise Exception(
+                        "# ADMISSION RULE> Error: If type f_spread is given, only core can be used as resource type."
+                    )
 
     if (
         estimate_job_nb_resources_in_spread(

etc/oar/admission_rules.d/29_f_co_loc.py

@@ -115,7 +115,9 @@ def estimate_job_nb_resources_in_spread(
         for prop_res in resource_desc:
             for resource_value in prop_res["resources"]:
                 if not resource_value["resource"] == "core":
-                    raise Exception("# ADMISSION RULE> Error: If type f_co_loc is given, only core can be used as resource type.")
+                    raise Exception(
+                        "# ADMISSION RULE> Error: If type f_co_loc is given, only core can be used as resource type."
+                    )
 
     if (
         estimate_job_nb_resources_in_spread(

etc/oar/admission_rules.d/30_exclusive.py

@@ -1,6 +1,7 @@
 if "exclusive" in types:
 
     import math
+
     resource_set = ResourceSet(session, config)
     resources_itvs = resource_set.roid_itvs
 

oar/kao/custom_scheduling.py

@@ -1,11 +1,13 @@
+import itertools
+
 from procset import ProcSet
 
 from oar.lib.globals import get_logger
 from oar.lib.hierarchy import find_resource_hierarchies_scattered
-import itertools
 
 logger = get_logger("oar.custom_scheduling")
 
+
 def compact(itvs_slots, hy_res_rqts, hy, beginning_slotset, reverse=True):
     """
     Given a job resource request and a set of resources this function tries to find a matching allocation.
@@ -32,15 +34,29 @@ def compact(itvs_slots, hy_res_rqts, hy, beginning_slotset, reverse=True):
 
         itvs_cts_slots = constraints & itvs_slots
 
-        hy_nodes = sorted(hy['network_address'], key=lambda i: len(i & itvs_cts_slots), reverse=reverse)
+        # create a nodes Procset list sorted by min/max free cores
+        hy_nodes = sorted(
+            hy["network_address"],
+            key=lambda i: len(i & itvs_cts_slots),
+            reverse=reverse,
+        )
 
         hy_levels = []
         for node in hy_nodes:
-            n_cpus=list(filter(lambda p: p.issubset(node),hy['cpu']))
-            n_cpus=sorted(n_cpus, key=lambda i: len(i & itvs_cts_slots), reverse=reverse)
-            hy_levels+=list(map(ProcSet, itertools.chain.from_iterable(map(iter, n_cpus))))
-
-        hy_levels=[hy_levels]
+            # collect cpu Procset for particular node
+            n_cpus = list(filter(lambda p: p.issubset(node), hy["cpu"]))
+            # sort cpu Procset list sorted by min/max free cores
+            n_cpus = sorted(
+                n_cpus, key=lambda i: len(i & itvs_cts_slots), reverse=reverse
+            )
+            # map cpu Procset to core procset
+            hy_levels += list(
+                map(ProcSet, itertools.chain.from_iterable(map(iter, n_cpus)))
+            )
+
+        # there is an Admission Rule that blocks other resources than core
+        # so only 1 resource type will be given
+        hy_levels = [hy_levels]
 
         res = find_resource_hierarchies_scattered(
             itvs_cts_slots, list(hy_levels), hy_nbs
@@ -87,15 +103,29 @@ def spread(itvs_slots, hy_res_rqts, hy, beginning_slotset, reverse=False):
 
         # Select unused resources first (top-down).
         try:
-            hy_nodes = sorted(hy['network_address'], key=lambda i: len(i & itvs_cts_slots), reverse=reverse)
+            # create a nodes Procset list sorted by min/max free cores
+            hy_nodes = sorted(
+                hy["network_address"],
+                key=lambda i: len(i & itvs_cts_slots2),
+                reverse=reverse,
+            )
 
             hy_levels = []
             for node in hy_nodes:
-                n_cpus=list(filter(lambda p: p.issubset(node),hy['cpu']))
-                n_cpus=sorted(n_cpus, key=lambda i: len(i & itvs_cts_slots), reverse=reverse)
-                hy_levels+=list(map(ProcSet, itertools.chain.from_iterable(map(iter, n_cpus))))
-
-            hy_levels=[hy_levels]
+                # collect cpu Procset for particular node
+                n_cpus = list(filter(lambda p: p.issubset(node), hy["cpu"]))
+                # sort cpu Procset list sorted by min/max free cores
+                n_cpus = sorted(
+                    n_cpus, key=lambda i: len(i & itvs_cts_slots2), reverse=reverse
+                )
+                # map cpu Procset to core procset
+                hy_levels += list(
+                    map(ProcSet, itertools.chain.from_iterable(map(iter, n_cpus)))
+                )
+
+            # there is an Admission Rule that blocks other resources than core
+            # so only 1 resource type will be given
+            hy_levels = [hy_levels]
 
         except Exception as e:
             logger.info(e)

oar/lib/resource_handling.py

@@ -1,15 +1,13 @@
 # coding: utf-8
 """ Functions to handle resource"""
-import os
-from typing import Any, List
-
-from sqlalchemy.orm import Session
 import io
 import os
+from typing import Any, List
 
 import pandas as pd
 from joblib import load
 from sqlalchemy import and_, distinct, func, or_, text
+from sqlalchemy.orm import Session
 from sqlalchemy.orm.exc import NoResultFound
 
 import oar.lib.tools as tools