Merge pull request #209 from unity-sds/fix-sbg-preprocess-dag

Fixing the SBG Preprocess DAG
unity-sds · Sep 23, 2024 · 9df0efc · 9df0efc
2 parents 8542f8b + 0a76ae4
commit 9df0efc
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 6 deletions.
diff --git a/airflow/dags/cwl_dag.py b/airflow/dags/cwl_dag.py
@@ -25,7 +25,6 @@
 # The Kubernetes namespace within which the Pod is run (it must already exist)
 POD_NAMESPACE = "sps"
 POD_LABEL = "cwl_task"
-# SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0"
 SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0"
 
 NODE_POOL_DEFAULT = "airflow-kubernetes-pod-operator"

diff --git a/airflow/dags/sbg_preprocess_cwl_dag.py b/airflow/dags/sbg_preprocess_cwl_dag.py
@@ -39,13 +39,18 @@
 
 # common parameters
 CONTAINER_RESOURCES = k8s.V1ResourceRequirements(
-    requests={"ephemeral-storage": "5Gi"},
+    requests={
+        "memory": "4Gi",
+        # "cpu": "8",
+        "ephemeral-storage": "10Gi",
+    },
 )
-INPUT_PROCESSING_LABELS = ["SBG", "CWL"]
+INPUT_PROCESSING_LABELS = ["SBG", "CWL", "Preprocess"]
 
 dag = DAG(
     dag_id="sbg_preprocess_cwl_dag",
     description="SBG Preprocess Workflow as CWL",
+    dag_display_name="CWL Preprocess DAG",
     tags=["SBG", "Unity", "SPS", "NASA", "JPL"],
     is_paused_upon_creation=False,
     catchup=False,
@@ -56,13 +61,13 @@
         "cwl_workflow": Param(
             DEFAULT_CWL_WORKFLOW,
             type="string",
-            title="CWL workflow",
+            title="SBG Preprocess workflow",
             description="The SBG Pre-process CWL workflow URL",
         ),
         "cwl_args": Param(
             DEFAULT_CWL_ARGUMENTS,
             type="string",
-            title="CWL workflow parameters",
+            title="SBG Preprocess workflow parameters",
             description="The SBG Pre-process YAML parameters URL",
         ),
     },
@@ -95,7 +100,7 @@ def setup(ti=None, **context):
     in_cluster=True,
     get_logs=True,
     startup_timeout_seconds=1800,
-    arguments=["{{ params.cwl_workflow }}", "{{ params.cwl_args }}"],
+    arguments=["-w", "{{ params.cwl_workflow }}", "-j", "{{ params.cwl_args }}", "-e", "None"],
     container_security_context={"privileged": True},
     container_resources=CONTAINER_RESOURCES,
     container_logs=True,