Merge develop into main before release 24.2

.github/workflows/integration_tests.yml

@@ -30,6 +30,7 @@ jobs:
         pip install -e ".[test]"
 
     - name: MCP Venue Dev - Integration tests
+      id: mcp_venue_dev_integration_tests
       continue-on-error: true
       env:
         AIRFLOW_WEBSERVER_PASSWORD: ${{ secrets.MCP_VENUE_DEV_AIRFLOW_WEBSERVER_PASSWORD }}
@@ -39,6 +40,7 @@ jobs:
         --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_DEV_AIRFLOW_ENDPOINT }}
 
     # - name: MCP Venue Test - Integration tests
+    #   id: mcp_venue_test_integration_tests
     #   continue-on-error: true
     #   run: |
     #     pytest -vv --gherkin-terminal-reporter \
@@ -58,7 +60,7 @@ jobs:
           exit 1
         fi
 
-        # Uncomment this block if MCP Venue Test Integration tests are re-enabled
+        # Uncomment this block when MCP Venue Test Integration tests are re-enabled
         # if [ "$test_status" != "success" ]; then
         #   echo "MCP Venue Test Integration Tests failed."
         #   exit 1

.github/workflows/smoke_tests.yml

@@ -57,16 +57,17 @@ jobs:
         --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_DEV_AIRFLOW_ENDPOINT }} \
         --ogc-processes-endpoint=${{ github.event.inputs.MCP_VENUE_DEV_OGC_PROCESSES_ENDPOINT || vars.MCP_VENUE_DEV_OGC_PROCESSES_ENDPOINT }}
 
-    - name: MCP Venue Test - Smoke tests
-      id: mcp_venue_test_smoke_tests
-      env:
-        AIRFLOW_WEBSERVER_PASSWORD: ${{ secrets.MCP_VENUE_TEST_AIRFLOW_WEBSERVER_PASSWORD }}
-      continue-on-error: true
-      run: |
-        pytest -vv --gherkin-terminal-reporter \
-        unity-test/system/smoke \
-        --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_AIRFLOW_ENDPOINT || vars.MCP_VENUE_TEST_AIRFLOW_ENDPOINT }} \
-        --ogc-processes-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_OGC_PROCESSES_ENDPOINT || vars.MCP_VENUE_TEST_OGC_PROCESSES_ENDPOINT }}
+# Temporary: comment out checks on MCP venue test until the SPS is redeployed
+#    - name: MCP Venue Test - Smoke tests
+#      id: mcp_venue_test_smoke_tests
+#      env:
+#        AIRFLOW_WEBSERVER_PASSWORD: ${{ secrets.MCP_VENUE_TEST_AIRFLOW_WEBSERVER_PASSWORD }}
+#      continue-on-error: true
+#      run: |
+#        pytest -vv --gherkin-terminal-reporter \
+#        unity-test/system/smoke \
+#        --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_AIRFLOW_ENDPOINT || vars.MCP_VENUE_TEST_AIRFLOW_ENDPOINT }} \
+#        --ogc-processes-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_OGC_PROCESSES_ENDPOINT || vars.MCP_VENUE_TEST_OGC_PROCESSES_ENDPOINT }}
 
     - name: MCP Venue Ops - Smoke tests
       id: mcp_venue_ops_smoke_tests
@@ -77,8 +78,9 @@ jobs:
         pytest -vv --gherkin-terminal-reporter \
         unity-test/system/smoke/step_defs/test_airflow_api_health.py \
         --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_OPS_AIRFLOW_ENDPOINT || vars.MCP_VENUE_OPS_AIRFLOW_ENDPOINT }} \
+        --ogc-processes-endpoint=${{ github.event.inputs.MCP_VENUE_OPS_OGC_PROCESSES_ENDPOINT || vars.MCP_VENUE_OPS_OGC_PROCESSES_ENDPOINT }}
 
-    - name: MCP Venue SBG Dev - Smoke tests
+    - name: MCP SBG DEV - Smoke tests
       id: mcp_sbg_dev_smoke_tests
       env:
         AIRFLOW_WEBSERVER_PASSWORD: ${{ secrets.MCP_VENUE_SBG_DEV_AIRFLOW_WEBSERVER_PASSWORD }}
@@ -94,22 +96,18 @@ jobs:
       if: always()
       run: |
         dev_status=${{ steps.mcp_venue_dev_smoke_tests.outcome }}
-        test_status=${{ steps.mcp_venue_test_smoke_tests.outcome }}
         ops_status=${{ steps.mcp_venue_ops_smoke_tests.outcome }}
         sbg_dev_status=${{ steps.mcp_sbg_dev_smoke_tests.outcome }}
         echo "Dev Smoke Tests: $dev_status"
-        echo "Test Smoke Tests: $test_status"
         echo "Ops Smoke Tests: $ops_status"
         echo "SBG Dev Smoke Tests: $sbg_dev_status"
 
-        if [ "$dev_status" != "success" ] || [ "$test_status" != "success" ] || [ "$ops_status" != "success" ] || [ "$sbg_dev_status" != "success" ]; then
+        # FIXME: must re-enable [ "$test_status" != "success" ]
+        if [ "$dev_status" != "success" ] || [ "$ops_status" != "success" ] || [ "$sbg_dev_status" != "success" ]; then
           echo "One or more smoke tests failed."
           if [ "$dev_status" != "success" ]; then
             echo "MCP Venue Dev Smoke Tests failed."
           fi
-          if [ "$test_status" != "success" ]; then
-            echo "MCP Venue Test Smoke Tests failed."
-          fi
           if [ "$ops_status" != "success" ]; then
             echo "MCP Venue Ops Smoke Tests failed."
           fi

airflow/dags/cwl_dag.py

@@ -40,18 +40,20 @@
 DEFAULT_CWL_ARGUMENTS = json.dumps({"message": "Hello Unity"})
 
 # Alternative arguments to execute SBG Pre-Process
-# DEFAULT_CWL_WORKFLOW = "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.cwl"
+# DEFAULT_CWL_WORKFLOW =  "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.cwl"
 # DEFAULT_CWL_ARGUMENTS = "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.dev.yml"
 
 # Alternative arguments to execute SBG end-to-end
-# DEFAULT_CWL_WORKFLOW = "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.cwl"
-# DEFAULT_CWL_ARGUMENTS = json.dumps({"input_processing_labels": ["label1", "label2"], "input_cmr_stac": "https://cmr.earthdata.nasa.gov/search/granules.stac?collection_concept_id=C2408009906-LPCLOUD&temporal[]=2023-08-10T03:41:03.000Z,2023-08-10T03:41:03.000Z", "input_unity_dapa_client": "40c2s0ulbhp9i0fmaph3su9jch", "input_unity_dapa_api": "https://d3vc8w9zcq658.cloudfront.net", "input_crid": "001", "output_collection_id": "urn:nasa:unity:unity:dev:SBG-L1B_PRE___1", "output_data_bucket": "sps-dev-ds-storage"})
+# DEFAULT_CWL_WORKFLOW =  "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/L1-to-L2-e2e.cwl"
+# DEFAULT_CWL_ARGUMENTS = "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/L1-to-L2-e2e.dev.yml"
 
+# Alternative arguments to execute SBG end-to-end
+# unity_sps_sbg_debug.txt
 CONTAINER_RESOURCES = k8s.V1ResourceRequirements(
     requests={
         # "cpu": "2660m",  # 2.67 vCPUs, specified in milliCPUs
         # "memory": "22Gi",  # Rounded to 22 GiB for easier specification
-        "ephemeral-storage": "30Gi"
+        "ephemeral-storage": "10Gi"
     },
     # limits={
     #    # "cpu": "2660m",  # Optional: set the same as requests if you want a fixed allocation
@@ -120,7 +122,7 @@ def setup(ti=None, **context):
     startup_timeout_seconds=1800,
     arguments=["{{ params.cwl_workflow }}", "{{ params.cwl_args }}"],
     container_security_context={"privileged": True},
-    # container_resources=CONTAINER_RESOURCES,
+    container_resources=CONTAINER_RESOURCES,
     container_logs=True,
     volume_mounts=[
         k8s.V1VolumeMount(name="workers-volume", mount_path=WORKING_DIR, sub_path="{{ dag_run.run_id }}")

airflow/dags/sbg_L1_to_L2_e2e_cwl_dag.py

@@ -33,7 +33,7 @@
 INPUT_AUX_STAC = '{"numberMatched":{"total_size":1},"numberReturned":1,"stac_version":"1.0.0","type":"FeatureCollection","links":[{"rel":"self","href":"https://d3vc8w9zcq658.cloudfront.net/am-uds-dapa/collections/urn:nasa:unity:unity:dev:SBG-L1B_PRE___1/items?limit=10"},{"rel":"root","href":"https://d3vc8w9zcq658.cloudfront.net"}],"features":[{"type":"Feature","stac_version":"1.0.0","id":"urn:nasa:unity:unity:dev:SBG-AUX___1:sRTMnet_v120","properties":{"datetime":"2024-02-14T22:04:41.078000Z","start_datetime":"2024-01-03T13:19:36Z","end_datetime":"2024-01-03T13:19:48Z","created":"2024-01-03T13:19:36Z","updated":"2024-02-14T22:05:25.248000Z","status":"completed","provider":"unity"},"geometry":{"type":"Point","coordinates":[0,0]},"links":[{"rel":"collection","href":"."}],"assets":{"sRTMnet_v120.h5":{"href":"s3://sps-dev-ds-storage/urn:nasa:unity:unity:dev:SBG-AUX___1/urn:nasa:unity:unity:dev:SBG-AUX___1:sRTMnet_v120.h5/sRTMnet_v120.h5","title":"sRTMnet_v120.h5","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["data"]},"sRTMnet_v120_aux.npz":{"href":"s3://sps-dev-ds-storage/urn:nasa:unity:unity:dev:SBG-AUX___1/urn:nasa:unity:unity:dev:SBG-AUX___1:sRTMnet_v120.h5/sRTMnet_v120_aux.npz","title":"sRTMnet_v120_aux.npz","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["data"]}},"bbox":[-180,-90,180,90],"stac_extensions":[],"collection":"urn:nasa:unity:unity:dev:SBG-AUX___1"}]}'
 
 dag = DAG(
-    dag_id="sbg-l1-to-l2-e2e-cwl-dag",
+    dag_id="sbg_L1_to_L2_e2e_cwl_dag",
     description="SBG L1 to L2 End-To-End Workflow as CWL",
     tags=["SBG", "Unity", "SPS", "NASA", "JPL"],
     is_paused_upon_creation=False,

airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py

@@ -365,15 +365,15 @@ def setup(ti=None, **context):
     "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.cwl"
 )
 preprocess_task = KubernetesPodOperator(
-    retries=5,
+    retries=0,
     task_id="SBG_Preprocess",
     namespace=POD_NAMESPACE,
     name="sbg-preprocess-pod",
     image=SPS_DOCKER_CWL_IMAGE,
     service_account_name="airflow-worker",
     in_cluster=True,
     get_logs=True,
-    startup_timeout_seconds=1200,
+    startup_timeout_seconds=1800,
     arguments=[SBG_PREPROCESS_CWL, "{{ti.xcom_pull(task_ids='Setup', key='preprocess_args')}}"],
     container_security_context={"privileged": True},
     container_resources=CONTAINER_RESOURCES,
@@ -403,15 +403,15 @@ def setup(ti=None, **context):
 )
 isofit_task = KubernetesPodOperator(
     # wait_until_job_complete=True,
-    retries=5,
+    retries=0,
     task_id="SBG_Isofit",
     namespace=POD_NAMESPACE,
     name="sbg-isofit",
     image=SPS_DOCKER_CWL_IMAGE,
     service_account_name="airflow-worker",
     in_cluster=True,
     get_logs=True,
-    startup_timeout_seconds=1200,
+    startup_timeout_seconds=1800,
     arguments=[SBG_ISOFIT_CWL, "{{ti.xcom_pull(task_ids='Setup', key='isofit_args')}}"],
     container_security_context={"privileged": True},
     container_resources=CONTAINER_RESOURCES,
@@ -441,15 +441,15 @@ def setup(ti=None, **context):
 )
 resample_task = KubernetesPodOperator(
     # wait_until_job_complete=True,=True,
-    retries=5,
+    retries=0,
     task_id="SBG_Resample",
     namespace=POD_NAMESPACE,
     name="sbg-resample-pod",
     image=SPS_DOCKER_CWL_IMAGE,
     service_account_name="airflow-worker",
     in_cluster=True,
     get_logs=True,
-    startup_timeout_seconds=1200,
+    startup_timeout_seconds=1800,
     arguments=[SBG_RESAMPLE_CWL, "{{ti.xcom_pull(task_ids='Setup', key='resample_args')}}"],
     container_security_context={"privileged": True},
     container_resources=CONTAINER_RESOURCES,
@@ -477,15 +477,15 @@ def setup(ti=None, **context):
 SBG_REFLECT_CORRECT_CWL = "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/reflect-correct/sbg-reflect-correct-workflow.cwl"
 reflect_correct_task = KubernetesPodOperator(
     # wait_until_job_complete=True,=True,
-    retries=5,
+    retries=0,
     task_id="SBG_Reflect",
     namespace=POD_NAMESPACE,
     name="sbg-reflect-pod",
     image=SPS_DOCKER_CWL_IMAGE,
     service_account_name="airflow-worker",
     in_cluster=True,
     get_logs=True,
-    startup_timeout_seconds=1200,
+    startup_timeout_seconds=1800,
     arguments=[SBG_REFLECT_CORRECT_CWL, "{{ti.xcom_pull(task_ids='Setup', key='reflect_correct_args')}}"],
     container_security_context={"privileged": True},
     container_resources=CONTAINER_RESOURCES,
@@ -515,15 +515,15 @@ def setup(ti=None, **context):
 )
 frcover_task = KubernetesPodOperator(
     # wait_until_job_complete=True,=True,
-    retries=5,
+    retries=0,
     task_id="SBG_Frcover",
     namespace=POD_NAMESPACE,
     name="sbg-frcover-pod",
     image=SPS_DOCKER_CWL_IMAGE,
     service_account_name="airflow-worker",
     in_cluster=True,
     get_logs=True,
-    startup_timeout_seconds=1200,
+    startup_timeout_seconds=1800,
     arguments=[SBG_FRCOVER_CWL, "{{ti.xcom_pull(task_ids='Setup', key='frcover_args')}}"],
     container_security_context={"privileged": True},
     container_resources=CONTAINER_RESOURCES,