Skip to content

Commit

Permalink
quick fix async_task not running on Terrascope
Browse files Browse the repository at this point in the history
ZkJobRegistry was recently disabled on Terrascope but async_task directly relies on it.
Errors re: missing jobs are not considered fatal, by design.

{"message": "job not found; assuming user deleted it in the meanwhile", "levelname": "WARNING", "name": "__main__", "created": 1726475564.4724538, "filename": "async_task.py", "lineno": 283, "process": 12, "exc_info": "Traceback (most recent call last):\n  File \"/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/job_registry.py\", line 396, in _read\n    data, stat = self._zk.get(path)\n  File \"/opt/venv/lib64/python3.8/site-packages/kazoo/client.py\", line 1165, in get\n    return self.get_async(path, watch=watch).get()\n  File \"/opt/venv/lib64/python3.8/site-packages/kazoo/handlers/utils.py\", line 75, in get\n    raise self._exception\nkazoo.exceptions.NoNodeError: /openeo/integrationtests/jobs/ongoing/f689e77d-f188-40ca-b12b-3e278f0ad68f/j-2409161ca9c248e5986a11f20e61b26a\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/job_registry.py\", line 403, in _read\n    data, stat = self._zk.get(path)\n  File \"/opt/venv/lib64/python3.8/site-packages/kazoo/client.py\", line 1165, in get\n    return self.get_async(path, watch=watch).get()\n  File \"/opt/venv/lib64/python3.8/site-packages/kazoo/handlers/utils.py\", line 75, in get\n    raise self._exception\nkazoo.exceptions.NoNodeError: /openeo/integrationtests/jobs/done/f689e77d-f188-40ca-b12b-3e278f0ad68f/j-2409161ca9c248e5986a11f20e61b26a\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/async_task.py\", line 247, in main\n    job_info = registry.get_job(batch_job_id, user_id)\n  File \"/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/job_registry.py\", line 246, in get_job\n    job_info, _ = self._read(\n  File \"/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/job_registry.py\", line 406, in _read\n    raise JobNotFoundException(job_id) from e\nopeneo_driver.errors.JobNotFoundException: The batch job j-2409161ca9c248e5986a11f20e61b26a does not exist.", "job_id": "j-2409161ca9c248e5986a11f20e61b26a"}

#863
  • Loading branch information
bossie committed Sep 16, 2024
1 parent 72df128 commit 2ffb3af
Showing 1 changed file with 1 addition and 1 deletion.
2 changes: 1 addition & 1 deletion openeogeotrellis/backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -1664,7 +1664,7 @@ def _start_job(self, job_id: str, user: User, get_vault_token: Callable[[str], s
]
):
job_dependencies = self._schedule_and_get_dependencies(
supports_async_tasks=not ConfigParams().is_kube_deploy,
supports_async_tasks=not ConfigParams().is_kube_deploy and get_backend_config().use_zk_job_registry,
process_graph=job_process_graph,
api_version=api_version,
user_id=user_id,
Expand Down

0 comments on commit 2ffb3af

Please sign in to comment.