fixed typo in the train_model step resubmission part of step.command

opuspocus/pipeline_steps/generate_vocab.py

@@ -46,7 +46,7 @@ def step_name(self):
     def _cmd_header_str(self) -> str:
         return super()._cmd_header_str(
             n_cpus=8,
-            mem=20
+            mem=80
         )
 
     def _cmd_vars_str(self) -> str:

opuspocus/pipeline_steps/train_model.py

@@ -269,7 +269,7 @@ def _cmd_body_str(self) -> str:
 echo $new_jid > `pwd`/step.jobid
 
 # Update the job dependencies
-for job in `sqeueu --me --format "%i $E" | grep ":$SLURM_JOBID" | grep -v ^$new_jid | cut -d" " -f1`; do
+for job in `sqeueu --me --format "%i %E" | grep ":$SLURM_JOBID" | grep -v ^$new_jid | cut -d" " -f1`; do
     echo Updating dependencies of job $job... >&2
     update_str=$(squeue --me --format "%i %E" \\
         | grep ^$job \\

scripts/resubmit_step.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Resubmits step and updates its dependants
+set -euo pipefail
+
+STEP_DIR=$1
+SLURM_PARAMS=$2
+
+SLURM_JOBID=$(cat $STEP_DIR/step.jobid)
+
+new_jid=$( sbatch --parsable $SLURM_PARAMS $STEP_DIR/step.command )
+
+echo $new_jid > $STEP_DIR/step.jobid
+echo RUNNING > $STEP_DIR/step.state
+
+# Update the dependent steps
+for job in $(sqeueu --me --format "%i %E" | grep ":$SLURM_JOBID" | grep -v ^$new_jid | cut -d" " -f1); do
+    echo Updating dependencies of job $job... >&2
+    update_str=$(squeue --me --format "%i %E" \
+        | grep ^$job \
+        | cut -d" " -f2 \
+        | sed "s/([^)]*)//g;s/$SLURM_JOBID/$new_jid/" \
+    )
+    scontrol update JobId=$job dependency=$update_str
+done

scripts/slurm_time_to_seconds.py

@@ -1,9 +1,20 @@
 #!/usr/bin/env python3
 import sys
 
-time = [int(x) for x in str(sys.argv[1]).split(":")]
+
+def hhmmss_to_seconds(t):
+    t_s = 0
+    for i in range(len(t) - 1):
+        t_s = 60 * (t_s + t[i])
+    return t_s + t[-1]
+
+time_str = str(sys.argv[1])
 time_s = 0
-for i in range(len(time) - 1):
-    time_s = 60 * (time_s + time[i])
-time_s += time[-1]
+
+time_str_daytime = sys.argv[1].split('-')
+if len(time_str_daytime) == 2:
+    time_s = int(time_str_daytime[0]) * 24 * 60 * 60
+time = [int(x) for x in time_str_daytime[-1].split(':')]
+time_s += hhmmss_to_seconds(time)
+
 print(time_s)