diff --git a/opuspocus/pipeline_steps/generate_vocab.py b/opuspocus/pipeline_steps/generate_vocab.py index eb047f2..ac6102e 100644 --- a/opuspocus/pipeline_steps/generate_vocab.py +++ b/opuspocus/pipeline_steps/generate_vocab.py @@ -46,7 +46,7 @@ def step_name(self): def _cmd_header_str(self) -> str: return super()._cmd_header_str( n_cpus=8, - mem=20 + mem=80 ) def _cmd_vars_str(self) -> str: diff --git a/opuspocus/pipeline_steps/train_model.py b/opuspocus/pipeline_steps/train_model.py index 8c61c41..48744e9 100644 --- a/opuspocus/pipeline_steps/train_model.py +++ b/opuspocus/pipeline_steps/train_model.py @@ -269,7 +269,7 @@ def _cmd_body_str(self) -> str: echo $new_jid > `pwd`/step.jobid # Update the job dependencies -for job in `sqeueu --me --format "%i $E" | grep ":$SLURM_JOBID" | grep -v ^$new_jid | cut -d" " -f1`; do +for job in `sqeueu --me --format "%i %E" | grep ":$SLURM_JOBID" | grep -v ^$new_jid | cut -d" " -f1`; do echo Updating dependencies of job $job... >&2 update_str=$(squeue --me --format "%i %E" \\ | grep ^$job \\ diff --git a/scripts/resubmit_step.sh b/scripts/resubmit_step.sh new file mode 100755 index 0000000..4f0e810 --- /dev/null +++ b/scripts/resubmit_step.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Resubmits step and updates its dependants +set -euo pipefail + +STEP_DIR=$1 +SLURM_PARAMS=$2 + +SLURM_JOBID=$(cat $STEP_DIR/step.jobid) + +new_jid=$( sbatch --parsable $SLURM_PARAMS $STEP_DIR/step.command ) + +echo $new_jid > $STEP_DIR/step.jobid +echo RUNNING > $STEP_DIR/step.state + +# Update the dependent steps +for job in $(sqeueu --me --format "%i %E" | grep ":$SLURM_JOBID" | grep -v ^$new_jid | cut -d" " -f1); do + echo Updating dependencies of job $job... >&2 + update_str=$(squeue --me --format "%i %E" \ + | grep ^$job \ + | cut -d" " -f2 \ + | sed "s/([^)]*)//g;s/$SLURM_JOBID/$new_jid/" \ + ) + scontrol update JobId=$job dependency=$update_str +done diff --git a/scripts/slurm_time_to_seconds.py b/scripts/slurm_time_to_seconds.py index 5400a73..d46408e 100644 --- a/scripts/slurm_time_to_seconds.py +++ b/scripts/slurm_time_to_seconds.py @@ -1,9 +1,20 @@ #!/usr/bin/env python3 import sys -time = [int(x) for x in str(sys.argv[1]).split(":")] + +def hhmmss_to_seconds(t): + t_s = 0 + for i in range(len(t) - 1): + t_s = 60 * (t_s + t[i]) + return t_s + t[-1] + +time_str = str(sys.argv[1]) time_s = 0 -for i in range(len(time) - 1): - time_s = 60 * (time_s + time[i]) -time_s += time[-1] + +time_str_daytime = sys.argv[1].split('-') +if len(time_str_daytime) == 2: + time_s = int(time_str_daytime[0]) * 24 * 60 * 60 +time = [int(x) for x in time_str_daytime[-1].split(':')] +time_s += hhmmss_to_seconds(time) + print(time_s)