From 0f2ce216c215d3fc50b7986fcff60713802e338e Mon Sep 17 00:00:00 2001 From: Natacha Beck Date: Fri, 1 Apr 2016 16:35:41 -0400 Subject: [PATCH 1/3] FSL randomise is now parralel. --- README.md | 2 +- {cbrain_task/fsl_melodic/bin => bin}/fsl_sub | 204 ++++++++++++------ .../fsl_randomise/bourreau/fsl_randomise.rb | 14 +- cbrain_task_descriptors/fsl_sub.json | 12 +- 4 files changed, 162 insertions(+), 70 deletions(-) rename {cbrain_task/fsl_melodic/bin => bin}/fsl_sub (76%) diff --git a/README.md b/README.md index e6e9f91b..676069fd 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ rake cbrain:plugins:install:plugins * If you want FSL tools to be parallelized in CBRAIN (limited to FSL Melodic for now), replace the fsl_sub script in your FSL installation with the one in the plugin: ```bash -cp cbrain-plugins-neuro/cbrain_task/fsl_melodic/bin/fsl_sub ${FSLDIR}/bin +cp cbrain-plugins-neuro/bin/fsl_sub ${FSLDIR}/bin ``` * Restart your execution server (with the interface, click stop, then start). diff --git a/cbrain_task/fsl_melodic/bin/fsl_sub b/bin/fsl_sub similarity index 76% rename from cbrain_task/fsl_melodic/bin/fsl_sub rename to bin/fsl_sub index 5105bf73..d95604b9 100755 --- a/cbrain_task/fsl_melodic/bin/fsl_sub +++ b/bin/fsl_sub @@ -1,7 +1,7 @@ #!/bin/sh -# Copyright (C) 2007-2013 University of Oxford -# Authors: Dave Flitney, Stephen Smith and Matthew Webster +# Copyright (C) 2007-2014 University of Oxford +# Authors: Dave Flitney, Stephen Smith, Matthew Webster and Duncan Mortimer # Part of FSL - FMRIB's Software Library # http://www.fmrib.ox.ac.uk/fsl @@ -64,6 +64,7 @@ # Innovation Limited ("Isis"), the technology transfer company of the # University, to negotiate a licence. Contact details are: # innovation@isis.ox.ac.uk quoting reference DE/9564. +export LC_ALL=C ########################################################################### # Edit this file in order to setup FSL to use your local compute @@ -82,11 +83,13 @@ METHOD=SGE unset module if [ "x$SGE_ROOT" = "x" ] ; then - if [ "x$CBRAIN_WORKDIR" = "x" ] ; then METHOD=NONE - else - METHOD=CBRAIN - fi +else + QCONF=`which qconf` + if [ "x$QCONF" = "x" ]; then + METHOD=NONE + echo "Warning: SGE_ROOT environment variable is set but Grid Engine software not found, will run locally" >&2 + fi fi # stop submitted scripts from submitting jobs themselves @@ -95,9 +98,26 @@ if [ "X$FSLSUBALREADYRUN" = "Xtrue" ] ; then echo "Warning: job on queue attempted to submit parallel jobs - running jobs serially instead" >&2 fi +if [ "X$METHOD" = "XNONE" ]; then + QCONF=echo +fi FSLSUBALREADYRUN=true export FSLSUBALREADYRUN +############################# +# CBRAIN ADDENDUM #1 +# Support for method 'CBRAIN' +# +# Triggered by having the following +# environment variables set: +# - CBRAIN_WORKDIR +# Also uses optionally: +# - CBRAIN_SHARE_WD_TID +############################# +if test -n "$CBRAIN_WORKDIR" ; then + METHOD="CBRAIN" +fi + ########################################################################### # The following auto-decides what cluster queue to use. The calling # FSL program will probably use the -T option when calling fsl_sub, @@ -219,7 +239,7 @@ omp_pe='openmp' queue=long.q queueCmd=" -q long.q " mailto=`whoami`@fmrib.ox.ac.uk -MailOpts="n" +MailOpts="a" ########################################################################### @@ -269,7 +289,7 @@ while [ $1 != -- ] ; do -q) queue=$2 queueCmd=" -q $queue " - qconf -sq $queue 2>&1 >/dev/null + $QCONF -sq $queue >/dev/null 2>&1 if [ $? -eq 1 ]; then echo "Invalid queue specified!" exit 127 @@ -303,8 +323,19 @@ while [ $1 != -- ] ; do shift;; -t) taskfile=$2 - tasks=`wc -l $taskfile | awk '{print $1}'` - sge_tasks="-t 1-$tasks" + if [ -f "$taskfile" ] ; then + tasks=`wc -l $taskfile | awk '{print $1}'` + if [ $tasks -ne 0 ]; then + sge_tasks="-t 1-$tasks" + else + echo "Task file ${taskfile} is empty" + echo "Should be a text file listing all the commands to run!" + exit -1 + fi + else + echo "Task file (${taskfile}) does not exist" + exit -1 + fi shift;; -N) JobName=$2; @@ -315,7 +346,15 @@ while [ $1 != -- ] ; do -l) LogOpts="-o $2 -e $2"; LogDir="${2}/"; - mkdir -p $2; + if [ ! -e ${2} ]; then + mkdir -p $2 + else + echo ${2} | grep '/dev/null' >/dev/null 2>&1 + if [ $? -eq 1 ] && [ -f ${2} ]; then + echo "Log destination is a file (should be a folder)" + exit -1 + fi + fi shift;; -F) scriptmode=1; @@ -337,6 +376,19 @@ shift # Don't change the following (but keep scrolling down!) ########################################################################### +if [ -z "$taskfile" ] && [ -z "$1" ]; then + echo "Either supply a command to run or a parallel task file" + exit -1 +fi + +if [ -z "$taskfile" ] && [ ! -x "$1" ]; then + which $1 >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "The command you have requested cannot be found or is not executable" + exit -1 + fi +fi + if [ "x$JobName" = x ] ; then if [ "x$taskfile" != x ] ; then JobName=`basename $taskfile` @@ -345,15 +397,9 @@ if [ "x$JobName" = x ] ; then fi fi -if [ "x$tasks" != x ] && [ ! -f "$taskfile" ] ; then - echo $taskfile: invalid input! - echo Should be a text file listing all the commands to run! - exit -1 -fi - if [ "x$tasks" != "x" ] && [ "x$@" != "x" ] ; then - echo $@ - echo Spurious input after parsing command line! + echo "Spurious input after parsing command line: \"$@\"!" + echo "You appear to have specified both a task file and a command to run" exit -1 fi @@ -383,7 +429,7 @@ case $METHOD in if [ "x$peName" != x ]; then # Is this a configured PE? - qconf -sp $peName 2>&1 >/dev/null + $QCONF -sp $peName >/dev/null 2>&1 if [ $? -eq 1 ]; then echo $@ @@ -393,13 +439,13 @@ case $METHOD in # Get a list of queues configured for this PE and confirm that the queue # we have submitted to has that PE set up. - qstat -g c -pe $peName 2>&1 >/dev/null + qstat -g c -pe $peName >/dev/null 2>&1 if [ $? -eq 1 ]; then echo "No parallel environments configured!" exit -1 fi - qstat -g c -pe $peName | sed '1,2d' | awk '{ print $1 }' | grep ^$queue 2>&1 >/dev/null + qstat -g c -pe $peName | sed '1,2d' | awk '{ print $1 }' | grep ^$queue >/dev/null 2>&1 if [ $? -eq 1 ]; then echo $@ @@ -452,16 +498,9 @@ EOF if [ $verbose -eq 1 ] ; then echo executing: $@ >&2 fi - # Security check: won't run anything else than FSL commands - if [[ $1 != ${FSLDIR}* ]] - then - echo "Refusing to run command '$*' because it doesn't start with '${FSLDIR}'." - exit 1 - fi /bin/sh < ${LogDir}${JobName}.o$$ 2> ${LogDir}${JobName}.e$$ -# use 'exec $@' rather than just '$@' to avoid injections (e.g. 'ls ; rm * -Rf') -exec $@ +$@ EOF1 ERR=$? if [ $ERR -ne 0 ] ; then @@ -488,58 +527,99 @@ EOF2 echo $$ ;; + +############################# +# CBRAIN ADDENDUM #2 +# Support for method "CBRAIN" +############################# + CBRAIN) + + # This function preps a set of files that + # CBRAIN will recuperate, and cause the creation + # of a new CBRAIN task of type 'FslSub'. + # + # The function receives in $1 a number, usually an + # arbitrary task counter, and in the rest of the arguments + # a full command to run. E.g. + # + # submit_cbrain 2 ls -l + # + # The function will create a ".sh" file with the command, and + # a ".json" descriptor file for CBRAIN to pick up and process. function submit_cbrain { + + cbrain_subtask_number="$1"; shift; + if [ $verbose -eq 1 ] ; then - echo Submitting to CBRAIN: $* >&2 + echo Submitting subtask $cbrain_subtask_number to CBRAIN: $* >&2 fi - # Uses CBRAIN's mechanism for - # CBRAIN tasks to submit new tasks - FILE=`mktemp ${CBRAIN_WORKDIR}/new-task-XXXX.json` - cat << NEWTASK > ${FILE} + + pushd "$CBRAIN_WORKDIR" >/dev/null # save current workdir and switch to CBRAIN's + + # Prepare an ID to uses CBRAIN's mechanism for + # submit new subtasks. + sub_id=`mktemp ${cbrain_subtask_number}_XXXX` # e.g. "0_u3JK" + rm -f "$sub_id" # handle non-needed empty file just created + prefix=".new-task-$sub_id" # used for both .json and .sh files + + # Create the bash script + cat <"$prefix.sh" +# +# This file created automatically by $0 +# +# This is the code for CBRAIN local subtask $prefix +# + +$@ + +BASH_SCRIPT + + # Create the JSON descriptor + cat <"$prefix.json" { "tool-class": "CbrainTask::FslSub", -"description": "$*", +"description": "$JobName", +"share_wd_tid": "$CBRAIN_SHARE_WD_TID", "parameters": { - "fsl_sub_options": "$*" + "subtask_id": "$sub_id" }, -"prerequisites": "${jid}", +"prerequisites": "$jid", "required-to-post-process": true } -NEWTASK +JSON_FILE # Waits for the task id for up to 300 seconds - TASK_FILE="${FILE%%.*}.cbid" - x=0 - while [ "$x" -lt 300 -a ! -e ${TASK_FILE} ]; do - x=$((x+1)) - sleep 1 + cbrain_id_file="$prefix.cbid" + SECONDS=0 # bash auto-increment magical variable magic magic + while [ "$SECONDS" -lt 300 -a ! -e "$cbrain_id_file" ] ; do + sleep 3 done - if [ -e ${TASK_FILE} ] - then - cat ${TASK_FILE} # prints CBRAIN task id - else - echo "Error" - fi + test -r "$cbrain_id_file" && cat "$cbrain_id_file" + test ! -r "$cbrain_id_file" && echo "NoIDForCbrainSubtask" - } # end of function submit_cbrain + popd >/dev/null # return to current directory, wherever it was + } # end of generic function submit_cbrain - if [ "x$tasks" = "x" ] ; then - submit_cbrain $@ + # Main CBRAIN handling for FSL starts here + if [ "x$tasks" = "x" ] ; then # tasks is actually just the number of tasks if -t was supplied. + submit_cbrain 0 "$@" else if [ $verbose -eq 1 ] ; then echo "Submitting commands in: $taskfile" >&2 fi - n=1 - while [ $n -le $tasks ] ; do - line=`sed -n -e ''${n}'p' $taskfile` - if [ $verbose -eq 1 ] ; then - echo Submitting to CBRAIN: $line >&2 - fi - submit_cbrain ${line} - n=`expr $n + 1` - done + subtask_counter=0 + cb_subtask_ids="" + cat "$taskfile" | perl -ne 'print unless /^\s*$|^\s*#/' > "$taskfile.cleaned" + while read -r command ; do + subtask_counter=$(( $subtask_counter + 1 )) + cbid=`submit_cbrain $subtask_counter "$command"` + test -n "$cb_subtask_ids" && cb_subtask_ids="$cb_subtask_ids," + cb_subtask_ids="$cb_subtask_ids$cbid" + done < "$taskfile.cleaned" + rm -f "$taskfile.cleaned" + echo $cb_subtask_ids # all subtask IDs separated by commas fi ;; diff --git a/cbrain_task/fsl_randomise/bourreau/fsl_randomise.rb b/cbrain_task/fsl_randomise/bourreau/fsl_randomise.rb index e7d68d72..142c74b5 100644 --- a/cbrain_task/fsl_randomise/bourreau/fsl_randomise.rb +++ b/cbrain_task/fsl_randomise/bourreau/fsl_randomise.rb @@ -28,6 +28,10 @@ class CbrainTask::FslRandomise < ClusterTask include RestartableTask include RecoverableTask + def self.properties #:nodoc: + super.merge :can_submit_new_tasks => true + end + def setup #:nodoc: params = self.params @@ -103,13 +107,15 @@ def cluster_commands #:nodoc: output_option = "#{output_dir}/#{common_string}" + # FSL randomise_parallel execution commands + # All boolean options with_T = params[:carry_t] == "1" ? "-T" : "" with_F = params[:carry_f] == "1" ? "-F" : "" with_x = params[:output_voxelwise] == "1" ? "-x" : "" with_R = params[:output_raw] == "1" ? "-R" : "" - cmd = "randomise" + cmd = "randomise_parallel" cmd += " -i #{inputfile}" cmd += " -o #{output_option}" cmd += " -d #{mat}" @@ -121,6 +127,12 @@ def cluster_commands #:nodoc: cmd += " -m #{mask}" cmd += " #{with_T} #{with_F} #{with_x} #{with_R}" + # Export of the CBRAIN_WORKDIR variable is used by + # fsl_sub to determine if task has to be parallelized. + # In our case, workdir is exported only for group analyses because + # individual analyses will not be parallelized. + cmds << "export CBRAIN_WORKDIR=#{self.full_cluster_workdir} # To make fsl_sub submit tasks to CBRAIN" + cmds << "export CBRAIN_SHARE_WD_TID=#{self.id}" cmds << "echo Starting Randomise" cmds << "echo running #{cmd}" cmds << cmd diff --git a/cbrain_task_descriptors/fsl_sub.json b/cbrain_task_descriptors/fsl_sub.json index 93e7f284..b18b210a 100644 --- a/cbrain_task_descriptors/fsl_sub.json +++ b/cbrain_task_descriptors/fsl_sub.json @@ -1,16 +1,16 @@ { "name": "fsl_sub", - "tool-version": "1.1", + "tool-version": "2.0", "description": "A wrapper for FSL sub. Assumes that task will run on a file system shared with the submission process, that input files are already in place, and that output files will be handled by another process.", - "command-line": "fsl_sub [FSL_SUB_OPTIONS]", + "command-line": "/bin/bash .new-task-[SUBTASK_ID].sh", "schema-version": "0.2", "inputs": [ { - "id": "fsl_sub_options", - "name": "Options", + "id": "subtask_id", + "name": "Subtask ID", "type": "String", - "description": "Options passed to fsl_sub.", - "command-line-key": "[FSL_SUB_OPTIONS]", + "description": "The ID of a subtask descriptor created by fsl_sub", + "command-line-key": "[SUBTASK_ID]", "list": false, "optional": false, "command-line-flag": "", From e404f294238c9f985dfff0f126c0bfe4a2daa354 Mon Sep 17 00:00:00 2001 From: Natacha Beck Date: Wed, 13 Apr 2016 14:54:29 -0400 Subject: [PATCH 2/3] Fix according to the PR comments --- bin/fsl_sub | 75 ++++++++++++++-------------- cbrain_task_descriptors/fsl_sub.json | 2 +- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/bin/fsl_sub b/bin/fsl_sub index d95604b9..6fa4ea3c 100755 --- a/bin/fsl_sub +++ b/bin/fsl_sub @@ -6,20 +6,20 @@ # Part of FSL - FMRIB's Software Library # http://www.fmrib.ox.ac.uk/fsl # fsl@fmrib.ox.ac.uk -# +# # Developed at FMRIB (Oxford Centre for Functional Magnetic Resonance # Imaging of the Brain), Department of Clinical Neurology, Oxford # University, Oxford, UK -# -# +# +# # LICENCE -# +# # FMRIB Software Library, Release 5.0 (c) 2012, The University of # Oxford (the "Software") -# +# # The Software remains the property of the University of Oxford ("the # University"). -# +# # The Software is distributed "AS IS" under this Licence solely for # non-commercial use in the hope that it will be useful, but in order # that the University as a charitable foundation protects its assets for @@ -31,13 +31,13 @@ # all responsibility for the use which is made of the Software. It # further disclaims any liability for the outcomes arising from using # the Software. -# +# # The Licensee agrees to indemnify the University and hold the # University harmless from and against any and all claims, damages and # liabilities asserted by third parties (including claims for # negligence) which arise directly or indirectly from the use of the # Software or the sale of any products based on the Software. -# +# # No part of the Software may be reproduced, modified, transmitted or # transferred in any form or by any means, electronic or mechanical, # without the express permission of the University. The permission of @@ -48,7 +48,7 @@ # transmitted product. You may be held legally responsible for any # copyright infringement that is caused or encouraged by your failure to # abide by these terms and conditions. -# +# # You are not permitted under this Licence to use this Software # commercially. Use for which any financial return is received shall be # defined as commercial use, and includes (1) integration of all or part @@ -92,18 +92,6 @@ else fi fi -# stop submitted scripts from submitting jobs themselves -if [ "X$FSLSUBALREADYRUN" = "Xtrue" ] ; then - METHOD=NONE - echo "Warning: job on queue attempted to submit parallel jobs - running jobs serially instead" >&2 -fi - -if [ "X$METHOD" = "XNONE" ]; then - QCONF=echo -fi -FSLSUBALREADYRUN=true -export FSLSUBALREADYRUN - ############################# # CBRAIN ADDENDUM #1 # Support for method 'CBRAIN' @@ -118,6 +106,19 @@ if test -n "$CBRAIN_WORKDIR" ; then METHOD="CBRAIN" fi +# stop submitted scripts from submitting jobs themselves +if [ "X$FSLSUBALREADYRUN" = "Xtrue" ] ; then + METHOD=NONE + echo "Warning: job on queue attempted to submit parallel jobs - running jobs serially instead" >&2 +fi + +if [ "X$METHOD" = "XNONE" ]; then + QCONF=echo +fi +FSLSUBALREADYRUN=true +export FSLSUBALREADYRUN + + ########################################################################### # The following auto-decides what cluster queue to use. The calling # FSL program will probably use the -T option when calling fsl_sub, @@ -139,7 +140,7 @@ map_qname () else queue=verylong.q fi - queueCmd=" -q $queue " + queueCmd=" -q $queue " #echo "Estimated time was $1 mins: queue name is $queue" } @@ -166,12 +167,12 @@ $command -q short.q gzip *.img *.hdr $command -a darwin regscript rawdata outputdir ... -T Estimated job length in minutes, used to auto-set queue name - -q Possible values for are "verylong.q", "long.q" + -q Possible values for are "verylong.q", "long.q" and "short.q". See below for details Default is "long.q". -a Architecture [e.g., darwin or lx24-amd64] - -p Lower priority [0:-1024] default = 0 - -M Who to email, default = `whoami`@fmrib.ox.ac.uk + -p Lower priority [0:-1024] default = 0 + -M Who to email, default = `whoami`@fmrib.ox.ac.uk -j Place a hold on this task until job jid has completed -t Specify a task file of commands to execute in parallel -N Specify jobname as it will appear on queue @@ -190,7 +191,7 @@ There are several batch queues configured on the cluster, each with defined CPU time limits. All queues, except bigmem.q, have a 8GB memory limit. veryshort.q:This queue is for jobs which last under 30mins. -short.q: This queue is for jobs which last up to 4h. +short.q: This queue is for jobs which last up to 4h. long.q: This queue is for jobs which last less than 24h. Jobs run with a nice value of 10. verylong.q: This queue is for jobs which will take longer than 24h CPU time. @@ -288,7 +289,7 @@ while [ $1 != -- ] ; do shift;; -q) queue=$2 - queueCmd=" -q $queue " + queueCmd=" -q $queue " $QCONF -sq $queue >/dev/null 2>&1 if [ $? -eq 1 ]; then echo "Invalid queue specified!" @@ -346,7 +347,7 @@ while [ $1 != -- ] ; do -l) LogOpts="-o $2 -e $2"; LogDir="${2}/"; - if [ ! -e ${2} ]; then + if [ ! -e ${2} ]; then mkdir -p $2 else echo ${2} | grep '/dev/null' >/dev/null 2>&1 @@ -389,7 +390,7 @@ if [ -z "$taskfile" ] && [ ! -x "$1" ]; then fi fi -if [ "x$JobName" = x ] ; then +if [ "x$JobName" = x ] ; then if [ "x$taskfile" != x ] ; then JobName=`basename $taskfile` else @@ -405,7 +406,7 @@ fi if [ "x$peName" != "x" ]; then # If the PE name is 'openmp' then limit the number of threads to those specified - + if [ "X$peName" = "X$omp_pe" ]; then OMP_NUM_THREADS=$peThreads export OMP_NUM_THREADS @@ -465,14 +466,14 @@ case $METHOD in else sge_command="qsub $LogOpts $sge_arch $sge_hold" fi - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo sge_command: $sge_command >&2 echo executing: $@ >&2 fi exec $sge_command $@ | awk '{print $3}' else sge_command="qsub -V -cwd $queueCmd $pe_options -M $mailto -N $JobName -m $MailOpts $LogOpts $sge_arch $sge_hold $sge_tasks" - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo sge_command: $sge_command >&2 echo control file: $taskfile >&2 fi @@ -495,7 +496,7 @@ EOF NONE) if [ "x$tasks" = "x" ] ; then - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo executing: $@ >&2 fi @@ -508,14 +509,14 @@ EOF1 exit $ERR fi else - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo "Running commands in: $taskfile" >&2 fi n=1 while [ $n -le $tasks ] ; do line=`sed -n -e ''${n}'p' $taskfile` - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo executing: $line >&2 fi /bin/sh < ${LogDir}${JobName}.o$$.$n 2> ${LogDir}${JobName}.e$$.$n @@ -523,7 +524,7 @@ $line EOF2 n=`expr $n + 1` done - fi + fi echo $$ ;; @@ -580,7 +581,7 @@ BASH_SCRIPT { "tool-class": "CbrainTask::FslSub", "description": "$JobName", -"share_wd_tid": "$CBRAIN_SHARE_WD_TID", +"share-wd-tid": "$CBRAIN_SHARE_WD_TID", "parameters": { "subtask_id": "$sub_id" }, diff --git a/cbrain_task_descriptors/fsl_sub.json b/cbrain_task_descriptors/fsl_sub.json index b18b210a..889170b1 100644 --- a/cbrain_task_descriptors/fsl_sub.json +++ b/cbrain_task_descriptors/fsl_sub.json @@ -2,7 +2,7 @@ "name": "fsl_sub", "tool-version": "2.0", "description": "A wrapper for FSL sub. Assumes that task will run on a file system shared with the submission process, that input files are already in place, and that output files will be handled by another process.", - "command-line": "/bin/bash .new-task-[SUBTASK_ID].sh", + "command-line": "fsl_sub /bin/bash .new-task-[SUBTASK_ID].sh", "schema-version": "0.2", "inputs": [ { From a654211c95d0f16f8858f3c0503bdcb9eb83f2ed Mon Sep 17 00:00:00 2001 From: Natacha Beck Date: Wed, 13 Apr 2016 15:22:55 -0400 Subject: [PATCH 3/3] Revert white space in bin/fsl_sub --- bin/fsl_sub | 49 ++++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/bin/fsl_sub b/bin/fsl_sub index 6fa4ea3c..ddbb1c6c 100755 --- a/bin/fsl_sub +++ b/bin/fsl_sub @@ -6,20 +6,20 @@ # Part of FSL - FMRIB's Software Library # http://www.fmrib.ox.ac.uk/fsl # fsl@fmrib.ox.ac.uk -# +# # Developed at FMRIB (Oxford Centre for Functional Magnetic Resonance # Imaging of the Brain), Department of Clinical Neurology, Oxford # University, Oxford, UK -# -# +# +# # LICENCE -# +# # FMRIB Software Library, Release 5.0 (c) 2012, The University of # Oxford (the "Software") -# +# # The Software remains the property of the University of Oxford ("the # University"). -# +# # The Software is distributed "AS IS" under this Licence solely for # non-commercial use in the hope that it will be useful, but in order # that the University as a charitable foundation protects its assets for @@ -31,13 +31,13 @@ # all responsibility for the use which is made of the Software. It # further disclaims any liability for the outcomes arising from using # the Software. -# +# # The Licensee agrees to indemnify the University and hold the # University harmless from and against any and all claims, damages and # liabilities asserted by third parties (including claims for # negligence) which arise directly or indirectly from the use of the # Software or the sale of any products based on the Software. -# +# # No part of the Software may be reproduced, modified, transmitted or # transferred in any form or by any means, electronic or mechanical, # without the express permission of the University. The permission of @@ -48,7 +48,7 @@ # transmitted product. You may be held legally responsible for any # copyright infringement that is caused or encouraged by your failure to # abide by these terms and conditions. -# +# # You are not permitted under this Licence to use this Software # commercially. Use for which any financial return is received shall be # defined as commercial use, and includes (1) integration of all or part @@ -118,7 +118,6 @@ fi FSLSUBALREADYRUN=true export FSLSUBALREADYRUN - ########################################################################### # The following auto-decides what cluster queue to use. The calling # FSL program will probably use the -T option when calling fsl_sub, @@ -140,7 +139,7 @@ map_qname () else queue=verylong.q fi - queueCmd=" -q $queue " + queueCmd=" -q $queue " #echo "Estimated time was $1 mins: queue name is $queue" } @@ -167,12 +166,12 @@ $command -q short.q gzip *.img *.hdr $command -a darwin regscript rawdata outputdir ... -T Estimated job length in minutes, used to auto-set queue name - -q Possible values for are "verylong.q", "long.q" + -q Possible values for are "verylong.q", "long.q" and "short.q". See below for details Default is "long.q". -a Architecture [e.g., darwin or lx24-amd64] - -p Lower priority [0:-1024] default = 0 - -M Who to email, default = `whoami`@fmrib.ox.ac.uk + -p Lower priority [0:-1024] default = 0 + -M Who to email, default = `whoami`@fmrib.ox.ac.uk -j Place a hold on this task until job jid has completed -t Specify a task file of commands to execute in parallel -N Specify jobname as it will appear on queue @@ -191,7 +190,7 @@ There are several batch queues configured on the cluster, each with defined CPU time limits. All queues, except bigmem.q, have a 8GB memory limit. veryshort.q:This queue is for jobs which last under 30mins. -short.q: This queue is for jobs which last up to 4h. +short.q: This queue is for jobs which last up to 4h. long.q: This queue is for jobs which last less than 24h. Jobs run with a nice value of 10. verylong.q: This queue is for jobs which will take longer than 24h CPU time. @@ -289,7 +288,7 @@ while [ $1 != -- ] ; do shift;; -q) queue=$2 - queueCmd=" -q $queue " + queueCmd=" -q $queue " $QCONF -sq $queue >/dev/null 2>&1 if [ $? -eq 1 ]; then echo "Invalid queue specified!" @@ -347,7 +346,7 @@ while [ $1 != -- ] ; do -l) LogOpts="-o $2 -e $2"; LogDir="${2}/"; - if [ ! -e ${2} ]; then + if [ ! -e ${2} ]; then mkdir -p $2 else echo ${2} | grep '/dev/null' >/dev/null 2>&1 @@ -390,7 +389,7 @@ if [ -z "$taskfile" ] && [ ! -x "$1" ]; then fi fi -if [ "x$JobName" = x ] ; then +if [ "x$JobName" = x ] ; then if [ "x$taskfile" != x ] ; then JobName=`basename $taskfile` else @@ -406,7 +405,7 @@ fi if [ "x$peName" != "x" ]; then # If the PE name is 'openmp' then limit the number of threads to those specified - + if [ "X$peName" = "X$omp_pe" ]; then OMP_NUM_THREADS=$peThreads export OMP_NUM_THREADS @@ -466,14 +465,14 @@ case $METHOD in else sge_command="qsub $LogOpts $sge_arch $sge_hold" fi - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo sge_command: $sge_command >&2 echo executing: $@ >&2 fi exec $sge_command $@ | awk '{print $3}' else sge_command="qsub -V -cwd $queueCmd $pe_options -M $mailto -N $JobName -m $MailOpts $LogOpts $sge_arch $sge_hold $sge_tasks" - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo sge_command: $sge_command >&2 echo control file: $taskfile >&2 fi @@ -496,7 +495,7 @@ EOF NONE) if [ "x$tasks" = "x" ] ; then - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo executing: $@ >&2 fi @@ -509,14 +508,14 @@ EOF1 exit $ERR fi else - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo "Running commands in: $taskfile" >&2 fi n=1 while [ $n -le $tasks ] ; do line=`sed -n -e ''${n}'p' $taskfile` - if [ $verbose -eq 1 ] ; then + if [ $verbose -eq 1 ] ; then echo executing: $line >&2 fi /bin/sh < ${LogDir}${JobName}.o$$.$n 2> ${LogDir}${JobName}.e$$.$n @@ -524,7 +523,7 @@ $line EOF2 n=`expr $n + 1` done - fi + fi echo $$ ;;