Differences

This shows you the differences between two versions of the page.

--- howto:bioinformatics:gnu-parallel [2016/05/12 14:31]
dane [Fault tolerant Example]
+++ howto:bioinformatics:gnu-parallel [2023/04/17 12:40] (current)
ischeepers
@@ Line 8: / Line 8: @@
 ==== Simple Example ====
-What this example does is that it finds a whole lot of scripts in a directory and gets gnu parallel to run them. The -j 6 options tells gnu parallel to assume that each sub-job requires 6 cores.
+What this example does is that it finds a whole lot of scripts in a directory and gets gnu parallel to run them. The -j 6 options tells gnu parallel to run 6 jobs per node, while the -M and --sshdelay 0.2 options tells it to use ssh's ControlMaster and to have a delay between establishing links to the same node.
 <file bash gnu_parallel.qsub>
 #!/bin/bash
 #PBS -e /mnt/lustre/users/USERNAME/test_scripts/gnu_parallel.stderr.out
-#PBS -o /mnt/lsutre/users/USERNAME/test_scripts/gnu_parallel.stdout.out
+#PBS -o /mnt/lustre/users/USERNAME/test_scripts/gnu_parallel.stdout.out
 #PBS -V
 #PBS -P PROGRAMMESHORTNAME
@@ Line 24: / Line 24: @@
 #PBS -mb
-module add chpc/gnu/parallel-20160422
+module add chpc/BIOMODULES
+module add gnu-parallel
 WORKING_DIR=/mnt/lustre/users/USERNAME/test_scripts
@@ Line 31: / Line 33: @@
 cd ${WORKING_DIR}
-ls ${WORKING_DIR}/gnup_scripts/* | parallel -j 6 -u --sshloginfile ${PBS_NODEFILE} "cd ${WORKING_DIR}/gnup_scripts; {} {}"
+ls ${WORKING_DIR}/gnup_scripts/* | parallel -M --sshdelay 0.2 -j 6 -u --sshloginfile ${PBS_NODEFILE} "cd ${WORKING_DIR}/gnup_scripts; {} {}"
 </file>
@@ Line 120: / Line 122: @@
 cd ${PBS_O_WORKDIR}
-module add chpc/gnu/parallel-20160422
 module add chpc/BIOMODULES
-module add ncbi-blast/2.3.0/gcc
+module add blast
+module add gnu-parallel
+#The module sets the env variable below
+# and provides the path to the databases
+#BLASTDB="/mnt/lustre/bsp/DB/BLAST"
+# To blast against eg, simply use -db nt
+# To see all databases, "ls $BLASTDB/*.?al"
-BLASTDB="/lustre/bsp/blast/nt/nt"
 BLASTCMD=$(which blastn)
-BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24"
+BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24 -db nt"
 INPUTDIRS="DATE/*"
-ls ${INPUTDIRS}/*.fasta | parallel -j 24 -u --sshloginfile ${PBS_NODEFILE} "cd ${PBS_O_WORKDIR}; ${BLASTCMD} -db ${BLASTDB} -query {} ${BLASTARGS} -out {}.xml && gzip --best {} {}.xml"
+ls ${INPUTDIRS}/*.fasta | parallel -M --sshdelay 0.2 -j 1 -u --sshloginfile ${PBS_NODEFILE} "cd ${PBS_O_WORKDIR}; ${BLASTCMD} -query {} ${BLASTARGS} -out {}.xml && gzip --best {} {}.xml"
 </file>
@@ Line 141: / Line 149: @@
 <file bash blast.advanced.gnup.qsub>
 #!/bin/bash
-#PBS -e /mnt/lustre/users/USERNAME/blastjobs/blast.stderr.out
+#PBS -e /mnt/lustre/users/USERNAME/blastjobs/blast.advanced.stderr.out
-#PBS -o /mnt/lustre/users/USERNAME/blastjobs/blast.stderr.out
+#PBS -o /mnt/lustre/users/USERNAME/blastjobs/blast.advanced.stderr.out
 #PBS -P PROGRAMMESHORTNAME
 #PBS -M youremail@address
-#PBS -l select=4:ncpus=24:nodetype=haswell_reg
+#PBS -l select=4:ncpus=24:mem=120gb:nodetype=haswell_reg
 #PBS -l walltime=00:60:00
 #PBS -q normal
@@ Line 152: / Line 160: @@
 cd ${PBS_O_WORKDIR}
-module add chpc/gnu/parallel-20160422
 module add chpc/BIOMODULES
-module add ncbi-blast/2.3.0/gcc
+module add blast
+module add gnu-parallel
-#MODULES="chpc/gnu/parallel-20160422 chpc/BIOMODULES ncbi-blast/2.3.0/gcc"
+BLASTDB="/mnt/lustre/bsp/NCBI/BLAST"
-BLASTDBDIR="/lustre/bsp/blast/nt"
+DB="nt"
-BLASTDB="nt"
 BLASTCMD=$(which blastn)
 BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24"
@@ Line 168: / Line 176: @@
 for node in ${NODES}
 do
-  ssh ${node} "mkdir -p /dev/shm/${USER}/BLAST && cp -r ${BLASTDBDIR}/${BLASTDB}* /dev/shm/${USER}/BLAST && echo 'successfully added DBs on ${node}' || exit 1" &
+  ssh ${node} "mkdir -p /dev/shm/${USER}/BLAST && cp -r ${BLASTDB}/${DB}* /dev/shm/${USER}/BLAST && echo 'successfully added DBs on ${node}' || exit 1" &
 done
 wait  # wait for parallel copies to finish
-ls ${INPUTDIRS}/*.fa | parallel -j 24 -u --sshloginfile ${PBS_NODEFILE} "cd ${PBS_O_WORKDIR}; ${BLASTCMD} -db /dev/shm/${USER}/BLAST/${BLASTDB} -query {} ${BLASTARGS} -out {}.xml && gzip --best {} {}.xml"
+ls ${INPUTDIRS}/*.fa | parallel -j 1 -u --sshloginfile ${PBS_NODEFILE} "cd ${PBS_O_WORKDIR}; ${BLASTCMD} -db /dev/shm/${USER}/BLAST/${BLASTDB} -query {} ${BLASTARGS} -out {}.xml && gzip --best {} {}.xml"
 # clean up ram disk
@@ Line 182: / Line 190: @@
 wait
+</file>
+==== Advanced Generalised Gnu Parallel ====
+The idea behind this example is that you can create a list of commands that need running and then give this to the script. If sub-jobs fail there will be an easily parsed log file so that individual steps can be quickly identified. If sub-jobs have been successfully run, they're logged as such and aren't re-run on subsequent job submissions.
+This first script is a list of helper functions that are used in the main script.
+<file bash log_support.sh>
+#!/bin/bash
+##  Some simple helper log functions to track jobs success / failure   ##
+## Created by Dane Kennedy @ the Centre for High Performance Computing ##
+############## Create some useful functions ##############
+# Echos the current date/time in a nice format
+function now { echo -n "$( date +"%F %X" )"; }
+# Appends to log file
+function log () {
+  if [[ -v LOGFILE ]]
+  then
+    echo -e "$( now ): ${HOSTNAME}: $@" >> "${LOGFILE}"
+  else
+    echo -e "$( now ): ${HOSTNAME}: $@"
+  fi
+}
+#Appends to log file and exits
+function log_fail () {
+  log "$@"
+  exit 1
+}
+# returns true (in the bash sense of 0 exit status meaning success) if line exists in log file
+function check_log () {
+  if [[ -v LOGFILE ]]
+  then
+    if [[ -e ${LOGFILE} ]]
+    then
+      if $( grep -q "$@" ${LOGFILE} )
+      then
+        return 0
+      fi
+    fi
+  fi
+  return 1
+}
+# Checks for a line on the LOG file. If is exists, it doesn't repeat. If it's not there, it runs. If
+# the run is successful is records it as such.
+function check_run (){
+  SUCCESS_LINE="\"$@\" SUCCESSUL"
+  FAIL_LINE="\"$@\" fail."
+  if check_log "${SUCCESS_LINE}"
+  then
+    log "\"$@\" already successfully run. Not repeating."
+    return 0
+  fi
+  log "Running \"$@\""
+  $@ \
+    && { log "${SUCCESS_LINE}"; return 0; } \
+    || { log "${FAIL_LINE}"; return 1; }
+}
+# Same as above but exit 1's on fail.
+function check_run_abort (){
+  if ! check_run "$@"
+  then
+    log "Aborting."
+  fi
+}
+</file>
+<file bash myjob.sh>
+#!/bin/bash
+#PBS -e /mnt/lustre/users/USERNAME/gnup/stderr.out
+#PBS -o /mnt/lustre/users/USERNAME/gnup/stdout.out
+#PBS -P PROGRAMMESHORTNAME
+#PBS -M youremailaddress
+#PBS -l select=2:ncpus=24:mpiprocs=16:nodetype=haswell_reg
+#PBS -l walltime=48:00:00
+#PBS -q normal
+#PBS -m be
+#PBS -r n
+module add chpc/BIOMODULES
+module add gnu-parallel
+JOBSPERNODE=16
+# Make sure we start the job in the right place.
+cd -P ${PBS_O_WORKDIR}
+# Set up logging stuffs
+source log_support.sh
+LOGFILE="gnuparallel.text.log"
+THIS="GNU PARALLEL TEST"
+# First check if analysis has been run before. If it has abort...
+if check_log "${THIS} COMPLETED SUCCESSFULLY"
+then
+        log "${THIS} already successfully completed. Not repeating."
+        exit 0
+else
+        log "Beginning ${THIS}"
+fi
+# Point to Job File -- this contains a list of commands to run, one per line
+JOBFILE="gnu_parallel.jobs"
+# pass the commands on to gnu parallel which runs them with "check_run". It will record
+# a successful completion if all sub-jobs complete successfully :-). Woot.
+cat ${JOBFILE} | parallel -M --sshdelay 0.2 -j ${JOBSPERNODE} -u --sshloginfile ${PBS_NODEFILE} \
+  "cd -P \"${PBS_O_WORKDIR}\"; . log_support.sh; LOGFILE=\"${LOGFILE}\"; check_run {}" \
+  && log "${THIS} COMPLETED SUCCESSFULLY" \
+  || log "${THIS} incomplete."
 </file>

wiki.chpc.ac.za

User Tools

Site Tools

Differences

Page Tools