Differences

This shows you the differences between two versions of the page.

--- howto:bioinformatics:gnu-parallel [2016/05/06 09:45]
dane [Fault tolerant Example]
+++ howto:bioinformatics:gnu-parallel [2023/04/17 12:40] (current)
ischeepers
@@ Line 8: / Line 8: @@
 ==== Simple Example ====
-What this example does is that it finds a whole lot of scripts in a directory and gets gnu parallel to run them. The -j 6 options tells gnu parallel to assume that each sub-job requires 6 cores.
+What this example does is that it finds a whole lot of scripts in a directory and gets gnu parallel to run them. The -j 6 options tells gnu parallel to run 6 jobs per node, while the -M and --sshdelay 0.2 options tells it to use ssh's ControlMaster and to have a delay between establishing links to the same node.
 <file bash gnu_parallel.qsub>
 #!/bin/bash
 #PBS -e /mnt/lustre/users/USERNAME/test_scripts/gnu_parallel.stderr.out
-#PBS -o /mnt/lsutre/users/USERNAME/test_scripts/gnu_parallel.stdout.out
+#PBS -o /mnt/lustre/users/USERNAME/test_scripts/gnu_parallel.stdout.out
 #PBS -V
 #PBS -P PROGRAMMESHORTNAME
@@ Line 24: / Line 24: @@
 #PBS -mb
-module add chpc/gnu/parallel-20160422
+module add chpc/BIOMODULES
+module add gnu-parallel
 WORKING_DIR=/mnt/lustre/users/USERNAME/test_scripts
@@ Line 31: / Line 33: @@
 cd ${WORKING_DIR}
-ls ${WORKING_DIR}/gnup_scripts/* | parallel -j 6 -u --sshloginfile ${PBS_NODEFILE} "cd ${WORKING_DIR}/gnup_scripts; {} {}"
+ls ${WORKING_DIR}/gnup_scripts/* | parallel -M --sshdelay 0.2 -j 6 -u --sshloginfile ${PBS_NODEFILE} "cd ${WORKING_DIR}/gnup_scripts; {} {}"
 </file>
@@ Line 43: / Line 45: @@
 </file> and make sure they are all runnable, i.e. <code bash>chmod u+x gnup.test.sh</code>
-After running <code bash>qsub gnu_parallel.qsub</code> if you look inside <code bash>/mnt/lsutre/users/USERNAME/test_scripts/gnu_parallel.stdout.out</code> you should see something like: <file bash>
+After running <code bash>qsub gnu_parallel.qsub</code> if you look inside <code bash>/mnt/lsutre/users/USERNAME/test_scripts/gnu_parallel.stdout.out</code> you should see something like: <file>
 Hello World! Main gnu parallel test thingy running here
 Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.12.sh :-)
 Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.10.sh :-)
-Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.13.sh :-)
+.
-Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.14.sh :-)
+.
-Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.15.sh :-)
+.
-Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.16.sh :-)
-Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.17.sh :-)
-Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.18.sh :-)
-Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.19.sh :-)
-Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.1.sh :-)
-Hello World! The time is 05/05/2016 15:24:26 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.20.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.11.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.21.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.23.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.22.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.24.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.25.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.26.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.27.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.28.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.29.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.2.sh :-)
-Hello World! The time is 05/05/2016 15:24:27 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.30.sh :-)
-Hello World! The time is 05/05/2016 15:24:28 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.31.sh :-)
-Hello World! The time is 05/05/2016 15:24:28 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.32.sh :-)
-Hello World! The time is 05/05/2016 15:24:28 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.33.sh :-)
-Hello World! The time is 05/05/2016 15:24:28 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.34.sh :-)
-Hello World! The time is 05/05/2016 15:24:28 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.35.sh :-)
-Hello World! The time is 05/05/2016 15:24:28 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.36.sh :-)
-Hello World! The time is 05/05/2016 15:24:29 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.3.sh :-)
-Hello World! The time is 05/05/2016 15:24:29 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.4.sh :-)
-Hello World! The time is 05/05/2016 15:24:29 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.5.sh :-)
-Hello World! The time is 05/05/2016 15:24:29 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.6.sh :-)
-Hello World! The time is 05/05/2016 15:24:29 and I'm running on host: cnode0282. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.7.sh :-)
-Hello World! The time is 05/05/2016 15:24:29 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.8.sh :-)
 Hello World! The time is 05/05/2016 15:24:29 and I'm running on host: cnode0281. I'm running task /home/dane/test_scripts/gnup_scripts/gnup.test.9.sh :-)
 </file>
@@ Line 108: / Line 80: @@
 </file>
-So if we run our same PBS script, but this time point at a directory containing, say 50, copies of the above script then we see: *will follow as soon as cluster is online again"
+So if we run our same PBS script, but this time point at a directory containing, say 50, copies of the above script then we see:
+<code bash>
+[dane@login1]$ ls gnup_scripts/*.sh | wc -l; ls gnup_scripts/*.log | wc -l
+
+
+[dane@login1]$ cat gnup_scripts/*.log | grep STARTING | wc -l; cat gnup_scripts/*.log | grep COMPLETE | wc -l ; cat gnup_scripts/*.log | grep "NOT REPEATING" | wc -l
+
+
+
+[dane@login1]$ cat gnu_parallel.stdout.out | grep "SUCCESS" | wc -l; cat gnu_parallel.stdout.out | grep "Failure" | wc -l
+
+
+[dane@login1]$
+</code>
+So we can see that it ran 50 times, and failed 16 out of 50 times. We can run again and we see:
+<code bash>
+[dane@login1]$ cat gnup_scripts/*.log | grep STARTING | wc -l; cat gnup_scripts/*.log | grep COMPLETE | wc -l ; cat gnup_scripts/*.log | grep "NOT REPEATING" | wc -l
+
+
+
+[dane@login1]$ cat gnu_parallel.stdout.out | grep "SUCCESS" | wc -l; cat gnu_parallel.stdout.out | grep "Failure" | wc -l
+
+
+</code> i.e. we successfully ran another 11 jobs...
+==== Fault tolerant Blast ====
+So the idea around this script is that one has a working directory where you in turn have tree structure that might look something like <code bash>DATE/EXPERIMENT/*.fasta</code>. Here gnu parallel simply takes as input each fasta file and does the blast on each with the parameters provided.
+<file bash blast.gnup.qsub>
+#!/bin/bash
+#PBS -e /mnt/lustre/users/USERNAME/blastjobs/blast.stderr.out
+#PBS -o /mnt/lustre/users/USERNAME/blastjobs/blast.stderr.out
+#PBS -P PROGRAMMESHORTNAME
+#PBS -M youremail@address
+#PBS -l select=4:ncpus=24:nodetype=haswell_reg
+#PBS -l walltime=00:60:00
+#PBS -q normal
+#PBS -m be
+cd ${PBS_O_WORKDIR}
+module add chpc/BIOMODULES
+module add blast
+module add gnu-parallel
+#The module sets the env variable below
+# and provides the path to the databases
+#BLASTDB="/mnt/lustre/bsp/DB/BLAST"
+# To blast against eg, simply use -db nt
+# To see all databases, "ls $BLASTDB/*.?al"
+BLASTCMD=$(which blastn)
+BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24 -db nt"
+INPUTDIRS="DATE/*"
+ls ${INPUTDIRS}/*.fasta | parallel -M --sshdelay 0.2 -j 1 -u --sshloginfile ${PBS_NODEFILE} "cd ${PBS_O_WORKDIR}; ${BLASTCMD} -query {} ${BLASTARGS} -out {}.xml && gzip --best {} {}.xml"
+</file>
+The idea around the fault tolerance is very simple -- when the blast has been successfully run, the file will be zipped and given a .gz suffix and won't be found in subsequent blasts (which explicitly look for .fasta).
+==== Advanced Fault tolerant Blast ====
+This takes the previous example one step further and is useful for large jobs with very many independent fasta files (and this individual blasts). The added step involves copying the blast database into ram disk at the beginning of the job.
+<file bash blast.advanced.gnup.qsub>
+#!/bin/bash
+#PBS -e /mnt/lustre/users/USERNAME/blastjobs/blast.advanced.stderr.out
+#PBS -o /mnt/lustre/users/USERNAME/blastjobs/blast.advanced.stderr.out
+#PBS -P PROGRAMMESHORTNAME
+#PBS -M youremail@address
+#PBS -l select=4:ncpus=24:mem=120gb:nodetype=haswell_reg
+#PBS -l walltime=00:60:00
+#PBS -q normal
+#PBS -m be
+cd ${PBS_O_WORKDIR}
+module add chpc/BIOMODULES
+module add blast
+module add gnu-parallel
+BLASTDB="/mnt/lustre/bsp/NCBI/BLAST"
+DB="nt"
+BLASTCMD=$(which blastn)
+BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24"
+INPUTDIRS="2016-05-12/*"
+NODES=$(cat ${PBS_NODEFILE} | sort | uniq)
+# copy blast databases to ram disk
+for node in ${NODES}
+do
+  ssh ${node} "mkdir -p /dev/shm/${USER}/BLAST && cp -r ${BLASTDB}/${DB}* /dev/shm/${USER}/BLAST && echo 'successfully added DBs on ${node}' || exit 1" &
+done
+wait  # wait for parallel copies to finish
+ls ${INPUTDIRS}/*.fa | parallel -j 1 -u --sshloginfile ${PBS_NODEFILE} "cd ${PBS_O_WORKDIR}; ${BLASTCMD} -db /dev/shm/${USER}/BLAST/${BLASTDB} -query {} ${BLASTARGS} -out {}.xml && gzip --best {} {}.xml"
+# clean up ram disk
+for node in ${NODES}
+do
+  ssh ${node} "rm -rf /dev/shm/${USER}/BLAST && echo 'successfully deleted DBs on ${node}' || exit 1" &
+done
+wait
+</file>
+==== Advanced Generalised Gnu Parallel ====
+The idea behind this example is that you can create a list of commands that need running and then give this to the script. If sub-jobs fail there will be an easily parsed log file so that individual steps can be quickly identified. If sub-jobs have been successfully run, they're logged as such and aren't re-run on subsequent job submissions.
+This first script is a list of helper functions that are used in the main script.
+<file bash log_support.sh>
+#!/bin/bash
+##  Some simple helper log functions to track jobs success / failure   ##
+## Created by Dane Kennedy @ the Centre for High Performance Computing ##
+############## Create some useful functions ##############
+# Echos the current date/time in a nice format
+function now { echo -n "$( date +"%F %X" )"; }
+# Appends to log file
+function log () {
+  if [[ -v LOGFILE ]]
+  then
+    echo -e "$( now ): ${HOSTNAME}: $@" >> "${LOGFILE}"
+  else
+    echo -e "$( now ): ${HOSTNAME}: $@"
+  fi
+}
+#Appends to log file and exits
+function log_fail () {
+  log "$@"
+  exit 1
+}
+# returns true (in the bash sense of 0 exit status meaning success) if line exists in log file
+function check_log () {
+  if [[ -v LOGFILE ]]
+  then
+    if [[ -e ${LOGFILE} ]]
+    then
+      if $( grep -q "$@" ${LOGFILE} )
+      then
+        return 0
+      fi
+    fi
+  fi
+  return 1
+}
+# Checks for a line on the LOG file. If is exists, it doesn't repeat. If it's not there, it runs. If
+# the run is successful is records it as such.
+function check_run (){
+  SUCCESS_LINE="\"$@\" SUCCESSUL"
+  FAIL_LINE="\"$@\" fail."
+  if check_log "${SUCCESS_LINE}"
+  then
+    log "\"$@\" already successfully run. Not repeating."
+    return 0
+  fi
+  log "Running \"$@\""
+  $@ \
+    && { log "${SUCCESS_LINE}"; return 0; } \
+    || { log "${FAIL_LINE}"; return 1; }
+}
+# Same as above but exit 1's on fail.
+function check_run_abort (){
+  if ! check_run "$@"
+  then
+    log "Aborting."
+  fi
+}
+</file>
+<file bash myjob.sh>
+#!/bin/bash
+#PBS -e /mnt/lustre/users/USERNAME/gnup/stderr.out
+#PBS -o /mnt/lustre/users/USERNAME/gnup/stdout.out
+#PBS -P PROGRAMMESHORTNAME
+#PBS -M youremailaddress
+#PBS -l select=2:ncpus=24:mpiprocs=16:nodetype=haswell_reg
+#PBS -l walltime=48:00:00
+#PBS -q normal
+#PBS -m be
+#PBS -r n
+module add chpc/BIOMODULES
+module add gnu-parallel
+JOBSPERNODE=16
+# Make sure we start the job in the right place.
+cd -P ${PBS_O_WORKDIR}
+# Set up logging stuffs
+source log_support.sh
+LOGFILE="gnuparallel.text.log"
+THIS="GNU PARALLEL TEST"
+# First check if analysis has been run before. If it has abort...
+if check_log "${THIS} COMPLETED SUCCESSFULLY"
+then
+        log "${THIS} already successfully completed. Not repeating."
+        exit 0
+else
+        log "Beginning ${THIS}"
+fi
+# Point to Job File -- this contains a list of commands to run, one per line
+JOBFILE="gnu_parallel.jobs"
+# pass the commands on to gnu parallel which runs them with "check_run". It will record
+# a successful completion if all sub-jobs complete successfully :-). Woot.
+cat ${JOBFILE} | parallel -M --sshdelay 0.2 -j ${JOBSPERNODE} -u --sshloginfile ${PBS_NODEFILE} \
+  "cd -P \"${PBS_O_WORKDIR}\"; . log_support.sh; LOGFILE=\"${LOGFILE}\"; check_run {}" \
+  && log "${THIS} COMPLETED SUCCESSFULLY" \
+  || log "${THIS} incomplete."
+</file>

wiki.chpc.ac.za

User Tools

Site Tools

Differences

Page Tools