wiki.chpc.ac.za

User Tools

Site Tools

You are here: start » howto » bioinformatics » gnu-parallel
Trace:
howto:bioinformatics:gnu-parallel

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revision Previous revision
Next revision 		Previous revision
howto:bioinformatics:gnu-parallel [2016/05/12 15:26]
dane fixed my understanding of "-j" parameter 		howto:bioinformatics:gnu-parallel [2023/04/17 12:40] (current)
ischeepers
Line 8: Line 8:
 ==== Simple Example ==== ==== Simple Example ====
  
-What this example does is that it finds a whole lot of scripts in a directory and gets gnu parallel to run them. The -j 6 options tells gnu parallel to assume that each sub-job requires 6 cores.+What this example does is that it finds a whole lot of scripts in a directory and gets gnu parallel to run them. The -j 6 options tells gnu parallel to run 6 jobs per node, while the -M and --sshdelay 0.2 options tells it to use ssh's ControlMaster and to have a delay between establishing links to the same node.
  
 <file bash gnu_parallel.qsub> <file bash gnu_parallel.qsub>
 #!/bin/bash #!/bin/bash
 #PBS -e /mnt/lustre/users/USERNAME/test_scripts/gnu_parallel.stderr.out #PBS -e /mnt/lustre/users/USERNAME/test_scripts/gnu_parallel.stderr.out
-#PBS -o /mnt/lsutre/users/USERNAME/test_scripts/gnu_parallel.stdout.out+#PBS -o /mnt/lustre/users/USERNAME/test_scripts/gnu_parallel.stdout.out
 #PBS -V #PBS -V
 #PBS -P PROGRAMMESHORTNAME #PBS -P PROGRAMMESHORTNAME
Line 24: Line 24:
 #PBS -mb #PBS -mb
  
-module add chpc/gnu/parallel-20160422+ 
 +module add chpc/BIOMODULES 
 +module add gnu-parallel
  
 WORKING_DIR=/mnt/lustre/users/USERNAME/test_scripts WORKING_DIR=/mnt/lustre/users/USERNAME/test_scripts
Line 31: Line 33:
 cd ${WORKING_DIR} cd ${WORKING_DIR}
  
-ls ${WORKING_DIR}/gnup_scripts/* | parallel -j 6 -u --sshloginfile ${PBS_NODEFILE} "cd ${WORKING_DIR}/gnup_scripts; {} {}"+ls ${WORKING_DIR}/gnup_scripts/* | parallel -M --sshdelay 0.2 -j 6 -u --sshloginfile ${PBS_NODEFILE} "cd ${WORKING_DIR}/gnup_scripts; {} {}"
 </file> </file>
  
Line 120: Line 122:
 cd ${PBS_O_WORKDIR} cd ${PBS_O_WORKDIR}
  
-module add chpc/gnu/parallel-20160422 
 module add chpc/BIOMODULES module add chpc/BIOMODULES
-module add ncbi-blast/2.3.0/gcc+module add blast 
 +module add gnu-parallel 
 + 
 + 
 +#The module sets the env variable below 
 +# and provides the path to the databases 
 +#BLASTDB="/mnt/lustre/bsp/DB/BLAST" 
 +# To blast against eg, simply use -db nt 
 +# To see all databases, "ls $BLASTDB/*.?al"
  
-BLASTDB="/lustre/bsp/blast/nt/nt" 
 BLASTCMD=$(which blastn) BLASTCMD=$(which blastn)
-BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24"+BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24 -db nt"
 INPUTDIRS="DATE/*" INPUTDIRS="DATE/*"
  
-ls ${INPUTDIRS}/*.fasta | parallel -j 1 -u --sshloginfile ${PBS_NODEFILE} "cd ${PBS_O_WORKDIR}; ${BLASTCMD} -db ${BLASTDB} -query {} ${BLASTARGS} -out {}.xml && gzip --best {} {}.xml"+ls ${INPUTDIRS}/*.fasta | parallel -M --sshdelay 0.2 -j 1 -u --sshloginfile ${PBS_NODEFILE} "cd ${PBS_O_WORKDIR}; ${BLASTCMD} -query {} ${BLASTARGS} -out {}.xml && gzip --best {} {}.xml"
 </file> </file>
  
Line 145: Line 153:
 #PBS -P PROGRAMMESHORTNAME #PBS -P PROGRAMMESHORTNAME
 #PBS -M youremail@address #PBS -M youremail@address
-#PBS -l select=4:ncpus=24:nodetype=haswell_reg+#PBS -l select=4:ncpus=24:mem=120gb:nodetype=haswell_reg
 #PBS -l walltime=00:60:00 #PBS -l walltime=00:60:00
 #PBS -q normal #PBS -q normal
Line 152: Line 160:
 cd ${PBS_O_WORKDIR} cd ${PBS_O_WORKDIR}
  
-module add chpc/gnu/parallel-20160422+
 module add chpc/BIOMODULES module add chpc/BIOMODULES
-module add ncbi-blast/2.3.0/gcc+module add blast 
 +module add gnu-parallel
  
-BLASTDBDIR="/lustre/bsp/blast/nt" +BLASTDB="/mnt/lustre/bsp/NCBI/BLAST" 
-BLASTDB="nt"+DB="nt"
 BLASTCMD=$(which blastn) BLASTCMD=$(which blastn)
 BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24" BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24"
Line 167: Line 176:
 for node in ${NODES} for node in ${NODES}
 do do
-  ssh ${node} "mkdir -p /dev/shm/${USER}/BLAST && cp -r ${BLASTDBDIR}/${BLASTDB}* /dev/shm/${USER}/BLAST && echo 'successfully added DBs on ${node}' || exit 1" &+  ssh ${node} "mkdir -p /dev/shm/${USER}/BLAST && cp -r ${BLASTDB}/${DB}* /dev/shm/${USER}/BLAST && echo 'successfully added DBs on ${node}' || exit 1" &
 done done
  
Line 181: Line 190:
  
 wait wait
 +</file>
 +
 +==== Advanced Generalised Gnu Parallel ====
 +
 +The idea behind this example is that you can create a list of commands that need running and then give this to the script. If sub-jobs fail there will be an easily parsed log file so that individual steps can be quickly identified. If sub-jobs have been successfully run, they're logged as such and aren't re-run on subsequent job submissions.
 +
 +This first script is a list of helper functions that are used in the main script.
 +
 +<file bash log_support.sh>
 +#!/bin/bash
 +
 +##  Some simple helper log functions to track jobs success / failure   ##
 +## Created by Dane Kennedy @ the Centre for High Performance Computing ##
 +
 +############## Create some useful functions ##############
 +
 +# Echos the current date/time in a nice format
 +function now { echo -n "$( date +"%F %X" )"; }
 +
 +# Appends to log file
 +function log () {
 +  if [[ -v LOGFILE ]]
 +  then
 +    echo -e "$( now ): ${HOSTNAME}: $@" >> "${LOGFILE}"
 +  else
 +    echo -e "$( now ): ${HOSTNAME}: $@"
 +  fi
 +}
 +
 +#Appends to log file and exits
 +function log_fail () {
 +  log "$@"
 +  exit 1
 +}
 +
 +# returns true (in the bash sense of 0 exit status meaning success) if line exists in log file
 +function check_log () {
 +  if [[ -v LOGFILE ]]
 +  then
 +    if [[ -e ${LOGFILE} ]]
 +    then
 +      if $( grep -q "$@" ${LOGFILE} )
 +      then
 +        return 0
 +      fi
 +    fi
 +  fi
 +  return 1
 +}
 +
 +# Checks for a line on the LOG file. If is exists, it doesn't repeat. If it's not there, it runs. If
 +# the run is successful is records it as such.
 +function check_run (){
 +  SUCCESS_LINE="\"$@\" SUCCESSUL"
 +  FAIL_LINE="\"$@\" fail."
 +  if check_log "${SUCCESS_LINE}"
 +  then
 +    log "\"$@\" already successfully run. Not repeating."
 +    return 0
 +  fi
 +  log "Running \"$@\""
 +  $@ \
 +    && { log "${SUCCESS_LINE}"; return 0; } \
 +    || { log "${FAIL_LINE}"; return 1; }
 +}
 +
 +
 +# Same as above but exit 1's on fail.
 +function check_run_abort (){
 +  if ! check_run "$@"
 +  then
 +    log "Aborting."
 +  fi
 +}
 +
 +</file>
 +
 +<file bash myjob.sh>
 +#!/bin/bash
 +#PBS -e /mnt/lustre/users/USERNAME/gnup/stderr.out
 +#PBS -o /mnt/lustre/users/USERNAME/gnup/stdout.out
 +#PBS -P PROGRAMMESHORTNAME
 +#PBS -M youremailaddress
 +#PBS -l select=2:ncpus=24:mpiprocs=16:nodetype=haswell_reg
 +#PBS -l walltime=48:00:00
 +#PBS -q normal
 +#PBS -m be
 +#PBS -r n
 +
 +module add chpc/BIOMODULES
 +module add gnu-parallel
 +
 +JOBSPERNODE=16
 +
 +# Make sure we start the job in the right place.
 +cd -P ${PBS_O_WORKDIR}
 +
 +# Set up logging stuffs
 +source log_support.sh
 +LOGFILE="gnuparallel.text.log"
 +THIS="GNU PARALLEL TEST"
 +
 +# First check if analysis has been run before. If it has abort...
 +if check_log "${THIS} COMPLETED SUCCESSFULLY"
 +then
 +        log "${THIS} already successfully completed. Not repeating."
 +        exit 0
 +else
 +        log "Beginning ${THIS}"
 +fi
 +
 +# Point to Job File -- this contains a list of commands to run, one per line 
 +JOBFILE="gnu_parallel.jobs"
 +
 +# pass the commands on to gnu parallel which runs them with "check_run". It will record
 +# a successful completion if all sub-jobs complete successfully :-). Woot.
 +
 +cat ${JOBFILE} | parallel -M --sshdelay 0.2 -j ${JOBSPERNODE} -u --sshloginfile ${PBS_NODEFILE} \
 +  "cd -P \"${PBS_O_WORKDIR}\"; . log_support.sh; LOGFILE=\"${LOGFILE}\"; check_run {}" \
 +  && log "${THIS} COMPLETED SUCCESSFULLY" \
 +  || log "${THIS} incomplete."
 </file> </file>
  
/app/dokuwiki/data/attic/howto/bioinformatics/gnu-parallel.1463059618.txt.gz · Last modified: 2021/12/09 16:42 (external edit)

Page Tools

Except where otherwise noted, content on this wiki is licensed under the following license: CC Attribution-Noncommercial-Share Alike 3.0 Unported
CC Attribution-Noncommercial-Share Alike 3.0 Unported Donate Powered by PHP Valid HTML5 Valid CSS Driven by DokuWiki