#!/bin/bash
#PBS -e /mnt/lustre/users/USERNAME/gnup/stderr.out
#PBS -o /mnt/lustre/users/USERNAME/gnup/stdout.out
#PBS -P PROGRAMMESHORTNAME
#PBS -M youremailaddress
#PBS -l select=2:ncpus=24:mpiprocs=16:nodetype=haswell_reg
#PBS -l walltime=48:00:00
#PBS -q normal
#PBS -m be
#PBS -r n

module add chpc/BIOMODULES
module add gnu-parallel

JOBSPERNODE=16

# Make sure we start the job in the right place.
cd -P ${PBS_O_WORKDIR}

# Set up logging stuffs
source log_support.sh
LOGFILE="gnuparallel.text.log"
THIS="GNU PARALLEL TEST"

# First check if analysis has been run before. If it has abort...
if check_log "${THIS} COMPLETED SUCCESSFULLY"
then
        log "${THIS} already successfully completed. Not repeating."
        exit 0
else
        log "Beginning ${THIS}"
fi

# Point to Job File -- this contains a list of commands to run, one per line 
JOBFILE="gnu_parallel.jobs"

# pass the commands on to gnu parallel which runs them with "check_run". It will record
# a successful completion if all sub-jobs complete successfully :-). Woot.

cat ${JOBFILE} | parallel -M --sshdelay 0.2 -j ${JOBSPERNODE} -u --sshloginfile ${PBS_NODEFILE} \
  "cd -P \"${PBS_O_WORKDIR}\"; . log_support.sh; LOGFILE=\"${LOGFILE}\"; check_run {}" \
  && log "${THIS} COMPLETED SUCCESSFULLY" \
  || log "${THIS} incomplete."