#!/bin/bash
#PBS -e /mnt/lustre/users/USERNAME/blastjobs/blast.advanced.stderr.out
#PBS -o /mnt/lustre/users/USERNAME/blastjobs/blast.advanced.stderr.out
#PBS -P PROGRAMMESHORTNAME
#PBS -M youremail@address
#PBS -l select=4:ncpus=24:mem=120gb:nodetype=haswell_reg
#PBS -l walltime=00:60:00
#PBS -q normal
#PBS -m be

cd ${PBS_O_WORKDIR}


module add chpc/BIOMODULES
module add blast
module add gnu-parallel

BLASTDB="/mnt/lustre/bsp/NCBI/BLAST"
DB="nt"
BLASTCMD=$(which blastn)
BLASTARGS="-evalue 0.005 -num_alignments 20 -outfmt 5 -num_threads 24"
INPUTDIRS="2016-05-12/*"

NODES=$(cat ${PBS_NODEFILE} | sort | uniq)

# copy blast databases to ram disk
for node in ${NODES}
do
  ssh ${node} "mkdir -p /dev/shm/${USER}/BLAST && cp -r ${BLASTDB}/${DB}* /dev/shm/${USER}/BLAST && echo 'successfully added DBs on ${node}' || exit 1" &
done

wait  # wait for parallel copies to finish

ls ${INPUTDIRS}/*.fa | parallel -j 1 -u --sshloginfile ${PBS_NODEFILE} "cd ${PBS_O_WORKDIR}; ${BLASTCMD} -db /dev/shm/${USER}/BLAST/${BLASTDB} -query {} ${BLASTARGS} -out {}.xml && gzip --best {} {}.xml"

# clean up ram disk
for node in ${NODES}
do
  ssh ${node} "rm -rf /dev/shm/${USER}/BLAST && echo 'successfully deleted DBs on ${node}' || exit 1" &
done

wait