###These lines are for Moab
#MSUB -l nodes=2:ppn=16:gpus=4
#MSUB -l feature=c2070
#MSUB -l walltime=168:00:00
#MSUB -m be
#MSUB -V
#MSUB -o /GPU/home/username/stdout
#MSUB -e /GPU/home/username/stderr
#MSUB -d /GPU/home/username
#MSUB -mb
##### Running commands
echo "original machine file is:"
echo "++++++++++"
cat $PBS_NODEFILE
echo "++++++++++"
cat $PBS_NODEFILE|sed -e 's/.*/&-ib/'>$PBS_STEP_OUT.hostfile
echo "modified machine file is:"
echo "++++++++++"
cat $PBS_STEP_OUT.hostfile
exe=location of the executable
nproc=`cat $PBS_NODEFILE | wc -l`
cd /GPU/home/username/testjob/
mpirun $nproc $exe > /GPU/home/username/testjob/OUTPUT