#!/bin/bash 
### This 72-way example calls
### for 3 (virtual) nodes, each with 24 processor cores
### Please note that it is necessary to specify both ncpus and mpiprocs, and 
### for OpenFOAM these should be identical to each other.
### For your own benefit, try to estimate a realistic walltime request.  Over-estimating the 
### wallclock requirement interferes with efficient scheduling, will delay the launch of the job,
### and ties up more of your CPU-time allocation untill the job has finished.
#PBS -P projectid
#PBS -l select=3:ncpus=24:mpiprocs=24:nodetype=haswell_reg
#PBS -q normal
#PBS -l walltime=01:00:00
#PBS -o /mnt/lustre/users/username/foamJobs/job01/stdout
#PBS -e /mnt/lustre/users/username/foamJobs/job01/stderr
#PBS -m abe
#PBS -M username@email.co.za
### Source the openFOAM environment:
. /apps/chpc/compmech/CFD/OpenFOAM/OF2112
##### Running commands
# Set this environment variable explicitly.
export PBS_JOBDIR=/mnt/lustre/users/username/foamJobs/job01
# Explicitly change to the job directory
cd $PBS_JOBDIR
nproc=`cat $PBS_NODEFILE | wc -l`
exe=simpleFoam
#### These next statements build an appropriate decomposeParDict file 
#### based on the requested nunber of nodes
echo "FoamFile" > system/decomposeParDict
echo "{"  >> system/decomposeParDict
echo "  version             2.0;" >> system/decomposeParDict
echo "  format            ascii;" >> system/decomposeParDict
echo "  class        dictionary;" >> system/decomposeParDict
echo "  object decomposeParDict;" >> system/decomposeParDict
echo "}"  >> system/decomposeParDict
echo "numberOfSubdomains " $nproc ";" >> system/decomposeParDict
echo "method scotch;" >> system/decomposeParDict
#### End of decomposeParDict file 
decomposePar -force > decompose.out
## Issue the MPIRUN command.  Omit -iface ib0 when using OpenMPI versions
mpirun -iface ib0 -np $nproc $exe -parallel > foam.out
reconstructPar -latestTime > reconstruct.out
rm -rf processor*