#!/bin/bash 
### This 72-way example calls
### for 3 (virtual) nodes, each with 24 processor cores
### Please note that it is necessary to specify both ncpus and mpiprocs, and 
### for OpenFOAM these should be identical to each other.
### For your own benefit, try to estimate a realistic walltime request.  Over-estimating the 
### wallclock requirement interferes with efficient scheduling, will delay the launch of the job,
### and ties up more of your CPU-time allocation untill the job has finished.
#PBS -P projectid
#PBS -l select=3:ncpus=24:mpiprocs=24:nodetype=haswell_reg
#PBS -q normal
#PBS -l walltime=01:00:00
#PBS -o /mnt/lustre/users/username/foamJobs/job01/stdout
#PBS -e /mnt/lustre/users/username/foamJobs/job01/stderr
### Source the openFOAM environment:
. /home/apps/chpc/compmech/OpenFOAM/OF2512
# Set this environment variable explicitly.
export PBS_JOBDIR=/mnt/lustre/users/username/foamJobs/job01
# Explicitly change to the job directory
cd $PBS_JOBDIR
# Count the number of lines in the hosts file to get the number of processors
nproc=`cat $PBS_NODEFILE | wc -l `
#### These next statements build an appropriate decomposeParDict file
#### based on the requested nunber of processors
echo "FoamFile" > system/decomposeParDict
echo "{"  >> system/decomposeParDict
echo "  version             2.0;" >> system/decomposeParDict
echo "  format            ascii;" >> system/decomposeParDict
echo "  class        dictionary;" >> system/decomposeParDict
echo "  object decomposeParDict;" >> system/decomposeParDict
echo "}"  >> system/decomposeParDict
echo "numberOfSubdomains " $nproc ";" >> system/decomposeParDict
echo "method scotch;" >> system/decomposeParDict
#### End of decomposeParDict file
# Decompose the case into processor directories
decomposePar -force > decompose.out
# Run the solver, simpleFoam in this case
mpirun -np $nproc simpleFoam -parallel > simpleFoam.out
# Reconstruct the time directories from the processor directories
reconstructPar > reconstruct.out
# Clean up the processor directories
rm -r processor*