#!/bin/bash
#PBS -l select=2:ncpus=24:mpiprocs=24:nodetype=haswell_reg
#PBS -P MECH1234
#PBS -q normal
#PBS -l walltime=8:00:00
#PBS -o /mnt/lustre/users/username/DMExample/dm.out
#PBS -e /mnt/lustre/users/username/DMExample/dm.err
export STARCCMHOME=/apps/chpc/compmech/CFD/Siemens/15.04.008/STAR-CCM+15.04.008
export PYTHONHOME=$STARCCMHOME/designmanager/Ver2017.04/Python27
export HEEDS_ROOT=$STARCCMHOME/designmanager/Ver2017.04/LX64/solver
export LD_LIBRARY_PATH=$STARCCMHOME/designmanager/Ver2017.04/LX64:$STARCCMHOME/designmanager/Ver2017.04/LX64/solver:$LD_LIBRARY_PATH
export PATH=$STARCCMHOME/designmanager/Ver2017.04/LX64/solver:$STARCCMHOME/star/bin:$PATH
##### Set working directory.
export PBS_JOBDIR=/mnt/lustre3p/users/username/DMExample
cd $PBS_JOBDIR
#### Get the total number of cores
nproc=`cat $PBS_NODEFILE | wc -l`
# Build SSH tunnels for DM license server.  
# Use the existing tunnels on chpclic1 or dtn for the PoD license
# Assume that DM license is in port 27120 on the server license.unseen.ac.za 
# STAR-CCM+ will look first for a PoD license
# Bear in mind that each design point analysis will consume a solver process - this may 
#  consume PoD time very quickly
masternode=`hostname -s`
ssh -f jblogs@chpclic1 -L *:27120:license.unseen.ac.za:27120 -N
ssh -f jblogs@chpclic1 -L *:27121:license.unseen.ac.za:27121 -N
export CDLMD_LICENSE_FILE=1999@dtn:27120@$masternode
export LM_LICENSE_FILE=1999@dtn:27120@$masternode
##### Starccm+ run instruction.
starlaunch jobmanager  --command "$STARCCMHOME/star/bin/starccm+ -licpath 1999@$masternode:27120@$masternode -power -podkey <22-characterPoDKey>  -rsh ssh -batch run -mpi intel -np $nproc -machinefile $PBS_NODEFILE  ExampleDM_Project.dmprj" --slots 0  --batchsystem pbs  >> disasstar.out 2>&1
### Remove the ssh tunnels:
kill -9 `ps ux | grep "ssh -f" |  grep -o -E '[0-9]+' | head -1 | sed -e 's/^0\+//' `
kill -9 `ps ux | grep "ssh -f" |  grep -o -E '[0-9]+' | head -1 | sed -e 's/^0\+//' `