#!/bin/bash
#PBS -S /bin/bash
#PBS -N cpi-example
#PBS -j oe
#PBS -o ./cpi-example.out
#PBS -m n
#PBS -l nodes=3,walltime=00:01:00
#PBS -A projectName
#PBS -q jpsi
cd ${PBS_O_WORKDIR}

# print identifying info for this job
echo "Job ${PBS_JOBNAME} submitted from ${PBS_O_HOST} started "`date`" jobid ${PBS_JOBID}"

export OFED_DIR=/usr/local/scidac-ofed-1.4.2/install
export QMP_DIR=$OFED_DIR/qmp
export QLA_DIR=$OFED_DIR/qla
export QIO_DIR=$OFED_DIR/qio
export QDP_DIR=$OFED_DIR/qdp
export MPI_DIR=/usr/local/mvapich-new
export GCC_DIR=/usr/local/gcc-4.3.2
export MILC_DIR=/usr/local/MILC

# determine number of cores per host
coresPerNode=`cat /proc/cpuinfo | grep -c processor`
# count the number of nodes listed in PBS_NODEFILE
nNodes=$[`cat ${PBS_NODEFILE} | wc --lines`]
(( nNodes= 0 + nNodes ))
(( nCores = nNodes * coresPerNode ))
echo "NODEFILE nNodes=$nNodes ($nCores cores):"
cat ${PBS_NODEFILE}

# Always use fcp or rsync to stage any large input files from the cluster head node
# to your job's control worker node.
# All worker nodes have attached disk storage in /scratch
# Copy below is commented since there are no files to transfer
# fcp -c rsh -p cluster-head-node.fnal.gov:/project/myDataArea/myInputDataFile /scratch

echo "example MPI program see: /usr/local/mvapich/examples/cpi.c"
#The directory the job was submitted from is $PBS_O_WORKDIR.
echo ${PBS_O_WORKDIR}

cp ${MPI_DIR}/examples/cpi ${PBS_O_WORKDIR}
application=${PBS_O_WORKDIR}/cpi

echo
cpus=$nodes
echo "=== Run MPI application on $nCores cores (8 cores per node) ==="
$MPI_DIR/bin/mpirun_rsh -np $nCores $application

# Always use fcp to copy any large result files you want to keep back
# to the head node before exiting your script. The /scratch area on the
# workers is wiped clean between jobs.
#
# Copy below is commented since there are no files to transfer
# fcp -c rsh -p /scratch/myOutputDataFile cluster-head-node.fnal.gov:/project/myDataArea

exit