#! /bin/bash
#PBS -S /bin/bash
#PBS -N cpi-example
#PBS -j oe
#PBS -o ./cpi-example.out
#PBS -m n
#PBS -l nodes=3,walltime=00:01:00
#PBS -A projectName
#PBS -q kaon
cd ${PBS_O_WORKDIR}

# print identifying info for this job
echo "Job ${PBS_JOBNAME} submitted from ${PBS_O_HOST} started "`date`" jobid ${PBS_JOBID}"

# count the number of nodes listed in PBS_NODEFILE
nodes=$[`cat ${PBS_NODEFILE} | wc --lines`]
echo "Job allocated $nodes nodes"

# Always use pbs_rcp to stage any large input files from the head node (lqcd)
# to your job's control worker node.
# All worker nodes  have attached disk storage in /scratch
#
# Copy below is commented since there are no files to transfer
# /usr/local/pbs/sbin/pbs_rcp -p lqcd:/data/raid1/data/myDataArea/myInputDataFile /scratch

echo "example MPI program see: /usr/local/mvapich/examples/cpi.c"
application=${PBS_O_WORKDIR}/cpi

echo
cpus=$nodes
echo "=== Run MPI application on $cpus cpus (1 cpu per node) ==="
/usr/local/mvapich/bin/mpirun_rsh -rsh -np $cpus -hostfile $PBS_NODEFILE $application


# Always use pbs_rcp to copy any large result files you want to keep back
# to the head node before exiting your script. The /scratch area on the
# workers is wiped clean between jobs.
#
# There were no output files created in this example.
# /usr/local/pbs/sbin/pbs_rcp -p /scratch/myOutputDataFile lqcd:/data/raid1/data/myDataArea