#!/bin/sh
# RemoteRunjobAppScript:makeMPITemplate
# SubmissionScripts/Distributor/Runjob/APP/mpi

#echo $0 "$@"
#printenv | sort

@@{MODULEINITIALIZE}
@@{MODULESUNLOAD}
@@{MODULESLOAD}

SCRIPT=`basename $0`
SITE_SCRATCH_DIR=$3
TARBALL=${SITE_SCRATCH_DIR}/`basename $1 .gz`
WORKINGDIR=${SITE_SCRATCH_DIR}/$2
shift 3

@@{MPIRANKVARIABLE}=""
if   [ "_${MPIRUN_RANK}_" != "__" ] ; then
   @@{MPIRANKVARIABLE}=${MPIRUN_RANK}
elif [ "_${GMPI_ID}_" != "__" ] ; then
   @@{MPIRANKVARIABLE}=${GMPI_ID}
elif [ "_${RMS_RANK}_" != "__" ] ; then
   @@{MPIRANKVARIABLE}=${RMS_RANK}
elif [ "_${MXMPI_ID}_" != "__" ] ; then
   @@{MPIRANKVARIABLE}=${MXMPI_ID}
elif [ "_${MSTI_RANK}_" != "__" ] ; then
   @@{MPIRANKVARIABLE}=${MSTI_RANK}
elif [ "_${PMI_RANK}_" != "__" ] ; then
   @@{MPIRANKVARIABLE}=${PMI_RANK}
elif [ "_${PMI_ID}_" != "__" ] ; then
   @@{MPIRANKVARIABLE}=${PMI_ID}
elif [ "_${OMPI_MCA_ns_nds_vpid}_" != "__" ] ; then
   @@{MPIRANKVARIABLE}=${OMPI_MCA_ns_nds_vpid}
elif [ "_${OMPI_MCA_orte_ess_vpid}_" != "__" ] ; then
   @@{MPIRANKVARIABLE}=${OMPI_MCA_orte_ess_vpid}
else
   args=( "$@" )
   for (( i1=0 ; i1 < ${#args[@]} ; i1++ )) ; do
      if   [ "${args[i1]}" = "-p4pg" ] ; then
         p4rank=0
      elif [ "${args[i1]}" = "-p4wd" ] ; then
         (( i1++ ))
         args[${i1}]=${WORKINGDIR}
      elif [ "${args[i1]}" = "-p4rmrank" ] ; then
         (( i1++ ))
         p4rank=${args[i1]}
      fi
   done
   if [ "${p4rank}" = "0" ] ; then
      set -- ${args[@]}
   fi
   if [ "_${p4rank}_" != "__" ] ; then
      @@{MPIRANKVARIABLE}=${p4rank}
   fi
fi

if [ ! -d ${SITE_SCRATCH_DIR} ] ; then
   echo "Jobs directory ${SITE_SCRATCH_DIR} is missing"
   exit 13
fi

if [ "${USER}" = "" ] ; then
   if [ "${LOGNAME}" != "" ] ; then
      export USER=${LOGNAME}
   else
      export USER=HUBzero
   fi
fi

CURDIR=`pwd`

APP_PROGRAM=@@{EXECUTABLE}
if [ ! -x ${APP_PROGRAM} ] ; then
   if [ "${SITE_APPLICATION_ROOT}" != "" ] ; then
      if [ -x ${SITE_APPLICATION_ROOT}/${APP_PROGRAM} ] ; then
         APP_PROGRAM=${SITE_APPLICATION_ROOT}/${APP_PROGRAM}
      fi
   fi
fi

TIMEPATH=
for timePath in ${HOME}/bin/time /usr/bin/time /usr/local/bin/time ; do
   if [ -x ${timePath} ] ; then
      TIMEPATH=${timePath}
      break
   fi
done

if [ "_${@@{MPIRANKVARIABLE}}_" = "__" ] ; then
   mkdir -p ${WORKINGDIR}
   chmod 700 ${WORKINGDIR}
   cd ${WORKINGDIR}
   touch .__fileTimeMarker
   sleep 1
   date +"%s" > @@{TS_START}
   #
   echo "Unable to determine MPI rank"
   #

   date +"%s" > @@{TS_FINISH}

   # Allow for file/directory names with embedded spaces.
   find . -newer .__fileTimeMarker -not -name . -not -iname '*hub-proxy.*' -print0 | \
      xargs --null --no-run-if-empty tar rf ${TARBALL} --no-recursion

   cd ${CURDIR}
   rm -rf ${WORKINGDIR}
else
   if [ "${@@{MPIRANKVARIABLE}}" = "0" ] ; then

      mkdir -p ${WORKINGDIR}
      chmod 700 ${WORKINGDIR}
      tar xzm -C ${WORKINGDIR} -f ${SITE_SCRATCH_DIR}/@@{JOBID}_@@{INSTANCEID}_input.tar.gz
   #
      cd ${WORKINGDIR}
      sleep 2
      touch .__fileTimeMarker
      sleep 1
      date +"%s" > @@{TS_START}
   #
   # create output file now - if job times out the
   # file is available for transfer by condor
   #
      tar cf ${TARBALL} ./@@{TS_START}
   #
   fi

   until [ -f ${TARBALL} ] ; do
      sleep 2
   done

   cd ${WORKINGDIR}
   export PATH=${PATH}:${WORKINGDIR}
   @@{ENVIRONMENT}

   rankExtension=`printf "%06d" ${@@{MPIRANKVARIABLE}}`
   if   [ "${p4rank}" = "0" ] ; then
      ${TIMEPATH} --format "Command exited with status %x\nreal %e\nuser %U\nsys %S" -o @@{TIME_RESULTS}-${rankExtension} \
         ${APP_PROGRAM} @@{ARGUMENTS} "$@" < @@{STDIN}
   elif [ "${p4rank}" != "" ] ; then
      ${TIMEPATH} --format "Command exited with status %x\nreal %e\nuser %U\nsys %S" -o @@{TIME_RESULTS}-${rankExtension} \
         ${APP_PROGRAM}               "$@" < @@{STDIN}
   else
      ${TIMEPATH} --format "Command exited with status %x\nreal %e\nuser %U\nsys %S" -o @@{TIME_RESULTS}-${rankExtension} \
         ${APP_PROGRAM} @@{ARGUMENTS} "$@" < @@{STDIN}
   fi

   if [ "${@@{MPIRANKVARIABLE}}" = "0" ] ; then
      date +"%s" > @@{TS_FINISH}

      touch @@{TIME_RESULTS}-0
      cat @@{TIME_RESULTS}-[0-9]* >> @@{TIME_RESULTS}
      rm -f @@{TIME_RESULTS}-[0-9]*
   #
   # tar: Cannot update compressed archives
   # so recreate output file for transfer by condor
   #
   # Allow for file/directory names with embedded spaces.
      find . -newer .__fileTimeMarker -not -name . -not -iname '*hub-proxy.*' -print0 | \
         xargs --null --no-run-if-empty tar rf ${TARBALL} --no-recursion

      cd ${CURDIR}
      rm -rf ${WORKINGDIR}
   fi
fi
if [ -f ${TARBALL} ] ; then
   gzip ${TARBALL}
fi
