gerun

#!/bin/bash

# Parallel launcher for the various PEs on Legion.
# Dr Owain Kenway, UCL

# For licensing terms, see LICENSE.txt

# This variable should be set by MPI modules:
# GERUN_LAUNCHER

set -e

# See if GERUN_PATH is set, otherwise guess it out from $0.
export GERUN_PATH="${GERUN_PATH:-$(dirname $(readlink -f $(which $0)))}"

# Load function definitions.
source $GERUN_PATH/gerunlib.sh

stderrmsg 
stderrmsg "Note: Lines like this one prefixed with \"GERun:\" are for debugging"
stderrmsg "      purposes only and you do not need to report them to rc-support"
stderrmsg "      unless your job fails for other reasons."
stderrmsg 

# Make it possible to disable all output from GERun.
# This is to make it work with some testing regimes.
# To disable output, set GERUN_SILENT=yes
GERUN_SILENT="${GERUN_SILENT:-no}"

# To lowercase (only works in bash 4+)
GERUN_SILENT="${GERUN_SILENT,,}"

# Make sure user hasn't done something else odd.
if [[ "$GERUN_SILENT" == "true" ]]; then
  GERUN_SILENT=yes
fi

# Make sure it's available in launcher scripts if need be.
export GERUN_SILENT


# Check to see if we have a customised message and if so use it rather 
if [[ "$GERUN_SILENT" != "yes" ]]; then 
  if [[ -s "$GERUN_PATH/local-message" ]]
  then
    stderrcat "$GERUN_PATH/local-message"
  else
    stderrcat "$GERUN_PATH/gerun-message"
  fi
fi

# Importing exported shell functions through qrsh causes noisy error messages
# Let's unexport the module functions in case they're set to be exported
if [[ "$GERUN_SILENT" != "yes" ]]; then
    stderrmsg "Removing export flag from module shell functions..."
fi
if declare -F module >/dev/null; then
    export -nf module
    if [[ "$GERUN_SILENT" != "yes" ]]; then
        stderrmsg "  unexported: module"
    fi
fi
if declare -F _module_raw >/dev/null; then
    export -nf _module_raw
    if [[ "$GERUN_SILENT" != "yes" ]]; then
        stderrmsg "  unexported: _module_raw"
    fi
fi
if declare -F ml >/dev/null; then
    export -nf ml
    if [[ "$GERUN_SILENT" != "yes" ]]; then
        stderrmsg "  unexported: ml"
    fi
fi
if declare -F switchml >/dev/null; then
    export -nf switchml
    if [[ "$GERUN_SILENT" != "yes" ]]; then
        stderrmsg "  unexported: switchml"
    fi
fi
if [[ "$GERUN_SILENT" != "yes" ]]; then
    stderrmsg "Finished removing export flags."
fi


# We need to support mixed mode code on new Legion.

OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}"
NSLOTS="${NSLOTS:-1}"
GERUN_PROCS="$((NSLOTS/OMP_NUM_THREADS))"

export OMP_NUM_THREADS
export GERUN_PROCS

if [[ "$GERUN_SILENT" != "yes" ]]; then 
  stderrmsg
  stderrmsg "Using environment: $GERUN_LAUNCHER"
  stderrmsg "Running on $NSLOTS slots:"
  stderrprintf "  %3d MPI tasks\n" "$GERUN_PROCS"
  stderrprintf "  %3d threads per task\n" "$OMP_NUM_THREADS"
  stderrmsg "TMPDIR=$TMPDIR"
  stderrmsg
fi

if [[ -s "$TMPDIR/machines" ]]; then
  if [[ "$GERUN_SILENT" != "yes" ]]; then
    stderrmsg "Contents of machinefile:"
    stderrcat "$TMPDIR/machines"
    stderrmsg
  fi

# Create a sorted, unique machine file for mpiruns which work better with that.
  sort -u "$TMPDIR/machines" > "$TMPDIR/machines.unique"

  "$GERUN_PATH/gerun-$GERUN_LAUNCHER" "$@"
else
  if [[ "$GERUN_SILENT" != "yes" ]]; then
    if [[ "${PE:0:3}" == "smp" ]]; then
      stderrmsg "WARNING: SMP environment detected."
      stderrmsg "You appear to be trying to run an MPI package inside the smp parallel"
      stderrmsg "environment (i.e. you have requested -pe smp in your job script)."
      stderrmsg "There is no machine file available - running on only one node."
      stderrmsg "Falling back to basic mpirun."
      stderrmsg
    else
      stderrmsg "WARNING: No machine file found in $TMPDIR."
      stderrmsg "You appear to be running outside of a Grid Engine scheduled environment."
      stderrmsg "Falling back to basic mpirun."
      stderrmsg
    fi
  fi
  mpirun "$@"
fi