#!/bin/bash
#
# nompirun - start a non-traditional job on BlueGene
# Copyright 2010 Eric Van Hensbergen
# 
# USAGE: nompirun [-A cobalt_account] [-h brasil_host] [-p brasil_port] 
#				[-e key=value] [-n numnodes] [-t time] 
#				[-k kernelprofile] [-r rootpath]
#				cmd args
# 
# ARGUMENTS:
#	-A: Your cobalt account
#	-D: debug mode (only print the qsub command, don't run it)
#	-h: host running brasil daemon (defaults to current host)
#	-p: port running brasil daemon (should be unique per user)
#	-n: number of cpu nodes to provision 
#	-t: length of time to request
#	-k: kernel profile (default's to user's name
#	-e key=value: specify an environment variable to pass (can be repeated)
#	cmd args: command to run on I/O nodes (eventually compute nodes maybe)
#			(default to /bin/rc)
#
# ENVIRONMENT VARIABLES: (can be used to override defaults in place of cmdline
#	BRASIL_HOST - host running brasil daemon
#	BRASIL_PORT - port running brasil daemon (should be unique)
#	PLAN9_ROOT - local path to Plan 9 root file system
#	COBALT_ACCOUNT - cobalt account
#	HARE_RELEASE - path to HARE release files
#
# TODO:
#	- Add namespace control files
#	- Add centralservices host/port info
#	- Autostart (and cleanup) brasil if its not running
#
# BUGS:
#	Several I'm sure
#

# ENVIRONMENTAL DEFAULTS

kflag=plan9
pflag=$BRASIL_PORT
rflag=$PLAN9_ROOT
hflag=$BRASIL_HOST
aflag=$COBALT_ACCOUNT
release=$HARE_RELEASE
Dflag=
oflag=

# Command line defaults
nflag=64
tflag=30
eflag="MODE=nompirun"

# read IO node log
function read_io_log {
	echo     Waiting for boot....
	echo
	echo NOTE: console output will be IO Node only
	echo ----------------------------------------------------------------

	LOG_SUFFIX=${BLOCK_ID#*-}
	LOG_PREFIX=${LOG_SUFFIX%-*}
	LOGFILE=/bgsys/logs/BGP/$LOG_PREFIX-J00.log
	tail -n 0 -f $LOGFILE
	# TODO: filter log to just my job (or my userid)
	# TODO: Catch interrupt and cleanup job(?)
	# TODO: We should be able to give IO Node IPs automagically
}

while getopts 'DA:e:h:k:n:p:t:r:' OPTION
do
	case $OPTION in
	A)	aflag="$OPTARG"
		;;
	D)	Dflag=1
		;;
	r)	rflag="$OPTARG"
		;;
	h)	hflag="$OPTARG"
		;;
	p)	pflag="$OPTARG"
		;;
	n)	nflag=$OPTARG
		;;
	t)	tflag=$OPTARG
		;;
	e)	eflag+=":$OPTARG"
		;;
	k)
		kflag="$OPTARG"
		;;
	?)	printf "Usage: %s: [-D][-A cobalt_account] [-h brasil_host] [-p brasil_port] [-e key=value] [-n numnodes] [-t time] [-r rootpath][-k kernelprofile] args\n" $(basename $0) >&2
		exit 2
		;;
	esac
done
shift $((OPTIND - 1))

# SET DEFAULTS IF USER DOESNT SPECIFY
if [ -z $aflag ]
then
	aflag=BGPPlan9Meas
fi
if [ -z $rflag ]
then
	rflag=/home/ericvh/9
fi
if [ -z $hflag ]
then
	hflag=`hostname`
fi
if [ -z $pflag ]
then
	#printf "\nWARNING: using default port (5670)\nPlease set using BRASIL_PORT environment variable or -p command line argument\n\n"
	pflag=5670
fi
if [ -z $release ]
then
	release=/bgsys/argonne-utils/profiles/plan9/release/LATEST
fi

eflag+=":HOME=$HOME:BRASIL_ADDR=tcp!$hflag!$pflag:PLAN9_ROOT=$rflag"

#
# Make sure the user is trying to execute something, otherwise just 
# default to executing rc
#
COMMAND=$@

if [ -z "$COMMAND" ]
then
	COMMAND="$rflag/power/bin/rc"
fi

if [ -n "$Dflag" ]
then
	echo "qsub -A $aflag -t $tflag -n $nflag --kernel $kflag --env $eflag $COMMAND"
	exit
fi

# Verify sanity of object files about to be loaded
for i in /bgsys/argonne-utils/profiles/$kflag/{CNK,INK,CNS,uloader,ramdisk}
do
	out=`objdump -x $i`
	case "$out" in
	*"File format not recognized"*)
		echo $i: invalid executable format 1>2
		exit -1;;
	esac
done

#
# Check for lib profile
#
if [ ! -e $HOME/lib/profile ]
then
	mkdir -p $HOME/lib
	cp $release/lib/profile  $HOME/lib/profile
fi

JOBID=`qsub -A $aflag -t $tflag -n $nflag --kernel $kflag --env $eflag $COMMAND`
echo $JOBID
