# PARAM
#
#$ -P "P_creatis"
-#$ -l vmem=2G
+#$ -l vmem=4G
#$ -l fsize=1G
#$ -l ct=30:00:00
#$ -j yes
#$ -r no
+function transferJobData {
+ echo "Copying data back"
+ #ssh -i ${HOME}/.ssh/ccin2p3 linux1.dg.creatis.insa-lyon.fr "mkdir cc/$(basename ${OUTPUTDIR})/output.${JOB_ID%%.*}"
+ #rsync -av --remove-source-files -e "ssh -i ${HOME}/.ssh/ccin2p3" output/ "linux1.dg.creatis.insa-lyon.fr:./cc/$(basename ${OUTPUTDIR})/output.${JOB_ID%%.*}"
+ pwd
+ echo ${OUTPUTDIR}
+ mkdir "${OUTPUTDIR}/output.${JOB_ID%%.*}"
+ rsync -av --remove-source-files output/ "${OUTPUTDIR}/output.${JOB_ID%%.*}"
+ echo "Cleanup"
+ rm -r ${LOCALMACRODIR}
+}
function error {
-echo "ERROR: $1"
-exit 1
+ echo "ERROR: $1"
+ exit 1
}
function warning {
-echo "WARNING: $1"
+ echo "WARNING: $1"
}
+date
test -f ${HOME}/.bashrc && echo "Sourcing bashrc" && source ${HOME}/.bashrc
set -u
if test "$RELEASEDIR" = "NONE"
then
- echo Using $(which Gate)
- ldd $(which Gate)
+ echo Using $(which Gate)
+ ldd $(which Gate)
else
- test -d "${RELEASEDIR}" || error "can't find release"
- md5sum ${RELEASEDIR}/Gate
- test -f ${RELEASEDIR}/libGate.so && md5sum ${RELEASEDIR}/libGate.so
+ test -d "${RELEASEDIR}" || error "can't find release"
+ md5sum ${RELEASEDIR}/Gate
+ test -f ${RELEASEDIR}/libGate.so && md5sum ${RELEASEDIR}/libGate.so
- echo "Finding libraries"
- ROOTLIBS="${RELEASEDIR}/libCore.so:${RELEASEDIR}/libCint.so:${RELEASEDIR}/libRIO.so:${RELEASEDIR}/libNet.so:${RELEASEDIR}/libHist.so:${RELEASEDIR}/libGraf.so:${RELEASEDIR}/libGraf3d.so:${RELEASEDIR}/libGpad.so:${RELEASEDIR}/libTree.so:${RELEASEDIR}/libRint.so:${RELEASEDIR}/libPostscript.so:${RELEASEDIR}/libMatrix.so:${RELEASEDIR}/libPhysics.so:${RELEASEDIR}/libMathCore.so:${RELEASEDIR}/libThread.so:"
- echo "ROOTLIBS=${ROOTLIBS}"
- G4LIBS="$(for library in $(find "${RELEASEDIR}" -maxdepth 1 -name 'libG4*.so'); do echo -n "${library}:"; done)"
- echo "G4LIBS=${G4LIBS}"
- CLHEPLIBS="$(for library in $(find "${RELEASEDIR}" -maxdepth 1 -name 'libCLHEP*.so'); do echo -n "${library}:"; done)"
- echo "CLHEPLIBS=${CLHEPLIBS}"
- GATELIBS=""
- test -f ${RELEASEDIR}/libGate.so && GATELIBS="${RELEASEDIR}/libGate.so:"
- echo "GATELIBS=${GATELIBS}"
+ echo "Finding libraries"
+ ROOTLIBS="${RELEASEDIR}/libCore.so:${RELEASEDIR}/libCint.so:${RELEASEDIR}/libRIO.so:${RELEASEDIR}/libNet.so:${RELEASEDIR}/libHist.so:${RELEASEDIR}/libGraf.so:${RELEASEDIR}/libGraf3d.so:${RELEASEDIR}/libGpad.so:${RELEASEDIR}/libTree.so:${RELEASEDIR}/libRint.so:${RELEASEDIR}/libPostscript.so:${RELEASEDIR}/libMatrix.so:${RELEASEDIR}/libPhysics.so:${RELEASEDIR}/libMathCore.so:${RELEASEDIR}/libThread.so:"
+ echo "ROOTLIBS=${ROOTLIBS}"
+ G4LIBS="$(for library in $(find "${RELEASEDIR}" -maxdepth 1 -name 'libG4*.so'); do echo -n "${library}:"; done)"
+ echo "G4LIBS=${G4LIBS}"
+ CLHEPLIBS="$(for library in $(find "${RELEASEDIR}" -maxdepth 1 -name 'libCLHEP*.so'); do echo -n "${library}:"; done)"
+ echo "CLHEPLIBS=${CLHEPLIBS}"
+ GATELIBS=""
+ test -f ${RELEASEDIR}/libGate.so && GATELIBS="${RELEASEDIR}/libGate.so:"
+ echo "GATELIBS=${GATELIBS}"
fi
test -d "${MACRODIR}" && test -d "${MACRODIR}/mac" || error "invalid macro"
echo "Copying inputs"
LOCALMACRODIR=$(mktemp -d)
-trap "mv output ${OUTPUTDIR}/output.${JOB_ID%%.*} ; rm -r ${LOCALMACRODIR} ; exit 1" 1 2 3 15
+trap "wait ; transferJobData ; exit 1" 137 138 152 153
cd ${LOCALMACRODIR}
cp -r -L "${MACRODIR}"/{data,mac} .
mkdir output
date
if test "$RELEASEDIR" = "NONE"
then
- Gate ${PARAM} ${MACROFILE} || error "gate failed"
+ Gate ${PARAM} ${MACROFILE} || error "gate failed"
else
- LD_PRELOAD="${ROOTLIBS}${G4LIBS}${CLHEPLIBS}${GATELIBS}" \
- G4LEVELGAMMADATA="${RELEASEDIR}/PhotonEvaporation2.1" \
- G4RADIOACTIVEDATA="${RELEASEDIR}/RadioactiveDecay3.3" \
- G4LEDATA="${RELEASEDIR}/G4EMLOW6.19" \
- G4NEUTRONHPDATA="${RELEASEDIR}/G4NDL3.14" \
- G4ABLADATA="${RELEASEDIR}/G4ABLA3.0" \
- G4REALSURFACEDATA="${RELEASEDIR}/RealSurface1.0" \
- G4NEUTRONXSDATA="${RELEASEDIR}/G4NEUTRONXS1.0" \
- G4PIIDATA="${RELEASEDIR}/G4PII1.2" \
- /usr/bin/time --format="real %es\nuser %Us\nsys %Ss\nmaxmem %Mk" \
- ${RELEASEDIR}/Gate ${PARAM} ${MACROFILE} \
- || error "gate failed"
+ LD_PRELOAD="${ROOTLIBS}${G4LIBS}${CLHEPLIBS}${GATELIBS}" \
+ G4LEVELGAMMADATA="${RELEASEDIR}/PhotonEvaporation2.1" \
+ G4RADIOACTIVEDATA="${RELEASEDIR}/RadioactiveDecay3.3" \
+ G4LEDATA="${RELEASEDIR}/G4EMLOW6.19" \
+ G4NEUTRONHPDATA="${RELEASEDIR}/G4NDL3.14" \
+ G4ABLADATA="${RELEASEDIR}/G4ABLA3.0" \
+ G4REALSURFACEDATA="${RELEASEDIR}/RealSurface1.0" \
+ G4NEUTRONXSDATA="${RELEASEDIR}/G4NEUTRONXS1.0" \
+ G4PIIDATA="${RELEASEDIR}/G4PII1.2" \
+ /usr/bin/time --format="real %es\nuser %Us\nsys %Ss\nmaxmem %Mk" \
+ ${RELEASEDIR}/Gate ${PARAM} ${MACROFILE} \
+ || error "gate failed"
fi
date
-echo "Copying data back"
-ssh -i ${HOME}/.ssh/ccin2p3 linux1.dg.creatis.insa-lyon.fr "mkdir cc/$(basename ${OUTPUTDIR})/output.${JOB_ID%%.*}"
-rsync -av --remove-source-files -e "ssh -i ${HOME}/.ssh/ccin2p3" output/ "linux1.dg.creatis.insa-lyon.fr:./cc/$(basename ${OUTPUTDIR})/output.${JOB_ID%%.*}"
-
-echo "Cleanup"
-rm -r ${LOCALMACRODIR}
+transferJobData
date
echo "Success!!!"
function write_mhd_header {
local input_interfile="${1:?"provide input interfile"}"
- local output_mhd="$(dirname "${input_interfile}")/$(basename "${input_interfile}" ".hdr").mhd"
+ local output_mhd="$(dirname "${input_interfile}")/$(basename "${input_interfile}" ".hdr").hdr.mhd"
- check_interfile "${input_interfile}" || error "${input_interfile} isn't an interfile image"
+ check_interfile "${input_interfile}" || warning "${input_interfile} isn't an interfile image (or unknown format)"
local header_start='ObjectType = Image
NDims = 3
echo "${indent}this is a interfile image"
echo "${indent}creating mhd headers"
for partialoutputfile in $partialoutputfiles; do write_mhd_header "${partialoutputfile}"; done
- local mhd_partialoutputfiles="$(for partialoutputfile in $partialoutputfiles; do echo "${partialoutputfile%.*}.mhd"; done)"
- local mhd_mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}" ".hdr").mhd"
+ local mhd_partialoutputfiles="$(for partialoutputfile in $partialoutputfiles; do echo "${partialoutputfile%.*}.hdr.mhd"; done)"
+ local mhd_mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}" ".hdr").hdr.mhd"
merge_mhd_image "${mhd_mergedfile}" ${mhd_partialoutputfiles} || error "error while merging"
echo "${indent}cleaning mhd headers"
for mhd_partialoutputfile in $mhd_partialoutputfiles; do rm "${mhd_partialoutputfile}"; done
-#! /usr/local/bin/bash -lx
+#!/bin/bash -l
set -u
SCRIPTNAME="$(basename "${0}")"
files=`grep "control/execute" ${MACROFILE} | cut -d " " -f 2`
for i in $files
do
+ echo $i
cp $i ${OUTPUTDIR}/mac
done
PARAM=\"${PARAM}\" INDEX=${NJOBS} INDEXMAX=${NJOBSMAX} OUTPUTDIR=${OUTPUTDIR} RELEASEDIR=${RELEASEDIR} MACROFILE=${MACROFILE} MACRODIR=${MACRODIR} PBS_JOBID="local_${NJOBS}" bash "${JOBFILE}" > ${OUTPUTDIR}/gate_${NJOBS}.log &
elif test "$(dnsdomainname)" = "in2p3.fr"
then
- PROJECTGROUP=creatis
+ PROJECTGROUP=creatis
qsub -o "${OUTPUTDIR}" \
-N "gate.${RUNID}" \
-v "PARAM=\"${PARAM}\",INDEX=${NJOBS},INDEXMAX=${NJOBSMAX},OUTPUTDIR=${OUTPUTDIR},RELEASEDIR=${RELEASEDIR},MACROFILE=${MACROFILE},MACRODIR=${MACRODIR}" \
--- /dev/null
+#! /usr/local/bin/bash -lx
+
+set -u
+SCRIPTNAME="$(basename "${0}")"
+
+# -------------------------------------------------
+function error {
+ echo "ERROR: $1"
+ usage
+ exit 1
+}
+# -------------------------------------------------
+
+DEFAULTRELEASESUFFIX="NONE"
+DEFAULTNUMBEROFJOBS="10"
+
+# -------------------------------------------------
+function usage {
+ echo "${SCRIPTNAME} mac/main.mac njobs releasesuffix paramtogate"
+ echo "default njobs = ${DEFAULTNUMBEROFJOBS}"
+ echo "default releasesuffix = ${DEFAULTRELEASESUFFIX} (NONE means use Gate in PATH)"
+ echo "default paramtogate = \"\" (use \"\" around params and \\ in front of commas)"
+}
+# -------------------------------------------------
+
+test $# -eq 0 && usage && exit 0
+
+RELEASESUFFIX=${3:-"${DEFAULTRELEASESUFFIX}"}
+RELEASEDIR="${HOME}/releases/grid_release${RELEASESUFFIX}"
+if test "$(dnsdomainname)" = "in2p3.fr"
+then
+ JOBFILE="$(dirname $0)/gate_job_ccin2p3.job"
+else
+ JOBFILE="$(dirname $0)/gate_job_cluster.job"
+fi
+
+echo "Checking stuff"
+test -f ${JOBFILE} || error "can't find job file ${JOBFILE}"
+if test "${RELEASESUFFIX}" = "${DEFAULTRELEASESUFFIX}"
+then
+ RELEASEDIR="NONE"
+ which Gate 2>&1 >/dev/null || error "there is no Gate in the PATH"
+else
+ test -d ${RELEASEDIR} || error "invalid release dir ${RELEASEDIR}"
+fi
+MACRODIR=$(pwd)
+test -d ${MACRODIR}/mac && test -d ${MACRODIR}/data || error "invalid path"
+MACROFILE=${1:?"provide relative macro path"}
+test -f ${MACRODIR}/${MACROFILE} || error "invalid macro"
+if test "$(dnsdomainname)" = "in2p3.fr"
+then
+ OUTPUTDIR=$(mktemp -d -p "${MACRODIR}" run.XXXX || error "can't create temp dir")
+ ssh -i ${HOME}/.ssh/ccin2p3 linux1.dg.creatis.insa-lyon.fr mkdir -p "cc/$(basename ${OUTPUTDIR})"
+else
+ OUTPUTDIR=$(mktemp --tmpdir=${MACRODIR} -d run.XXXX || error "can't create temp dir")
+fi
+test -d ${OUTPUTDIR} || error "can't locate output dir"
+RUNID=${OUTPUTDIR##*.}
+NJOBS=${2:-"${DEFAULTNUMBEROFJOBS}"}
+NJOBSMAX=${NJOBS}
+PARAM="${4:-""}"
+
+echo "Lets roll!!"
+echo "runid is ${RUNID}"
+
+QSUB=$(which qsub 2> /dev/null)
+# echo "qsub is $(which qsub)"
+test -z "${QSUB}" && QSUB="noqsub"
+if test "${QSUB}" = "noqsub"
+then
+ echo "qsub is not found. Simply run Gate on multiple cores."
+fi
+
+test -z "${PARAM}" && echo "no param" || echo "param is ${PARAM}"
+if test "$RELEASESUFFIX" = "$DEFAULTRELEASESUFFIX"
+then
+ echo "Gate is $(which Gate)"
+else
+ echo "Gate release is $(basename ${RELEASEDIR})"
+fi
+echo "submitting ${NJOBS} jobs"
+
+PARAMFILE="${OUTPUTDIR}/params.txt"
+echo "njobs = ${NJOBS}" >> "${PARAMFILE}"
+echo "macro = ${MACROFILE}" >> "${PARAMFILE}"
+test -z "${PARAM}" || echo "param = ${PARAM}" >> "${PARAMFILE}"
+
+# Copy macros files (for log)
+mkdir ${OUTPUTDIR}/mac
+cp ${MACROFILE} ${OUTPUTDIR}/mac
+files=`grep "control/execute" ${MACROFILE} | cut -d " " -f 2`
+for i in $files
+do
+ cp $i ${OUTPUTDIR}/mac
+done
+
+while test $NJOBS -gt 0; do
+
+ if test "${QSUB}" = "noqsub"
+ then
+ echo "Launching Gate log in ${OUTPUTDIR}/gate_${NJOBS}.log"
+ PARAM=\"${PARAM}\" INDEX=${NJOBS} INDEXMAX=${NJOBSMAX} OUTPUTDIR=${OUTPUTDIR} RELEASEDIR=${RELEASEDIR} MACROFILE=${MACROFILE} MACRODIR=${MACRODIR} PBS_JOBID="local_${NJOBS}" bash "${JOBFILE}" > ${OUTPUTDIR}/gate_${NJOBS}.log &
+ elif test "$(dnsdomainname)" = "in2p3.fr"
+ then
+ PROJECTGROUP=creatis
+ qsub -o "${OUTPUTDIR}" \
+ -l sps=1 \
+ -N "gate.${RUNID}" \
+ -v "PARAM=\"${PARAM}\",INDEX=${NJOBS},INDEXMAX=${NJOBSMAX},OUTPUTDIR=${OUTPUTDIR},RELEASEDIR=${RELEASEDIR},MACROFILE=${MACROFILE},MACRODIR=${MACRODIR}" \
+ "${JOBFILE}" || error "submission error"
+ else
+ qsub -N "gatejob.${RUNID}" -o "${OUTPUTDIR}" \
+ -l sps=1 \
+ -v "PARAM=\"${PARAM}\",INDEX=${NJOBS},INDEXMAX=${NJOBSMAX},OUTPUTDIR=${OUTPUTDIR},RELEASEDIR=${RELEASEDIR},MACROFILE=${MACROFILE},MACRODIR=${MACRODIR}" \
+ "${JOBFILE}" || error "submission error"
+ fi
+
+ let NJOBS--
+done
+
+echo "runid is ${RUNID}"
+if test "$(dnsdomainname)" = "in2p3.fr"
+then
+ rsync -av --remove-source-files -e "ssh -i ${HOME}/.ssh/ccin2p3" ${OUTPUTDIR}/ "linux1.dg.creatis.insa-lyon.fr:./cc/$(basename ${OUTPUTDIR})" --exclude '.__afs*' --exclude "${OUTPUTDIR}/gatejob.*.o*"
+fi