From 24741436ad1313451efbc41f181a61d004122424 Mon Sep 17 00:00:00 2001 From: Simon Rit Date: Fri, 7 Mar 2014 09:57:32 +0100 Subject: [PATCH] Enabled cluster script for CC IN2P3 (based on Pierre's dev) --- cluster_tools/gate_job_ccin2p3.job | 102 +++++++++++++++++++++++ cluster_tools/gate_job_cluster.job | 3 - cluster_tools/gate_run_submit_cluster.sh | 31 +++++-- 3 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 cluster_tools/gate_job_ccin2p3.job diff --git a/cluster_tools/gate_job_ccin2p3.job b/cluster_tools/gate_job_ccin2p3.job new file mode 100644 index 0000000..b4fbe62 --- /dev/null +++ b/cluster_tools/gate_job_ccin2p3.job @@ -0,0 +1,102 @@ +#!/usr/bin/env bash -x +# +# MACRODIR +# MACROFILE +# RELEASEDIR +# OUTPUTDIR +# INDEX +# INDEXMAX +# PARAM +# +#$ -P "P_creatis" +#$ -l vmem=2G +#$ -l fsize=1G +#$ -l ct=30:00:00 +#$ -j yes +#$ -r no + + +function error { +echo "ERROR: $1" +exit 1 +} + +function warning { +echo "WARNING: $1" +} + +test -f ${HOME}/.bashrc && echo "Sourcing bashrc" && source ${HOME}/.bashrc +set -u + +echo "Checking" +uname -a +echo "MACRODIR=${MACRODIR:?"unknown MACRODIR"}" +echo "MACROFILE=${MACROFILE:?"unknown MACROFILE"}" +echo "RELEASEDIR=${RELEASEDIR:?"unknown RELEASEDIR"}" +echo "OUTPUTDIR=${OUTPUTDIR:?"unknown OUTPUTDIR"}" +echo "JOB_ID=${JOB_ID}" +echo "INDEX=${INDEX}" +echo "INDEXMAX=${INDEX}" +echo "PARAM=${PARAM}" + +if test "$RELEASEDIR" = "NONE" +then + echo Using $(which Gate) + ldd $(which Gate) +else + test -d "${RELEASEDIR}" || error "can't find release" + md5sum ${RELEASEDIR}/Gate + test -f ${RELEASEDIR}/libGate.so && md5sum ${RELEASEDIR}/libGate.so + + echo "Finding libraries" + ROOTLIBS="${RELEASEDIR}/libCore.so:${RELEASEDIR}/libCint.so:${RELEASEDIR}/libRIO.so:${RELEASEDIR}/libNet.so:${RELEASEDIR}/libHist.so:${RELEASEDIR}/libGraf.so:${RELEASEDIR}/libGraf3d.so:${RELEASEDIR}/libGpad.so:${RELEASEDIR}/libTree.so:${RELEASEDIR}/libRint.so:${RELEASEDIR}/libPostscript.so:${RELEASEDIR}/libMatrix.so:${RELEASEDIR}/libPhysics.so:${RELEASEDIR}/libMathCore.so:${RELEASEDIR}/libThread.so:" + echo "ROOTLIBS=${ROOTLIBS}" + G4LIBS="$(for library in $(find "${RELEASEDIR}" -maxdepth 1 -name 'libG4*.so'); do echo -n "${library}:"; done)" + echo "G4LIBS=${G4LIBS}" + CLHEPLIBS="$(for library in $(find "${RELEASEDIR}" -maxdepth 1 -name 'libCLHEP*.so'); do echo -n "${library}:"; done)" + echo "CLHEPLIBS=${CLHEPLIBS}" + GATELIBS="" + test -f ${RELEASEDIR}/libGate.so && GATELIBS="${RELEASEDIR}/libGate.so:" + echo "GATELIBS=${GATELIBS}" +fi +test -d "${MACRODIR}" && test -d "${MACRODIR}/mac" || error "invalid macro" + + +echo "Copying inputs" +LOCALMACRODIR=$(mktemp -d) +trap "mv output ${OUTPUTDIR}/output.${JOB_ID%%.*} ; rm -r ${LOCALMACRODIR} ; exit 1" 1 2 3 15 +cd ${LOCALMACRODIR} +cp -r -L "${MACRODIR}"/{data,mac} . +mkdir output + +# Enforce one thread +ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1 + +echo "Lauching macro" +date +if test "$RELEASEDIR" = "NONE" +then + Gate ${PARAM} ${MACROFILE} || error "gate failed" +else + LD_PRELOAD="${ROOTLIBS}${G4LIBS}${CLHEPLIBS}${GATELIBS}" \ + G4LEVELGAMMADATA="${RELEASEDIR}/PhotonEvaporation2.1" \ + G4RADIOACTIVEDATA="${RELEASEDIR}/RadioactiveDecay3.3" \ + G4LEDATA="${RELEASEDIR}/G4EMLOW6.19" \ + G4NEUTRONHPDATA="${RELEASEDIR}/G4NDL3.14" \ + G4ABLADATA="${RELEASEDIR}/G4ABLA3.0" \ + G4REALSURFACEDATA="${RELEASEDIR}/RealSurface1.0" \ + G4NEUTRONXSDATA="${RELEASEDIR}/G4NEUTRONXS1.0" \ + G4PIIDATA="${RELEASEDIR}/G4PII1.2" \ + /usr/bin/time --format="real %es\nuser %Us\nsys %Ss\nmaxmem %Mk" \ + ${RELEASEDIR}/Gate ${PARAM} ${MACROFILE} \ + || error "gate failed" +fi + +echo "Copying data back" +ssh -i ${HOME}/.ssh/ccin2p3 linux1.dg.creatis.insa-lyon.fr "mkdir cc/$(basename ${OUTPUTDIR})/output.${JOB_ID%%.*}" +rsync -av --remove-source-files -e "ssh -i ${HOME}/.ssh/ccin2p3" output/ "linux1.dg.creatis.insa-lyon.fr:./cc/$(basename ${OUTPUTDIR})/output.${JOB_ID%%.*}" + +echo "Cleanup" +rm -r ${LOCALMACRODIR} + +echo "Success!!!" diff --git a/cluster_tools/gate_job_cluster.job b/cluster_tools/gate_job_cluster.job index 1242dfd..1ece1a6 100644 --- a/cluster_tools/gate_job_cluster.job +++ b/cluster_tools/gate_job_cluster.job @@ -12,9 +12,6 @@ #PBS -l walltime=100:00:00 #PBS -j oe -#env -#pwd -#exit 1 function error { diff --git a/cluster_tools/gate_run_submit_cluster.sh b/cluster_tools/gate_run_submit_cluster.sh index 4df0be7..ab348d9 100755 --- a/cluster_tools/gate_run_submit_cluster.sh +++ b/cluster_tools/gate_run_submit_cluster.sh @@ -24,10 +24,14 @@ function usage { test $# -eq 0 && usage && exit 0 -SCRIPTDIR="${HOME}/git/gate-tests/bin" RELEASESUFFIX=${3:-"${DEFAULTRELEASESUFFIX}"} RELEASEDIR="${HOME}/releases/grid_release${RELEASESUFFIX}" -JOBFILE="$(dirname $0)/gate_job_cluster.job" +if test "$(dnsdomainname)" = "in2p3.fr" +then + JOBFILE="$(dirname $0)/gate_job_ccin2p3.job" +else + JOBFILE="$(dirname $0)/gate_job_cluster.job" +fi echo "Checking stuff" test -f ${JOBFILE} || error "can't find job file ${JOBFILE}" @@ -42,7 +46,13 @@ MACRODIR=$(pwd) test -d ${MACRODIR}/mac && test -d ${MACRODIR}/data || error "invalid path" MACROFILE=${1:?"provide relative macro path"} test -f ${MACRODIR}/${MACROFILE} || error "invalid macro" -OUTPUTDIR=$(mktemp --tmpdir=${MACRODIR} -d run.XXXX || error "can't create temp dir") +if test "$(dnsdomainname)" = "in2p3.fr" +then + OUTPUTDIR=$(mktemp -d -p "${MACRODIR}" run.XXXX || error "can't create temp dir") + ssh -i ${HOME}/.ssh/ccin2p3 linux1.dg.creatis.insa-lyon.fr mkdir -p "cc/$(basename ${OUTPUTDIR})" +else + OUTPUTDIR=$(mktemp --tmpdir=${MACRODIR} -d run.XXXX || error "can't create temp dir") +fi test -d ${OUTPUTDIR} || error "can't locate output dir" RUNID=${OUTPUTDIR##*.} NJOBS=${2:-"${DEFAULTNUMBEROFJOBS}"} @@ -88,10 +98,17 @@ while test $NJOBS -gt 0; do if test "${QSUB}" = "noqsub" then echo "Launching Gate log in ${OUTPUTDIR}/gate_${NJOBS}.log" - PARAM=\"${PARAM}\" INDEX=${NJOBS} INDEXMAX=${NJOBSMAX} SCRIPTDIR=${SCRIPTDIR} OUTPUTDIR=${OUTPUTDIR} RELEASEDIR=${RELEASEDIR} MACROFILE=${MACROFILE} MACRODIR=${MACRODIR} PBS_JOBID="local_${NJOBS}" bash "${JOBFILE}" > ${OUTPUTDIR}/gate_${NJOBS}.log & + PARAM=\"${PARAM}\" INDEX=${NJOBS} INDEXMAX=${NJOBSMAX} OUTPUTDIR=${OUTPUTDIR} RELEASEDIR=${RELEASEDIR} MACROFILE=${MACROFILE} MACRODIR=${MACRODIR} PBS_JOBID="local_${NJOBS}" bash "${JOBFILE}" > ${OUTPUTDIR}/gate_${NJOBS}.log & + elif test "$(dnsdomainname)" = "in2p3.fr" + then + PROJECTGROUP=creatis + qsub -o "${OUTPUTDIR}" \ + -N "gate.${RUNID}" \ + -v "PARAM=\"${PARAM}\",INDEX=${NJOBS},INDEXMAX=${NJOBSMAX},OUTPUTDIR=${OUTPUTDIR},RELEASEDIR=${RELEASEDIR},MACROFILE=${MACROFILE},MACRODIR=${MACRODIR}" \ + "${JOBFILE}" || error "submission error" else qsub -N "gatejob.${RUNID}" -o "${OUTPUTDIR}" \ - -v "PARAM=\"${PARAM}\",INDEX=${NJOBS},INDEXMAX=${NJOBSMAX},SCRIPTDIR=${SCRIPTDIR},OUTPUTDIR=${OUTPUTDIR},RELEASEDIR=${RELEASEDIR},MACROFILE=${MACROFILE},MACRODIR=${MACRODIR}" \ + -v "PARAM=\"${PARAM}\",INDEX=${NJOBS},INDEXMAX=${NJOBSMAX},OUTPUTDIR=${OUTPUTDIR},RELEASEDIR=${RELEASEDIR},MACROFILE=${MACROFILE},MACRODIR=${MACRODIR}" \ "${JOBFILE}" || error "submission error" fi @@ -99,3 +116,7 @@ while test $NJOBS -gt 0; do done echo "runid is ${RUNID}" +if test "$(dnsdomainname)" = "in2p3.fr" +then + rsync -av --remove-source-files -e "ssh -i ${HOME}/.ssh/ccin2p3" ${OUTPUTDIR}/ "linux1.dg.creatis.insa-lyon.fr:./cc/$(basename ${OUTPUTDIR})" +fi -- 2.47.1