Enabled cluster script for CC IN2P3 (based on Pierre's dev)

diff --git a/cluster_tools/gate_job_ccin2p3.job b/cluster_tools/gate_job_ccin2p3.job
new file mode 100644 (file)
index 0000000..b4fbe62
--- /dev/null
+++ b/cluster_tools/gate_job_ccin2p3.job
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash -x
+#
+# MACRODIR
+# MACROFILE
+# RELEASEDIR
+# OUTPUTDIR
+# INDEX
+# INDEXMAX
+# PARAM
+#
+#$ -P "P_creatis"
+#$ -l vmem=2G
+#$ -l fsize=1G
+#$ -l ct=30:00:00
+#$ -j yes
+#$ -r no
+
+
+function error {
+echo "ERROR: $1"
+exit 1
+}
+
+function warning {
+echo "WARNING: $1"
+}
+
+test -f ${HOME}/.bashrc && echo "Sourcing bashrc" && source ${HOME}/.bashrc
+set -u
+
+echo "Checking"
+uname -a
+echo "MACRODIR=${MACRODIR:?"unknown MACRODIR"}"
+echo "MACROFILE=${MACROFILE:?"unknown MACROFILE"}"
+echo "RELEASEDIR=${RELEASEDIR:?"unknown RELEASEDIR"}"
+echo "OUTPUTDIR=${OUTPUTDIR:?"unknown OUTPUTDIR"}"
+echo "JOB_ID=${JOB_ID}"
+echo "INDEX=${INDEX}"
+echo "INDEXMAX=${INDEX}"
+echo "PARAM=${PARAM}"
+
+if test "$RELEASEDIR" = "NONE"
+then
+       echo Using $(which Gate)
+       ldd $(which Gate)
+else
+       test -d "${RELEASEDIR}" || error "can't find release"
+       md5sum ${RELEASEDIR}/Gate
+       test -f ${RELEASEDIR}/libGate.so && md5sum ${RELEASEDIR}/libGate.so
+
+       echo "Finding libraries"
+       ROOTLIBS="${RELEASEDIR}/libCore.so:${RELEASEDIR}/libCint.so:${RELEASEDIR}/libRIO.so:${RELEASEDIR}/libNet.so:${RELEASEDIR}/libHist.so:${RELEASEDIR}/libGraf.so:${RELEASEDIR}/libGraf3d.so:${RELEASEDIR}/libGpad.so:${RELEASEDIR}/libTree.so:${RELEASEDIR}/libRint.so:${RELEASEDIR}/libPostscript.so:${RELEASEDIR}/libMatrix.so:${RELEASEDIR}/libPhysics.so:${RELEASEDIR}/libMathCore.so:${RELEASEDIR}/libThread.so:"
+       echo "ROOTLIBS=${ROOTLIBS}"
+       G4LIBS="$(for library in $(find "${RELEASEDIR}" -maxdepth 1 -name 'libG4*.so'); do echo -n "${library}:"; done)"
+       echo "G4LIBS=${G4LIBS}"
+       CLHEPLIBS="$(for library in $(find "${RELEASEDIR}" -maxdepth 1 -name 'libCLHEP*.so'); do echo -n "${library}:"; done)"
+       echo "CLHEPLIBS=${CLHEPLIBS}"
+       GATELIBS=""
+       test -f ${RELEASEDIR}/libGate.so && GATELIBS="${RELEASEDIR}/libGate.so:"
+       echo "GATELIBS=${GATELIBS}"
+fi
+test -d "${MACRODIR}" && test -d "${MACRODIR}/mac" || error "invalid macro"
+
+
+echo "Copying inputs"
+LOCALMACRODIR=$(mktemp -d)
+trap "mv output ${OUTPUTDIR}/output.${JOB_ID%%.*} ; rm -r ${LOCALMACRODIR} ; exit 1" 1 2 3 15
+cd ${LOCALMACRODIR}
+cp -r -L "${MACRODIR}"/{data,mac} .
+mkdir output
+
+# Enforce one thread
+ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1
+
+echo "Lauching macro"
+date
+if test "$RELEASEDIR" = "NONE"
+then
+       Gate ${PARAM} ${MACROFILE} || error "gate failed"
+else
+       LD_PRELOAD="${ROOTLIBS}${G4LIBS}${CLHEPLIBS}${GATELIBS}" \
+       G4LEVELGAMMADATA="${RELEASEDIR}/PhotonEvaporation2.1" \
+       G4RADIOACTIVEDATA="${RELEASEDIR}/RadioactiveDecay3.3" \
+       G4LEDATA="${RELEASEDIR}/G4EMLOW6.19" \
+       G4NEUTRONHPDATA="${RELEASEDIR}/G4NDL3.14" \
+       G4ABLADATA="${RELEASEDIR}/G4ABLA3.0" \
+       G4REALSURFACEDATA="${RELEASEDIR}/RealSurface1.0" \
+       G4NEUTRONXSDATA="${RELEASEDIR}/G4NEUTRONXS1.0" \
+       G4PIIDATA="${RELEASEDIR}/G4PII1.2" \
+       /usr/bin/time --format="real %es\nuser %Us\nsys %Ss\nmaxmem %Mk" \
+       ${RELEASEDIR}/Gate ${PARAM} ${MACROFILE} \
+       || error "gate failed"
+fi
+
+echo "Copying data back"
+ssh -i ${HOME}/.ssh/ccin2p3 linux1.dg.creatis.insa-lyon.fr "mkdir cc/$(basename ${OUTPUTDIR})/output.${JOB_ID%%.*}"
+rsync -av --remove-source-files -e "ssh -i ${HOME}/.ssh/ccin2p3" output/ "linux1.dg.creatis.insa-lyon.fr:./cc/$(basename ${OUTPUTDIR})/output.${JOB_ID%%.*}"
+
+echo "Cleanup"
+rm -r ${LOCALMACRODIR}
+
+echo "Success!!!"

diff --git a/cluster_tools/gate_job_cluster.job b/cluster_tools/gate_job_cluster.job
index 1242dfd06fd9ecb32e4fd9c20345f225b8d35354..1ece1a611b0018daee535bf346a551ddc2121da4 100644 (file)
--- a/cluster_tools/gate_job_cluster.job
+++ b/cluster_tools/gate_job_cluster.job
@@ -12,9 +12,6 @@
 #PBS -l walltime=100:00:00
 #PBS -j oe
 
-#env
-#pwd
-#exit 1
 
 
 function error {

diff --git a/cluster_tools/gate_run_submit_cluster.sh b/cluster_tools/gate_run_submit_cluster.sh
index 4df0be78dbad8d0b307f796bd68c5eddadb2047a..ab348d91dd94679e05d2c4e4bd0a750ee9fb64b7 100755 (executable)
--- a/cluster_tools/gate_run_submit_cluster.sh
+++ b/cluster_tools/gate_run_submit_cluster.sh
@@ -24,10 +24,14 @@ function usage {
 
 test $# -eq 0 && usage && exit 0
 
-SCRIPTDIR="${HOME}/git/gate-tests/bin"
 RELEASESUFFIX=${3:-"${DEFAULTRELEASESUFFIX}"}
 RELEASEDIR="${HOME}/releases/grid_release${RELEASESUFFIX}"
-JOBFILE="$(dirname $0)/gate_job_cluster.job"
+if test "$(dnsdomainname)" = "in2p3.fr"
+then
+    JOBFILE="$(dirname $0)/gate_job_ccin2p3.job"
+else
+    JOBFILE="$(dirname $0)/gate_job_cluster.job"
+fi
 
 echo "Checking stuff"
 test -f ${JOBFILE} || error "can't find job file ${JOBFILE}"
@@ -42,7 +46,13 @@ MACRODIR=$(pwd)
 test -d ${MACRODIR}/mac && test -d ${MACRODIR}/data || error "invalid path"
 MACROFILE=${1:?"provide relative macro path"}
 test -f ${MACRODIR}/${MACROFILE} || error "invalid macro"
-OUTPUTDIR=$(mktemp --tmpdir=${MACRODIR} -d run.XXXX || error "can't create temp dir")
+if test "$(dnsdomainname)" = "in2p3.fr"
+then
+    OUTPUTDIR=$(mktemp -d -p "${MACRODIR}" run.XXXX || error "can't create temp dir")
+    ssh -i ${HOME}/.ssh/ccin2p3 linux1.dg.creatis.insa-lyon.fr mkdir -p "cc/$(basename ${OUTPUTDIR})"
+else
+    OUTPUTDIR=$(mktemp --tmpdir=${MACRODIR} -d run.XXXX || error "can't create temp dir")
+fi
 test -d ${OUTPUTDIR} || error "can't locate output dir"
 RUNID=${OUTPUTDIR##*.}
 NJOBS=${2:-"${DEFAULTNUMBEROFJOBS}"}
@@ -88,10 +98,17 @@ while test $NJOBS -gt 0; do
     if test "${QSUB}" = "noqsub"
     then
         echo "Launching Gate log in ${OUTPUTDIR}/gate_${NJOBS}.log"
-        PARAM=\"${PARAM}\" INDEX=${NJOBS} INDEXMAX=${NJOBSMAX} SCRIPTDIR=${SCRIPTDIR} OUTPUTDIR=${OUTPUTDIR}  RELEASEDIR=${RELEASEDIR} MACROFILE=${MACROFILE} MACRODIR=${MACRODIR} PBS_JOBID="local_${NJOBS}" bash "${JOBFILE}" > ${OUTPUTDIR}/gate_${NJOBS}.log &
+        PARAM=\"${PARAM}\" INDEX=${NJOBS} INDEXMAX=${NJOBSMAX} OUTPUTDIR=${OUTPUTDIR}  RELEASEDIR=${RELEASEDIR} MACROFILE=${MACROFILE} MACRODIR=${MACRODIR} PBS_JOBID="local_${NJOBS}" bash "${JOBFILE}" > ${OUTPUTDIR}/gate_${NJOBS}.log &
+    elif test "$(dnsdomainname)" = "in2p3.fr"
+    then
+        PROJECTGROUP=creatis 
+        qsub -o "${OUTPUTDIR}" \
+             -N "gate.${RUNID}" \
+             -v "PARAM=\"${PARAM}\",INDEX=${NJOBS},INDEXMAX=${NJOBSMAX},OUTPUTDIR=${OUTPUTDIR},RELEASEDIR=${RELEASEDIR},MACROFILE=${MACROFILE},MACRODIR=${MACRODIR}" \
+             "${JOBFILE}" || error "submission error"
     else
         qsub -N "gatejob.${RUNID}" -o "${OUTPUTDIR}" \
-           -v "PARAM=\"${PARAM}\",INDEX=${NJOBS},INDEXMAX=${NJOBSMAX},SCRIPTDIR=${SCRIPTDIR},OUTPUTDIR=${OUTPUTDIR},RELEASEDIR=${RELEASEDIR},MACROFILE=${MACROFILE},MACRODIR=${MACRODIR}" \
+           -v "PARAM=\"${PARAM}\",INDEX=${NJOBS},INDEXMAX=${NJOBSMAX},OUTPUTDIR=${OUTPUTDIR},RELEASEDIR=${RELEASEDIR},MACROFILE=${MACROFILE},MACRODIR=${MACRODIR}" \
            "${JOBFILE}" || error "submission error"
     fi
 
@@ -99,3 +116,7 @@ while test $NJOBS -gt 0; do
 done
 
 echo "runid is ${RUNID}"
+if test "$(dnsdomainname)" = "in2p3.fr"
+then
+    rsync -av --remove-source-files -e "ssh -i ${HOME}/.ssh/ccin2p3" ${OUTPUTDIR}/ "linux1.dg.creatis.insa-lyon.fr:./cc/$(basename ${OUTPUTDIR})"
+fi