cluster_tools/gate_power_merge.sh

   1 #!/usr/bin/env bash
   2
   3 set -u
   4
   5 function error {
   6 echo "ERROR: $1"
   7 exit 1
   8 }
   9
  10 warning_count=0
  11 function warning {
  12 let "warning_count++"
  13 echo "MERGE_WARNING: $1"
  14 }
  15
  16 function start_bar {
  17 count_max="${1:?"provide count max"}"
  18 }
  19
  20 function update_bar {
  21 local count="${1:?"provide count"}"
  22 local message="${2:?"provide message"}"
  23 local percent=$(echo "100*${count}/${count_max}" | bc)
  24 #printf "[%03d/%03d] %3d%% %-80.80s\r" ${count} ${count_max} ${percent} "${message}"
  25 printf "[%03d/%03d] %3d%% %-80.80s\n" ${count} ${count_max} ${percent} "${message}"
  26 }
  27
  28 function end_bar {
  29 unset count_max
  30 #echo -ne '\n'
  31 }
  32
  33 rootMerger="clitkMergeRootFiles"
  34 test -x "./clitkMergeRootFiles" && rootMerger="./clitkMergeRootFiles"
  35
  36 function merge_root {
  37 local merged="$1"
  38 shift
  39 echo "  ${indent}entering root merger"
  40 echo "  ${indent}merger is ${rootMerger}"
  41 echo "  ${indent}creating ${merged}"
  42 #echo "######## $#"
  43 #echo "######## $*"
  44
  45 if test $# -eq 1
  46 then
  47     echo "  ${indent}just one partial file => just copy it"
  48     cp "$1" "${merged}"
  49     return
  50 fi
  51
  52 local count=0
  53 local arguments=" -o ${merged}"
  54 while test $# -gt 0
  55 do
  56     local partial="$1"
  57     shift
  58     let count++
  59     local arguments=" -i ${partial} ${arguments}"
  60 done
  61 ${rootMerger} ${arguments} > /dev/null || warning "error while calling ${rootMerger}"
  62 echo "  ${indent}merged ${count} files"
  63 }
  64
  65 statMerger="mergeStatFile.py"
  66 test -x "./mergeStatFile.sh" && statMerger="./mergeStatFile.sh"
  67
  68 function merge_stat {
  69 local merged="$1"
  70 shift
  71 echo "  ${indent}entering stat merger"
  72 echo "  ${indent}merger is ${statMerger}"
  73 echo "  ${indent}creating ${merged}"
  74 local count=0
  75 start_bar $#
  76 while test $# -gt 0
  77 do
  78     local partial="$1"
  79     shift
  80     let count++
  81
  82     if test ! -f "${merged}"
  83     then
  84         update_bar ${count} "copying first partial result ${partial}"
  85         cp "${partial}" "${merged}"
  86         continue
  87     fi
  88
  89     update_bar ${count} "adding ${partial}"
  90     ${statMerger} -i "${merged}" -j "${partial}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${statMerger}"
  91 done
  92 end_bar
  93 echo "  ${indent}merged ${count} files"
  94 }
  95
  96 txtImageMerger="clitkMergeAsciiDoseActor"
  97 test -f "./clitkMergeAsciiDoseActor" && txtImageMerger="./clitkMergeAsciiDoseActor"
  98
  99 function merge_txt_image {
 100 local merged="$1"
 101 shift
 102 echo "  ${indent}entering text image merger"
 103 echo "  ${indent}merger is ${txtImageMerger}"
 104 echo "  ${indent}creating ${merged}"
 105 local count=0
 106 start_bar $#
 107 while test $# -gt 0
 108 do
 109     local partial="$1"
 110     shift
 111     let count++
 112
 113     if test ! -f "${merged}"
 114     then
 115         update_bar ${count} "copying first partial result ${partial}"
 116         cp "${partial}" "${merged}"
 117         continue
 118     fi
 119
 120     update_bar ${count} "adding ${partial}"
 121     local header="$(cat "${merged}" | head -n 6)"
 122     local tmp="$(mktemp)"
 123     ${txtImageMerger} -i "${partial}" -j "${merged}" -o "${tmp}" 2> /dev/null > /dev/null || warning "error while calling ${txtImageMerger}"
 124     echo "${header}" > "${merged}"
 125     grep -v '## Merge' "${tmp}" >> "${merged}"
 126     rm "${tmp}"
 127 done
 128 end_bar
 129 echo "  ${indent}merged ${count} files"
 130 }
 131
 132 hdrImageMerger="clitkImageArithm"
 133 test -x "./clitkImageArithm" && hdrImageMerger="./clitkImageArithm"
 134
 135 function merge_hdr_image {
 136 local merged="$1"
 137 local merged_bin="${merged%.*}.img"
 138 shift
 139 echo "  ${indent}entering hdr image merger"
 140 echo "  ${indent}merger is ${hdrImageMerger}"
 141 echo "  ${indent}creating ${merged}"
 142 local count=0
 143 start_bar $#
 144 while test $# -gt 0
 145 do
 146     local partial="$1"
 147     local partial_bin="${partial%.*}.img"
 148     shift
 149     let count++
 150
 151     if test ! -f "${merged}"
 152     then
 153         update_bar ${count} "copying first partial result ${partial}"
 154         cp "${partial}" "${merged}"
 155         cp "${partial_bin}" "${merged_bin}"
 156         continue
 157     fi
 158
 159     update_bar ${count} "adding ${partial}"
 160     ${hdrImageMerger} -t 0 -i "${partial}" -j "${merged}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${hdrImageMerger}"
 161 done
 162 end_bar
 163 echo "  ${indent}merged ${count} files"
 164 }
 165
 166 function merge_dispatcher {
 167     local indent="  ** "
 168     local outputfile="${1:?"provide output filename"}"
 169     echo "merging ${outputfile}"
 170
 171     local partialoutputfiles="$(find "${rundir}" -type f -name "${outputfile}")"
 172     local nboutputfiles="$(echo "${partialoutputfiles}" | wc -l)"
 173     if test ${nboutputdirs} -ne ${nboutputfiles}
 174     then
 175         warning "missing files"
 176         return
 177     fi
 178
 179     local firstpartialoutputfile="$(echo "${partialoutputfiles}" | head -n 1)"
 180     local firstpartialoutputextension="${firstpartialoutputfile##*.}"
 181     echo "${indent}testing file type on ${firstpartialoutputfile}"
 182
 183     if test "${firstpartialoutputextension}" == "hdr"
 184     then
 185         echo "${indent}this is a analyse image"
 186         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 187         merge_hdr_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 188         return
 189     fi
 190
 191     if test "${firstpartialoutputextension}" == "root"
 192     then
 193         echo "${indent}this is a root file"
 194         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 195         merge_root "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 196         return
 197     fi
 198
 199     if test "${firstpartialoutputextension}" == "txt" && grep 'NumberOfEvent' "${firstpartialoutputfile}" > /dev/null
 200     then
 201         echo "${indent}this is a stat file"
 202         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 203         merge_stat "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 204         return
 205     fi
 206
 207     if test "${firstpartialoutputextension}" == "txt" && grep 'Resol' "${firstpartialoutputfile}" > /dev/null
 208     then
 209         local resol="$(sed -nr '/Resol/s/^.*=\s+\((.+)\)\s*$/\1/p' "${firstpartialoutputfile}")"
 210         local resolx="$(echo "${resol}" | cut -d',' -f1)"
 211         local resoly="$(echo "${resol}" | cut -d',' -f2)"
 212         local resolz="$(echo "${resol}" | cut -d',' -f3)"
 213         if test "${resol}" == "1,1,1"
 214         then
 215             echo "${indent}this is a txt integral value"
 216             local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 217             merge_txt_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 218             return
 219         fi
 220         if test \( "${resolx}" == "1" -a "${resoly}" == "1" \) -o \( "${resoly}" == "1" -a "${resolz}" == "1" \) -o \( "${resolz}" == "1" -a "${resolx}" == "1" \)
 221         then
 222             echo "${indent}this is a txt profile"
 223             local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 224             merge_txt_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 225             return
 226         fi
 227     fi
 228
 229     warning "unknown file type"
 230 }
 231
 232 echo "!!!! this is $0 v0.3f !!!!"
 233
 234 rundir="${1?"provide run dir"}"
 235 nboutputdirs="$(find "${rundir}" -mindepth 1 -type d -name 'output*' | wc -l)"
 236
 237 test ${nboutputdirs} -gt 0 || error "no output dir found"
 238 echo "found ${nboutputdirs} partial output dirs"
 239
 240 outputdir="results"
 241 if [ "${rundir}" != "." -a "${rundir##*.}" != "${rundir}" ]
 242 then
 243     outputdir="results.${rundir##*.}"
 244 fi
 245 outputdir="$(basename "${outputdir}")"
 246 echo "output dir is ${outputdir}"
 247 test -d "${outputdir}" && rm -r "${outputdir}"
 248 mkdir "${outputdir}"
 249
 250 for outputfile in $(find "${rundir}" -regextype 'posix-extended' -type f -regex '.*\.(hdr|root|txt)' | awk -F '/' '{ print $NF }' | sort | uniq)
 251 do
 252     merge_dispatcher "${outputfile}"
 253 done
 254
 255 if [ -f "${rundir}/params.txt" ]
 256 then
 257         echo "copying params file"
 258         cp "${rundir}/params.txt" "${outputdir}/params.txt"
 259 fi
 260
 261 echo "these was ${warning_count} warning(s)"