cluster_tools/gate_power_merge.sh

   1 #!/usr/bin/env bash
   2
   3 set -u
   4
   5 function error {
   6 echo "ERROR: $1"
   7 exit 1
   8 }
   9
  10 warning_count=0
  11 function warning {
  12 let "warning_count++"
  13 echo "MERGE_WARNING: $1"
  14 }
  15
  16 function start_bar {
  17 count_max="${1:?"provide count max"}"
  18 }
  19
  20 function update_bar {
  21 local count="${1:?"provide count"}"
  22 local message="${2:?"provide message"}"
  23 local percent=$(echo "100*${count}/${count_max}" | bc)
  24 #printf "[%03d/%03d] %3d%% %-80.80s\r" ${count} ${count_max} ${percent} "${message}"
  25 printf "[%03d/%03d] %3d%% %-80.80s\n" ${count} ${count_max} ${percent} "${message}"
  26 }
  27
  28 function end_bar {
  29 unset count_max
  30 #echo -ne '\n'
  31 }
  32
  33 rootMerger="clitkMergeRootFiles"
  34 test -x "./clitkMergeRootFiles" && rootMerger="./clitkMergeRootFiles"
  35
  36 function merge_root {
  37 local merged="$1"
  38 shift
  39 echo "  ${indent}entering root merger"
  40 echo "  ${indent}merger is ${rootMerger}"
  41 echo "  ${indent}creating ${merged}"
  42 #echo "######## $#"
  43 #echo "######## $*"
  44
  45 if test $# -eq 1
  46 then
  47     echo "  ${indent}just one partial file => just copy it"
  48     cp "$1" "${merged}"
  49     return
  50 fi
  51
  52 local count=0
  53 local arguments=" -o ${merged}"
  54 while test $# -gt 0
  55 do
  56     local partial="$1"
  57     shift
  58     let count++
  59     local arguments=" -i ${partial} ${arguments}"
  60 done
  61 ${rootMerger} ${arguments} > /dev/null || warning "error while calling ${rootMerger}"
  62 echo "  ${indent}merged ${count} files"
  63 }
  64
  65 statMerger="mergeStatFile.py"
  66 test -x "./mergeStatFile.sh" && statMerger="./mergeStatFile.sh"
  67
  68 function merge_stat {
  69 local merged="$1"
  70 shift
  71 echo "  ${indent}entering stat merger"
  72 echo "  ${indent}merger is ${statMerger}"
  73 echo "  ${indent}creating ${merged}"
  74 local count=0
  75 start_bar $#
  76 while test $# -gt 0
  77 do
  78     local partial="$1"
  79     shift
  80     let count++
  81
  82     if test ! -f "${merged}"
  83     then
  84         update_bar ${count} "copying first partial result ${partial}"
  85         cp "${partial}" "${merged}"
  86         continue
  87     fi
  88
  89     update_bar ${count} "adding ${partial}"
  90     ${statMerger} -i "${merged}" -j "${partial}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${statMerger}"
  91 done
  92 end_bar
  93 echo "  ${indent}merged ${count} files"
  94 }
  95
  96 txtImageMerger="clitkMergeAsciiDoseActor"
  97 test -f "./clitkMergeAsciiDoseActor" && txtImageMerger="./clitkMergeAsciiDoseActor"
  98
  99 function merge_txt_image {
 100 local merged="$1"
 101 shift
 102 echo "  ${indent}entering text image merger"
 103 echo "  ${indent}merger is ${txtImageMerger}"
 104 echo "  ${indent}creating ${merged}"
 105 local count=0
 106 start_bar $#
 107 while test $# -gt 0
 108 do
 109     local partial="$1"
 110     shift
 111     let count++
 112
 113     if test ! -f "${merged}"
 114     then
 115         update_bar ${count} "copying first partial result ${partial}"
 116         cp "${partial}" "${merged}"
 117         continue
 118     fi
 119
 120     update_bar ${count} "adding ${partial}"
 121     local header="$(cat "${merged}" | head -n 6)"
 122     local tmp="$(mktemp)"
 123     ${txtImageMerger} -i "${partial}" -j "${merged}" -o "${tmp}" 2> /dev/null > /dev/null || warning "error while calling ${txtImageMerger}"
 124     echo "${header}" > "${merged}"
 125     grep -v '## Merge' "${tmp}" >> "${merged}"
 126     rm "${tmp}"
 127 done
 128 end_bar
 129 echo "  ${indent}merged ${count} files"
 130 }
 131
 132 hdrImageMerger="clitkImageArithm"
 133 test -x "./clitkImageArithm" && hdrImageMerger="./clitkImageArithm"
 134
 135 function merge_hdr_image {
 136 local merged="$1"
 137 local merged_bin="${merged%.*}.img"
 138 shift
 139 echo "  ${indent}entering hdr image merger"
 140 echo "  ${indent}merger is ${hdrImageMerger}"
 141 echo "  ${indent}creating ${merged}"
 142 local count=0
 143 start_bar $#
 144 while test $# -gt 0
 145 do
 146     local partial="$1"
 147     local partial_bin="${partial%.*}.img"
 148     shift
 149     let count++
 150
 151     if test ! -f "${merged}"
 152     then
 153         update_bar ${count} "copying first partial result ${partial}"
 154         cp "${partial}" "${merged}"
 155         cp "${partial_bin}" "${merged_bin}"
 156         continue
 157     fi
 158
 159     update_bar ${count} "adding ${partial}"
 160     ${hdrImageMerger} -t 0 -i "${partial}" -j "${merged}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${hdrImageMerger}"
 161 done
 162 end_bar
 163 echo "  ${indent}merged ${count} files"
 164 }
 165
 166 mhdImageMerger="clitkImageArithm"
 167 test -x "./clitkImageArithm" && mhdImageMerger="./clitkImageArithm"
 168
 169 function merge_mhd_image {
 170 local merged="$1"
 171 local merged_bin="${merged%.*}.raw"
 172 shift
 173 echo "  ${indent}entering mhd image merger"
 174 echo "  ${indent}merger is ${mhdImageMerger}"
 175 echo "  ${indent}creating ${merged}"
 176 local count=0
 177 start_bar $#
 178 while test $# -gt 0
 179 do
 180     local partial="$1"
 181     local partial_bin="${partial%.*}.raw"
 182     shift
 183     let count++
 184
 185     if test ! -f "${merged}"
 186     then
 187         update_bar ${count} "copying first partial result ${partial}"
 188         cp "${partial}" "${merged}"
 189         cp "${partial_bin}" "${merged_bin}"
 190         continue
 191     fi
 192
 193     update_bar ${count} "adding ${partial}"
 194     ${mhdImageMerger} -t 0 -i "${partial}" -j "${merged}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${mhdImageMerger}"
 195 done
 196 end_bar
 197 echo "  ${indent}merged ${count} files"
 198 }
 199
 200 function merge_dispatcher {
 201     local indent="  ** "
 202     local outputfile="${1:?"provide output filename"}"
 203     echo "merging ${outputfile}"
 204
 205     local partialoutputfiles="$(find "${rundir}" -type f -name "${outputfile}")"
 206     local nboutputfiles="$(echo "${partialoutputfiles}" | wc -l)"
 207     if test ${nboutputdirs} -ne ${nboutputfiles}
 208     then
 209         warning "missing files"
 210         return
 211     fi
 212
 213     local firstpartialoutputfile="$(echo "${partialoutputfiles}" | head -n 1)"
 214     local firstpartialoutputextension="${firstpartialoutputfile##*.}"
 215     echo "${indent}testing file type on ${firstpartialoutputfile}"
 216
 217     if test "${firstpartialoutputextension}" == "hdr"
 218     then
 219         echo "${indent}this is a analyse image"
 220         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 221         merge_hdr_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 222         return
 223     fi
 224
 225     if test "${firstpartialoutputextension}" == "mhd"
 226     then
 227         echo "${indent}this is a mhd image"
 228         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 229         merge_mhd_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 230         return
 231     fi
 232
 233     if test "${firstpartialoutputextension}" == "root"
 234     then
 235         echo "${indent}this is a root file"
 236         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 237         merge_root "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 238         return
 239     fi
 240
 241     if test "${firstpartialoutputextension}" == "txt" && grep 'NumberOfEvent' "${firstpartialoutputfile}" > /dev/null
 242     then
 243         echo "${indent}this is a stat file"
 244         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 245         merge_stat "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 246         return
 247     fi
 248
 249     if test "${firstpartialoutputextension}" == "txt" && grep 'Resol' "${firstpartialoutputfile}" > /dev/null
 250     then
 251         local resol="$(sed -nr '/Resol/s/^.*=\s+\((.+)\)\s*$/\1/p' "${firstpartialoutputfile}")"
 252         local resolx="$(echo "${resol}" | cut -d',' -f1)"
 253         local resoly="$(echo "${resol}" | cut -d',' -f2)"
 254         local resolz="$(echo "${resol}" | cut -d',' -f3)"
 255         if test "${resol}" == "1,1,1"
 256         then
 257             echo "${indent}this is a txt integral value"
 258             local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 259             merge_txt_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 260             return
 261         fi
 262         if test \( "${resolx}" == "1" -a "${resoly}" == "1" \) -o \( "${resoly}" == "1" -a "${resolz}" == "1" \) -o \( "${resolz}" == "1" -a "${resolx}" == "1" \)
 263         then
 264             echo "${indent}this is a txt profile"
 265             local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 266             merge_txt_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
 267             return
 268         fi
 269     fi
 270
 271     if test "${firstpartialoutputextension}" == "txt"
 272     then
 273         echo "${indent}this is a non specific txt output"
 274         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
 275         echo "  ${indent}catting ${mergedfile}"
 276         cat ${partialoutputfiles} > "${mergedfile}" || error "error while merging"
 277         return
 278     fi
 279
 280     warning "unknown file type"
 281 }
 282
 283 echo "!!!! this is $0 v0.3h !!!!"
 284
 285 rundir="${1?"provide run dir"}"
 286 rundir="$(echo "${rundir}" | sed 's|/*$||')"
 287 nboutputdirs="$(find "${rundir}" -mindepth 1 -type d -name 'output*' | wc -l)"
 288
 289 test ${nboutputdirs} -gt 0 || error "no output dir found"
 290 echo "found ${nboutputdirs} partial output dirs"
 291
 292 outputdir="results"
 293 if [ "${rundir}" != "." -a "${rundir##*.}" != "${rundir}" ]
 294 then
 295     outputdir="results.${rundir##*.}"
 296 fi
 297 outputdir="$(basename "${outputdir}")"
 298 echo "output dir is ${outputdir}"
 299 test -d "${outputdir}" && rm -r "${outputdir}"
 300 mkdir "${outputdir}"
 301
 302 for outputfile in $(find "${rundir}" -regextype 'posix-extended' -type f -regex "${rundir}/output.*\.(hdr|mhd|root|txt)" | awk -F '/' '{ print $NF; }' | sort | uniq)
 303 do
 304     merge_dispatcher "${outputfile}"
 305 done
 306
 307 if [ -f "${rundir}/params.txt" ]
 308 then
 309         echo "copying params file"
 310         cp "${rundir}/params.txt" "${outputdir}/params.txt"
 311 fi
 312
 313 echo "these was ${warning_count} warning(s)"