13 echo "MERGE_WARNING: $1"
17 count_max="${1:?"provide count max"}"
21 local count="${1:?"provide count"}"
22 local message="${2:?"provide message"}"
23 local percent=$(echo "100*${count}/${count_max}" | bc)
24 #printf "[%03d/%03d] %3d%% %-80.80s\r" ${count} ${count_max} ${percent} "${message}"
25 printf "[%03d/%03d] %3d%% %-80.80s\n" ${count} ${count_max} ${percent} "${message}"
33 function check_interfile {
34 local input_interfile="${1:?"provide input interfile"}"
36 grep -qs '!INTERFILE :=' "${input_interfile}" || return 1
38 local header_byte_size=$(awk -F' ' '
40 /matrix size/ && $3 == "[1]" { xsize = $5; }
41 /matrix size/ && $3 == "[2]" { ysize = $5; }
42 /number of projections/ { zsize += $5; }
43 /number of bytes per pixel/ { byte_per_pixel = $7; }
44 END { print xsize * ysize * zsize * byte_per_pixel; }' "${input_interfile}")
46 local raw_interfile="$(dirname "${input_interfile}")/$(awk -F' := ' '/name of data file/ { print $2; }' "${input_interfile}")"
48 test -f "${raw_interfile}" || return 1
49 test $(stat -c%s "${raw_interfile}") -eq ${header_byte_size} || return 1
52 function write_mhd_header {
53 local input_interfile="${1:?"provide input interfile"}"
54 local output_mhd="$(dirname "${input_interfile}")/$(basename "${input_interfile}" ".hdr").mhd"
56 check_interfile "${input_interfile}" || error "${input_interfile} isn't an interfile image"
58 local header_start='ObjectType = Image
60 AcquisitionDate = none
62 BinaryDataByteOrderMSB = False
63 CompressedData = False
64 TransformMatrix = 1 0 0 0 1 0 0 0 1
66 CenterOfRotation = 0 0 0
68 AnatomicalOrientation = RIP'
70 echo "${header_start}" > "${output_mhd}"
73 /scaling factor/ && $4 == "[1]" { xspacing = $6; }
74 /scaling factor/ && $4 == "[2]" { yspacing = $6; }
75 END { print "ElementSpacing = " xspacing " " yspacing " 1"; }' "${input_interfile}" >> "${output_mhd}"
79 /matrix size/ && $3 == "[1]" { xsize = $5; }
80 /matrix size/ && $3 == "[2]" { ysize = $5; }
81 /number of projections/ { zsize += $5; }
82 END { print "DimSize = " xsize " " ysize " " zsize; }' "${input_interfile}" >> "${output_mhd}"
85 /number format/ { format = $2; }
86 /number of bytes per pixel/ { byte_per_pixel = $2 }
88 if (format == "unsigned integer" && byte_per_pixel == 8) { print "ElementType = MET_ULONG"; exit };
89 if (format == "unsigned integer" && byte_per_pixel == 4) { print "ElementType = MET_UINT"; exit };
90 if (format == "unsigned integer" && byte_per_pixel == 2) { print "ElementType = MET_USHORT"; exit };
91 if (format == "unsigned integer" && byte_per_pixel == 1) { print "ElementType = MET_UCHAR"; exit };
92 if (format == "integer" && byte_per_pixel == 8) { print "ElementType = MET_LONG"; exit };
93 if (format == "integer" && byte_per_pixel == 4) { print "ElementType = MET_INT"; exit };
94 if (format == "integer" && byte_per_pixel == 2) { print "ElementType = MET_SHORT"; exit };
95 if (format == "integer" && byte_per_pixel == 1) { print "ElementType = MET_CHAR"; exit };
96 if (format == "float" && byte_per_pixel == 8) { print "ElementType = MET_FLOAT"; exit };
97 if (format == "float" && byte_per_pixel == 4) { print "ElementType = MET_DOUBLE"; exit };
98 print "ElementType = MET_INT";
99 }' "${input_interfile}" >> "${output_mhd}"
102 /name of data file/ { print "ElementDataFile = " $2; }' "${input_interfile}" >> "${output_mhd}"
105 rootMerger="clitkMergeRootFiles"
106 test -x "./clitkMergeRootFiles" && rootMerger="./clitkMergeRootFiles"
108 function merge_root {
111 echo " ${indent}entering root merger"
112 echo " ${indent}merger is ${rootMerger}"
113 echo " ${indent}creating ${merged}"
119 echo " ${indent}just one partial file => just copy it"
125 local arguments=" -o ${merged}"
131 local arguments=" -i ${partial} ${arguments}"
133 ${rootMerger} ${arguments} > /dev/null || error "error while calling ${rootMerger}"
134 echo " ${indent}merged ${count} files"
137 statMerger="mergeStatFile.py"
138 test -x "./mergeStatFile.sh" && statMerger="./mergeStatFile.sh"
140 function merge_stat {
143 echo " ${indent}entering stat merger"
144 echo " ${indent}merger is ${statMerger}"
145 echo " ${indent}creating ${merged}"
154 if test ! -f "${merged}"
156 update_bar ${count} "copying first partial result ${partial}"
157 cp "${partial}" "${merged}"
161 update_bar ${count} "adding ${partial}"
162 ${statMerger} -i "${merged}" -j "${partial}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${statMerger}"
165 echo " ${indent}merged ${count} files"
168 doseMerger="mergeDosePerEnergyFile.sh"
169 test -x "./mergeDosePerEnergyFile.sh" && doseMerger="./mergeDosePerEnergyFile.sh"
171 function merge_dose {
174 echo " ${indent}entering dose merger"
175 echo " ${indent}merger is ${doseMerger}"
176 echo " ${indent}creating ${merged}"
185 if test ! -f "${merged}"
187 update_bar ${count} "copying first partial result ${partial}"
188 cp "${partial}" "${merged}"
192 update_bar ${count} "adding ${partial}"
193 ${doseMerger} -i "${merged}" -j "${partial}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${doseMerger}"
196 echo " ${indent}merged ${count} files"
199 doseMerger="mergeDoseByRegions.sh"
200 test -x "./mergeDoseByRegions.sh" && doseMerger="./mergeDoseByRegions.sh"
202 function merge_doseByRegions {
205 echo " ${indent}entering dose merger"
206 echo " ${indent}merger is ${doseMerger}"
207 echo " ${indent}creating ${merged}"
217 if test ! -f "${merged}"
219 update_bar ${count} "copying first partial result ${partial}"
220 cp "${partial}" "${merged}"
221 copyFirstPartialResult -i "${merged}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${doseMerger}"
225 update_bar ${count} "adding ${partial}"
226 addToPartialResult -i "${merged}" -j "${partial}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${doseMerger}"
228 divideUncertaintyResult -i "${merged}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${doseMerger}"
230 echo " ${indent}merged ${count} files"
233 txtImageMerger="clitkMergeAsciiDoseActor"
234 test -f "./clitkMergeAsciiDoseActor" && txtImageMerger="./clitkMergeAsciiDoseActor"
236 function merge_txt_image {
239 echo " ${indent}entering text image merger"
240 echo " ${indent}merger is ${txtImageMerger}"
241 echo " ${indent}creating ${merged}"
250 if test ! -f "${merged}"
252 update_bar ${count} "copying first partial result ${partial}"
253 cp "${partial}" "${merged}"
257 update_bar ${count} "adding ${partial}"
258 local header="$(cat "${merged}" | head -n 6)"
259 local tmp="$(mktemp)"
260 ${txtImageMerger} -i "${partial}" -j "${merged}" -o "${tmp}" 2> /dev/null > /dev/null || warning "error while calling ${txtImageMerger}"
261 echo "${header}" > "${merged}"
262 grep -v '## Merge' "${tmp}" >> "${merged}"
266 echo " ${indent}merged ${count} files"
269 hdrImageMerger="clitkImageArithm"
270 test -x "./clitkImageArithm" && hdrImageMerger="./clitkImageArithm"
272 function merge_hdr_image {
274 local merged_bin="${merged%.*}.img"
276 echo " ${indent}entering hdr image merger"
277 echo " ${indent}merger is ${hdrImageMerger}"
278 echo " ${indent}creating ${merged}"
284 local partial_bin="${partial%.*}.img"
288 if test ! -f "${merged}"
290 update_bar ${count} "copying first partial result ${partial}"
291 cp "${partial}" "${merged}"
292 cp "${partial_bin}" "${merged_bin}"
296 update_bar ${count} "adding ${partial}"
297 ${hdrImageMerger} -t 0 -i "${partial}" -j "${merged}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${hdrImageMerger}"
300 echo " ${indent}merged ${count} files"
303 mhdImageMerger="clitkImageArithm"
304 test -x "./clitkImageArithm" && mhdImageMerger="./clitkImageArithm"
306 function merge_mhd_image {
308 local merged_bin="${merged%.*}.raw"
309 local last_character=${merged#${merged%?}}
311 echo " ${indent}entering mhd image merger"
312 echo " ${indent}merger is ${mhdImageMerger}"
313 echo " ${indent}creating ${merged}"
319 local partial_bin="$(dirname "${partial}")/$(awk -F' = ' '/ElementDataFile/ { print $2; }' "${partial}")"
323 if test ! -f "${merged}"
325 update_bar ${count} "copying first partial result ${partial}"
326 cp "${partial}" "${merged}"
327 if test "$last_character" = "d"
329 cp "${partial_bin}" "${merged_bin%.*}.${partial_bin##*.}"
334 update_bar ${count} "adding ${partial}"
335 ${mhdImageMerger} -t 0 -i "${partial}" -j "${merged}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${mhdImageMerger}"
336 if test "$last_character" = "d" && test "${merged_bin}" != "${merged_bin%.*}.${partial_bin##*.}"
338 mv "${merged_bin}" "${merged_bin%.*}.${partial_bin##*.}"
339 sed -i "s/$(basename "${merged_bin}")/$(basename "${merged_bin%.*}.${partial_bin##*.}")/" "${merged}"
343 echo " ${indent}merged ${count} files"
346 function merge_dispatcher {
348 local outputfile="${1:?"provide output filename"}"
349 echo "merging ${outputfile}"
351 local partialoutputfiles="$(find -L "${rundir}" -mindepth 2 -type f -name "${outputfile}")"
352 local nboutputfiles="$(echo "${partialoutputfiles}" | wc -l)"
353 if test ${nboutputdirs} -ne ${nboutputfiles}
355 warning "missing files"
356 if ! test "${2}" == "--force"
362 local firstpartialoutputfile="$(echo "${partialoutputfiles}" | head -n 1)"
363 local firstpartialoutputextension="${firstpartialoutputfile##*.}"
364 echo "${indent}testing file type on ${firstpartialoutputfile}"
366 if test "${firstpartialoutputextension}" == "hdr" && test grep -qs 'INTERFILE' "${firstpartialoutputfile}"
368 echo "${indent}this is an interfile image"
369 echo "${indent}creating mhd headers"
370 for partialoutputfile in $partialoutputfiles; do write_mhd_header "${partialoutputfile}"; done
371 local mhd_partialoutputfiles="$(for partialoutputfile in $partialoutputfiles; do echo "${partialoutputfile%.*}.mhd"; done)"
372 local mhd_mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}" ".hdr").mhd"
373 merge_mhd_image "${mhd_mergedfile}" ${mhd_partialoutputfiles} || error "error while merging"
374 echo "${indent}cleaning mhd headers"
375 for mhd_partialoutputfile in $mhd_partialoutputfiles; do rm "${mhd_partialoutputfile}"; done
376 rm "${mhd_mergedfile}"
377 echo "${indent}copy interfile header"
378 cp "${firstpartialoutputfile}" "${outputdir}"
382 if test "${firstpartialoutputextension}" == "hdr"
384 echo "${indent}this is an analyse image"
385 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
386 merge_hdr_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
390 if test "${firstpartialoutputextension}" == "mhd" || test "${firstpartialoutputextension}" == "mha"
392 echo "${indent}this is a mhd image"
393 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
394 merge_mhd_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
398 if test "${firstpartialoutputextension}" == "root"
400 echo "${indent}this is a root file"
401 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
402 merge_root "${mergedfile}" ${partialoutputfiles} || error "error while merging"
406 if test "${firstpartialoutputextension}" == "txt" && grep -qs 'NumberOfEvent' "${firstpartialoutputfile}"
408 echo "${indent}this is a stat file"
409 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
410 merge_stat "${mergedfile}" ${partialoutputfiles} || error "error while merging"
414 if test "${firstpartialoutputextension}" == "txt" && grep -qs 'energydose' "${firstpartialoutputfile}"
416 echo "${indent}this is a dose file"
417 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
418 merge_dose "${mergedfile}" ${partialoutputfiles} || error "error while merging"
424 if test "${firstpartialoutputextension}" == "txt" && grep -qs 'Resol' "${firstpartialoutputfile}"
426 local resol="$(sed -nr '/Resol/s/^.*=\s+\((.+)\)\s*$/\1/p' "${firstpartialoutputfile}")"
427 local resolx="$(echo "${resol}" | cut -d',' -f1)"
428 local resoly="$(echo "${resol}" | cut -d',' -f2)"
429 local resolz="$(echo "${resol}" | cut -d',' -f3)"
430 if test "${resol}" == "1,1,1"
432 echo "${indent}this is a txt integral value"
433 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
434 merge_txt_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
437 if test \( "${resolx}" == "1" -a "${resoly}" == "1" \) -o \( "${resoly}" == "1" -a "${resolz}" == "1" \) -o \( "${resolz}" == "1" -a "${resolx}" == "1" \)
439 echo "${indent}this is a txt profile"
440 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
441 merge_txt_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
446 if test "${firstpartialoutputextension}" == "txt" && grep -qs 'vol(mm3)' "${firstpartialoutputfile}"
448 echo "${indent}this is a DoseByRegions file"
449 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
450 merge_doseByRegions "${mergedfile}" ${partialoutputfiles} || error "error while merging"
455 if test "${firstpartialoutputextension}" == "txt"
457 echo "${indent}this is a non specific txt output"
458 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
459 local nbdifferent="$(md5sum ${partialoutputfiles} | awk '{ print $1; }' | sort | uniq | wc -l)"
460 echo " ${indent}${nbdifferent} different files"
461 if test ${nbdifferent} -gt 1
463 echo " ${indent}catting to ${mergedfile}"
464 cat ${partialoutputfiles} > "${mergedfile}" || error "error while merging"
467 echo " ${indent}moving to ${mergedfile}"
468 cp "${firstpartialoutputfile}" "${mergedfile}" || error "error while merging"
473 error "unknown file type"
476 function merge_dispatcher_uncertainty {
478 local outputfile="${1:?"provide output filename"}"
480 local partialoutputfiles="$(find -L "${rundir}" -mindepth 2 -type f -name "${outputfile}")"
481 local nboutputfiles="$(echo "${partialoutputfiles}" | wc -l)"
482 if test ${nboutputdirs} -ne ${nboutputfiles}
484 warning "missing files"
485 if ! test "${2}" == "--force"
491 local firstpartialoutputfile="$(echo "${partialoutputfiles}" | head -n 1)"
492 local firstpartialoutputextension="${firstpartialoutputfile##*.}"
494 if [[ "${firstpartialoutputfile}" == *Uncertainty* ]]
496 if test "${firstpartialoutputextension}" == "mhd" || test "${firstpartialoutputextension}" == "mha"
498 echo "${indent}Uncertainty file found: ${firstpartialoutputfile}"
500 local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
501 summed_merged_file=${mergedfile//-Uncertainty/}
502 if [ ! -f ${summed_merged_file} ];
504 warning "${summed_merged_file} does not exist. Error, no uncertainty computed"
507 echo "${indent}${summed_merged_file} found"
508 ## search for Squared
509 squared_merged_file=${mergedfile//-Uncertainty/-Squared}
510 if [ ! -f ${squared_merged_file} ];
512 warning "${squared_merged_file} does not exist. Error, no uncertainty computed"
515 echo "${indent}${squared_merged_file} found"
516 ## search for NumberOfEvent
519 if [[ "$unamestr" == 'Darwin' ]]; then
520 files=$(find -L "${rundir}" -type f -regex ".*output.*[hdr|mhd|mha|root|txt]" | awk -F '/' '{ print $NF; }' | sort | uniq)
522 files=$(find -L "${rundir}" -regextype 'posix-extended' -type f -regex "${rundir}/output.*\.(hdr|mhd|mha|root|txt)" | awk -F '/' '{ print $NF; }' | sort | uniq)
525 for outputfile in ${files}
528 if grep -q 'NumberOfEvent' "${outputdir}/${outputfile}"
530 totalEvents="$(grep "NumberOfEvents" "${outputdir}/${outputfile}" | cut -d' ' -f4)"
531 echo "${indent}Find the NumberOfEvent in $outputfile: ${totalEvents}"
535 if test ${totalEvents} -gt 0
537 uncerImageMerger="clitkImageUncertainty"
538 test -x "./clitkImageUncertainty" && uncerImageMerger="./clitkImageUncertainty"
539 ${uncerImageMerger} -i ${summed_merged_file} -s ${squared_merged_file} -o ${mergedfile} -n ${totalEvents}
541 warning "${totalEvents} not positive. A at least one stat file (SimulationStatisticActor) must be provided. Error, no uncertainty computed"
545 error "merge_dispatcher_uncertainty does not handle ${firstpartialoutputfile} files"
551 echo "!!!! this is $0 v0.3k !!!!"
552 echo "Usage: gate_power_merge.sh run.dir [--force]"
553 echo " where --force allows to merge files even if the file is missing in some output directories"
555 rundir="${1?"provide run dir"}"
557 rundir="$(echo "${rundir}" | sed 's|/*$||')"
558 nboutputdirs="$(find "${rundir}" -mindepth 1 -type d -name 'output*' -o -type l -name 'output*' | wc -l)"
560 test ${nboutputdirs} -gt 0 || error "no output dir found"
561 echo "found ${nboutputdirs} partial output dirs"
564 if [ "${rundir}" != "." -a "${rundir##*.}" != "${rundir}" ]
566 outputdir="results.${rundir##*.}"
568 outputdir="$(basename "${outputdir}")"
569 echo "output dir is ${outputdir}"
571 test -d "${outputdir}" && rm -r "${outputdir}"
575 if [[ "$unamestr" == 'Darwin' ]]; then
576 files=$(find -L "${rundir}" -type f -regex ".*output.*[hdr|mhd|mha|root|txt]" | awk -F '/' '{ print $NF; }' | sort | uniq)
578 files=$(find -L "${rundir}" -regextype 'posix-extended' -type f -regex "${rundir}/output.*\.(hdr|mhd|mha|root|txt)" | awk -F '/' '{ print $NF; }' | sort | uniq)
581 for outputfile in ${files}
583 merge_dispatcher "${outputfile}" "${force}"
587 echo "Merging done. Special case for statistical uncertainty"
590 if [[ "$unamestr" == 'Darwin' ]]; then
591 files=$(find -L "${outputdir}" -type f -regex ".*[hdr|mhd|mha|root|txt]" | awk -F '/' '{ print $NF; }' | sort | uniq)
593 files=$(find -L "${outputdir}" -regextype 'posix-extended' -type f -regex "${outputdir}/.*\.(hdr|mhd|mha|root|txt)" | awk -F '/' '{ print $NF; }' | sort | uniq)
596 for outputfile in ${files}
598 merge_dispatcher_uncertainty "${outputfile}" "${force}"
601 echo "compute job statistics"
602 python computeEnlapsedTime.py ${rundir} gate
603 mv "statJobs.txt" "${outputdir}/statJobs.txt"
606 if [ -f "${rundir}/params.txt" ]
608 echo "copying params file"
609 cp "${rundir}/params.txt" "${outputdir}/params.txt"
612 if [ -d "${rundir}/mac" ]
614 echo "copying mac folder"
615 cp -r "${rundir}/mac" "${outputdir}/mac"
618 echo "these was ${warning_count} warning(s)"