]> Creatis software - clitk.git/blob - cluster_tools/gate_power_merge.sh
adapt power merge for osx
[clitk.git] / cluster_tools / gate_power_merge.sh
1 #!/usr/bin/env bash
2
3 set -u
4
5 function error {
6     echo "ERROR: $1"
7     exit 1
8 }
9
10 warning_count=0
11 function warning {
12     let "warning_count++"
13     echo "MERGE_WARNING: $1"
14 }
15
16 function start_bar {
17     count_max="${1:?"provide count max"}"
18 }
19
20 function update_bar {
21     local count="${1:?"provide count"}"
22     local message="${2:?"provide message"}"
23     local percent=$(echo "100*${count}/${count_max}" | bc)
24     #printf "[%03d/%03d] %3d%% %-80.80s\r" ${count} ${count_max} ${percent} "${message}"
25     printf "[%03d/%03d] %3d%% %-80.80s\n" ${count} ${count_max} ${percent} "${message}"
26 }
27
28 function end_bar {
29     unset count_max
30     #echo -ne '\n'
31 }
32
33 function check_interfile {
34     local input_interfile="${1:?"provide input interfile"}"
35
36     grep -qs '!INTERFILE :=' "${input_interfile}" || return 1
37
38     local header_byte_size=$(awk -F' ' '
39 BEGIN { zsize = 0; }
40 /matrix size/ && $3 == "[1]" { xsize = $5; }
41 /matrix size/ && $3 == "[2]" { ysize = $5; }
42 /number of projections/ { zsize += $5; }
43 /number of bytes per pixel/ { byte_per_pixel = $7; }
44 END { print xsize * ysize * zsize * byte_per_pixel; }' "${input_interfile}")
45
46     local raw_interfile="$(dirname "${input_interfile}")/$(awk -F' := ' '/name of data file/ { print $2; }' "${input_interfile}")"
47
48     test -f "${raw_interfile}" || return 1
49     test $(stat -c%s "${raw_interfile}") -eq ${header_byte_size} || return 1
50 }
51
52 function write_mhd_header {
53     local input_interfile="${1:?"provide input interfile"}"
54     local output_mhd="$(dirname "${input_interfile}")/$(basename "${input_interfile}" ".hdr").mhd"
55
56     check_interfile "${input_interfile}" || error "${input_interfile} isn't an interfile image"
57
58     local header_start='ObjectType = Image
59 NDims = 3
60 AcquisitionDate = none
61 BinaryData = True
62 BinaryDataByteOrderMSB = False
63 CompressedData = False
64 TransformMatrix = 1 0 0 0 1 0 0 0 1
65 Offset = 0 0 0
66 CenterOfRotation = 0 0 0
67 DistanceUnits = mm
68 AnatomicalOrientation = RIP'
69
70     echo "${header_start}" > "${output_mhd}"
71
72     awk -F' ' '
73 /scaling factor/ && $4 == "[1]" { xspacing = $6; }
74 /scaling factor/ && $4 == "[2]" { yspacing = $6; }
75 END { print "ElementSpacing = " xspacing " " yspacing " 1"; }' "${input_interfile}" >> "${output_mhd}"
76
77     awk -F' ' '
78 BEGIN { zsize = 0; }
79 /matrix size/ && $3 == "[1]" { xsize = $5; }
80 /matrix size/ && $3 == "[2]" { ysize = $5; }
81 /number of projections/ { zsize += $5; }
82 END { print "DimSize = " xsize " " ysize " " zsize; }' "${input_interfile}" >> "${output_mhd}"
83
84     awk -F' := ' '
85 /number format/ { format = $2; }
86 /number of bytes per pixel/ { byte_per_pixel = $2 }
87 END {
88 if (format == "unsigned integer" && byte_per_pixel == 8) { print "ElementType = MET_ULONG"; exit };
89 if (format == "unsigned integer" && byte_per_pixel == 4) { print "ElementType = MET_UINT"; exit };
90 if (format == "unsigned integer" && byte_per_pixel == 2) { print "ElementType = MET_USHORT"; exit };
91 if (format == "unsigned integer" && byte_per_pixel == 1) { print "ElementType = MET_UCHAR"; exit };
92 if (format == "integer" && byte_per_pixel == 8) { print "ElementType = MET_LONG"; exit };
93 if (format == "integer" && byte_per_pixel == 4) { print "ElementType = MET_INT"; exit };
94 if (format == "integer" && byte_per_pixel == 2) { print "ElementType = MET_SHORT"; exit };
95 if (format == "integer" && byte_per_pixel == 1) { print "ElementType = MET_CHAR"; exit };
96 if (format == "float" && byte_per_pixel == 8) { print "ElementType = MET_FLOAT"; exit };
97 if (format == "float" && byte_per_pixel == 4) { print "ElementType = MET_DOUBLE"; exit };
98 print "ElementType = MET_INT";
99 }' "${input_interfile}" >> "${output_mhd}"
100
101     awk -F' := ' '
102 /name of data file/ { print "ElementDataFile = " $2; }' "${input_interfile}" >> "${output_mhd}"
103 }
104
105 rootMerger="clitkMergeRootFiles"
106 test -x "./clitkMergeRootFiles" && rootMerger="./clitkMergeRootFiles"
107
108 function merge_root {
109     local merged="$1"
110     shift
111     echo "  ${indent}entering root merger"
112     echo "  ${indent}merger is ${rootMerger}"
113     echo "  ${indent}creating ${merged}"
114     #echo "######## $#"
115     #echo "######## $*"
116
117     if test $# -eq 1
118     then
119         echo "  ${indent}just one partial file => just copy it"
120         cp "$1" "${merged}"
121         return
122     fi
123
124     local count=0
125     local arguments=" -o ${merged}"
126     while test $# -gt 0
127     do
128         local partial="$1"
129         shift
130         let count++
131         local arguments=" -i ${partial} ${arguments}"
132     done
133     ${rootMerger} ${arguments} > /dev/null || error "error while calling ${rootMerger}"
134     echo "  ${indent}merged ${count} files"
135 }
136
137 statMerger="mergeStatFile.py"
138 test -x "./mergeStatFile.sh" && statMerger="./mergeStatFile.sh"
139
140 function merge_stat {
141     local merged="$1"
142     shift
143     echo "  ${indent}entering stat merger"
144     echo "  ${indent}merger is ${statMerger}"
145     echo "  ${indent}creating ${merged}"
146     local count=0
147     start_bar $#
148     while test $# -gt 0
149     do
150         local partial="$1"
151         shift
152         let count++
153
154         if test ! -f "${merged}"
155         then
156             update_bar ${count} "copying first partial result ${partial}"
157             cp "${partial}" "${merged}"
158             continue
159         fi
160
161         update_bar ${count} "adding ${partial}"
162         ${statMerger} -i "${merged}" -j "${partial}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${statMerger}"
163     done
164     end_bar
165     echo "  ${indent}merged ${count} files"
166 }
167
168 doseMerger="mergeDosePerEnegryFile.sh"
169 test -x "./mergeDosePerEnergyFile.sh" && doseMerger="./mergeDosePerEnergyFile.sh"
170
171 function merge_dose {
172     local merged="$1"
173     shift
174     echo "  ${indent}entering dose merger"
175     echo "  ${indent}merger is ${doseMerger}"
176     echo "  ${indent}creating ${merged}"
177     local count=0
178     start_bar $#
179     while test $# -gt 0
180     do
181         local partial="$1"
182         shift
183         let count++
184
185         if test ! -f "${merged}"
186         then
187             update_bar ${count} "copying first partial result ${partial}"
188             cp "${partial}" "${merged}"
189             continue
190         fi
191
192         update_bar ${count} "adding ${partial}"
193         ${doseMerger} -i "${merged}" -j "${partial}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${doseMerger}"
194     done
195     end_bar
196     echo "  ${indent}merged ${count} files"
197 }
198
199 txtImageMerger="clitkMergeAsciiDoseActor"
200 test -f "./clitkMergeAsciiDoseActor" && txtImageMerger="./clitkMergeAsciiDoseActor"
201
202 function merge_txt_image {
203     local merged="$1"
204     shift
205     echo "  ${indent}entering text image merger"
206     echo "  ${indent}merger is ${txtImageMerger}"
207     echo "  ${indent}creating ${merged}"
208     local count=0
209     start_bar $#
210     while test $# -gt 0
211     do
212         local partial="$1"
213         shift
214         let count++
215
216         if test ! -f "${merged}"
217         then
218             update_bar ${count} "copying first partial result ${partial}"
219             cp "${partial}" "${merged}"
220             continue
221         fi
222
223         update_bar ${count} "adding ${partial}"
224         local header="$(cat "${merged}" | head -n 6)"
225         local tmp="$(mktemp)"
226         ${txtImageMerger} -i "${partial}" -j "${merged}" -o "${tmp}" 2> /dev/null > /dev/null || warning "error while calling ${txtImageMerger}"
227         echo "${header}" > "${merged}"
228         grep -v '## Merge' "${tmp}" >> "${merged}"
229         rm "${tmp}"
230     done
231     end_bar
232     echo "  ${indent}merged ${count} files"
233 }
234
235 hdrImageMerger="clitkImageArithm"
236 test -x "./clitkImageArithm" && hdrImageMerger="./clitkImageArithm"
237
238 function merge_hdr_image {
239     local merged="$1"
240     local merged_bin="${merged%.*}.img"
241     shift
242     echo "  ${indent}entering hdr image merger"
243     echo "  ${indent}merger is ${hdrImageMerger}"
244     echo "  ${indent}creating ${merged}"
245     local count=0
246     start_bar $#
247     while test $# -gt 0
248     do
249         local partial="$1"
250         local partial_bin="${partial%.*}.img"
251         shift
252         let count++
253
254         if test ! -f "${merged}"
255         then
256             update_bar ${count} "copying first partial result ${partial}"
257             cp "${partial}" "${merged}"
258             cp "${partial_bin}" "${merged_bin}"
259             continue
260         fi
261
262         update_bar ${count} "adding ${partial}"
263         ${hdrImageMerger} -t 0 -i "${partial}" -j "${merged}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${hdrImageMerger}"
264     done
265     end_bar
266     echo "  ${indent}merged ${count} files"
267 }
268
269 mhdImageMerger="clitkImageArithm"
270 test -x "./clitkImageArithm" && mhdImageMerger="./clitkImageArithm"
271
272 function merge_mhd_image {
273     local merged="$1"
274     local merged_bin="${merged%.*}.raw"
275     local last_character=${merged#${merged%?}}
276     shift
277     echo "  ${indent}entering mhd image merger"
278     echo "  ${indent}merger is ${mhdImageMerger}"
279     echo "  ${indent}creating ${merged}"
280     local count=0
281     start_bar $#
282     while test $# -gt 0
283     do
284         local partial="$1"
285         local partial_bin="$(dirname "${partial}")/$(awk -F' = ' '/ElementDataFile/ { print $2; }' "${partial}")"
286         shift
287         let count++
288
289         if test ! -f "${merged}"
290         then
291             update_bar ${count} "copying first partial result ${partial}"
292             cp "${partial}" "${merged}"
293             if test "$last_character" = "d"
294             then
295                 cp "${partial_bin}" "${merged_bin%.*}.${partial_bin##*.}"
296             fi
297             continue
298         fi
299
300         update_bar ${count} "adding ${partial}"
301         ${mhdImageMerger} -t 0 -i "${partial}" -j "${merged}" -o "${merged}" 2> /dev/null > /dev/null || warning "error while calling ${mhdImageMerger}"
302         if test "$last_character" = "d" && test "${merged_bin}" != "${merged_bin%.*}.${partial_bin##*.}"
303         then
304             mv "${merged_bin}" "${merged_bin%.*}.${partial_bin##*.}"
305             sed -i "s/$(basename "${merged_bin}")/$(basename "${merged_bin%.*}.${partial_bin##*.}")/" "${merged}"
306         fi
307     done
308     end_bar
309     echo "  ${indent}merged ${count} files"
310 }
311
312 function merge_dispatcher {
313     local indent="  ** "
314     local outputfile="${1:?"provide output filename"}"
315     echo "merging ${outputfile}"
316
317     local partialoutputfiles="$(find -L "${rundir}" -mindepth 2 -type f -name "${outputfile}")"
318     local nboutputfiles="$(echo "${partialoutputfiles}" | wc -l)"
319     if test ${nboutputdirs} -ne ${nboutputfiles}
320     then
321         warning "missing files"
322         if ! test "${2}" == "--force"
323         then
324             return
325         fi
326     fi
327
328     local firstpartialoutputfile="$(echo "${partialoutputfiles}" | head -n 1)"
329     local firstpartialoutputextension="${firstpartialoutputfile##*.}"
330     echo "${indent}testing file type on ${firstpartialoutputfile}"
331
332     if test "${firstpartialoutputextension}" == "hdr" && test grep -qs 'INTERFILE' "${firstpartialoutputfile}"
333     then
334         echo "${indent}this is an interfile image"
335         echo "${indent}creating mhd headers"
336         for partialoutputfile in $partialoutputfiles; do write_mhd_header "${partialoutputfile}"; done
337         local mhd_partialoutputfiles="$(for partialoutputfile in $partialoutputfiles; do echo "${partialoutputfile%.*}.mhd"; done)"
338         local mhd_mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}" ".hdr").mhd"
339         merge_mhd_image "${mhd_mergedfile}" ${mhd_partialoutputfiles} || error "error while merging"
340         echo "${indent}cleaning mhd headers"
341         for mhd_partialoutputfile in $mhd_partialoutputfiles; do rm "${mhd_partialoutputfile}"; done
342         rm "${mhd_mergedfile}"
343         echo "${indent}copy interfile header"
344         cp "${firstpartialoutputfile}" "${outputdir}"
345         return
346     fi
347
348     if test "${firstpartialoutputextension}" == "hdr"
349     then
350         echo "${indent}this is an analyse image"
351         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
352         merge_hdr_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
353         return
354     fi
355
356     if test "${firstpartialoutputextension}" == "mhd" || test "${firstpartialoutputextension}" == "mha"
357     then
358         echo "${indent}this is a mhd image"
359         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
360         merge_mhd_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
361         return
362     fi
363
364     if test "${firstpartialoutputextension}" == "root"
365     then
366         echo "${indent}this is a root file"
367         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
368         merge_root "${mergedfile}" ${partialoutputfiles} || error "error while merging"
369         return
370     fi
371
372     if test "${firstpartialoutputextension}" == "txt" && grep -qs 'NumberOfEvent' "${firstpartialoutputfile}"
373     then
374         echo "${indent}this is a stat file"
375         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
376         merge_stat "${mergedfile}" ${partialoutputfiles} || error "error while merging"
377         return
378     fi
379
380     if test "${firstpartialoutputextension}" == "txt" && grep -qs 'energydose' "${firstpartialoutputfile}"
381     then
382         echo "${indent}this is a dose file"
383         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
384         merge_dose "${mergedfile}" ${partialoutputfiles} || error "error while merging"
385         return
386     fi
387
388
389
390     if test "${firstpartialoutputextension}" == "txt" && grep -qs 'Resol' "${firstpartialoutputfile}"
391     then
392         local resol="$(sed -nr '/Resol/s/^.*=\s+\((.+)\)\s*$/\1/p' "${firstpartialoutputfile}")"
393         local resolx="$(echo "${resol}" | cut -d',' -f1)"
394         local resoly="$(echo "${resol}" | cut -d',' -f2)"
395         local resolz="$(echo "${resol}" | cut -d',' -f3)"
396         if test "${resol}" == "1,1,1"
397         then
398             echo "${indent}this is a txt integral value"
399             local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
400             merge_txt_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
401             return
402         fi
403         if test \( "${resolx}" == "1" -a "${resoly}" == "1" \) -o \( "${resoly}" == "1" -a "${resolz}" == "1" \) -o \( "${resolz}" == "1" -a "${resolx}" == "1" \)
404         then
405             echo "${indent}this is a txt profile"
406             local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
407             merge_txt_image "${mergedfile}" ${partialoutputfiles} || error "error while merging"
408             return
409         fi
410     fi
411
412     if test "${firstpartialoutputextension}" == "txt"
413     then
414         echo "${indent}this is a non specific txt output"
415         local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
416         local nbdifferent="$(md5sum ${partialoutputfiles} | awk '{ print $1; }' | sort | uniq | wc -l)"
417         echo "  ${indent}${nbdifferent} different files"
418         if test ${nbdifferent} -gt 1
419         then
420             echo "  ${indent}catting to ${mergedfile}"
421             cat ${partialoutputfiles} > "${mergedfile}" || error "error while merging"
422             return
423         else
424             echo "  ${indent}moving to ${mergedfile}"
425             cp "${firstpartialoutputfile}" "${mergedfile}" || error "error while merging"
426             return
427         fi
428     fi
429
430     error "unknown file type"
431 }
432
433 function merge_dispatcher_uncertainty {
434     local indent="  ** "
435     local outputfile="${1:?"provide output filename"}"
436
437     local partialoutputfiles="$(find -L "${rundir}" -mindepth 2 -type f -name "${outputfile}")"
438     local nboutputfiles="$(echo "${partialoutputfiles}" | wc -l)"
439     if test ${nboutputdirs} -ne ${nboutputfiles}
440     then
441         warning "missing files"
442         if ! test "${2}" == "--force"
443         then
444             return
445         fi
446     fi
447
448     local firstpartialoutputfile="$(echo "${partialoutputfiles}" | head -n 1)"
449     local firstpartialoutputextension="${firstpartialoutputfile##*.}"
450
451     if [[ "${firstpartialoutputfile}" == *Uncertainty* ]]
452     then
453           if test "${firstpartialoutputextension}" == "mhd" || test "${firstpartialoutputextension}" == "mha"
454         then
455             echo "${indent}Uncertainty file found: ${firstpartialoutputfile}"
456             ## search for sum
457             local mergedfile="${outputdir}/$(basename "${firstpartialoutputfile}")"
458             summed_merged_file=${mergedfile//-Uncertainty/}
459             if [ ! -f ${summed_merged_file} ];
460             then
461                 warning "${summed_merged_file} does not exist. Error, no uncertainty computed"
462                 return;
463             fi
464             echo "${indent}${summed_merged_file} found"
465             ## search for Squared
466             squared_merged_file=${mergedfile//-Uncertainty/-Squared}
467             if [ ! -f ${squared_merged_file} ];
468             then
469                 warning "${squared_merged_file} does not exist. Error, no uncertainty computed"
470                 return;
471             fi
472             echo "${indent}${squared_merged_file} found"
473             ## search for NumberOfEvent
474             totalEvents=0;
475             unamestr=`uname`
476             if [[ "$unamestr" == 'Darwin' ]]; then
477                 files=$(find -L "${rundir}" -type f -regex ".*output.*[hdr|mhd|mha|root|txt]" | awk -F '/' '{ print $NF; }' | sort | uniq)
478             else
479                 files=$(find -L "${rundir}" -regextype 'posix-extended' -type f -regex "${rundir}/output.*\.(hdr|mhd|mha|root|txt)" | awk -F '/' '{ print $NF; }' | sort | uniq)
480             fi
481             echo $files
482             for outputfile in ${files}
483             do
484                 #echo $outputfile
485                 if grep -q 'NumberOfEvent' "${outputdir}/${outputfile}"
486                 then
487                     totalEvents="$(grep "NumberOfEvents" "${outputdir}/${outputfile}" | cut -d' ' -f4)"
488                     echo "${indent}Find the NumberOfEvent in $outputfile: ${totalEvents}"
489                 fi
490             done
491
492             if test ${totalEvents} -gt 0
493             then
494                 uncerImageMerger="clitkImageUncertainty"
495                 test -x "./clitkImageUncertainty" && uncerImageMerger="./clitkImageUncertainty"
496                 ${uncerImageMerger} -i ${summed_merged_file} -s ${squared_merged_file} -o ${mergedfile} -n ${totalEvents}
497             else
498                 warning "${totalEvents} not positive. A at least one stat file (SimulationStatisticActor) must be provided. Error, no uncertainty computed"
499                 return;
500             fi
501               else
502             error "merge_dispatcher_uncertainty does not handle ${firstpartialoutputfile} files"
503         fi
504     fi
505
506 }
507
508 echo "!!!! this is $0 v0.3k !!!!"
509 echo "Usage: gate_power_merge.sh run.dir [--force]"
510 echo "   where --force allows to merge files even if the file is missing in some output directories"
511
512 rundir="${1?"provide run dir"}"
513 force=${2:-""}
514 rundir="$(echo "${rundir}" | sed 's|/*$||')"
515 nboutputdirs="$(find "${rundir}" -mindepth 1 -type d -name 'output*' -o -type l -name 'output*' | wc -l)"
516
517 test ${nboutputdirs} -gt 0 || error "no output dir found"
518 echo "found ${nboutputdirs} partial output dirs"
519
520 outputdir="results"
521 if [ "${rundir}" != "." -a "${rundir##*.}" != "${rundir}" ]
522 then
523     outputdir="results.${rundir##*.}"
524 fi
525 outputdir="$(basename "${outputdir}")"
526 echo "output dir is ${outputdir}"
527
528 test -d "${outputdir}" && rm -r "${outputdir}"
529 mkdir "${outputdir}"
530
531 unamestr=`uname`
532 if [[ "$unamestr" == 'Darwin' ]]; then
533     files=$(find -L "${rundir}" -type f -regex ".*output.*[hdr|mhd|mha|root|txt]" | awk -F '/' '{ print $NF; }' | sort | uniq)
534 else
535     files=$(find -L "${rundir}" -regextype 'posix-extended' -type f -regex "${rundir}/output.*\.(hdr|mhd|mha|root|txt)" | awk -F '/' '{ print $NF; }' | sort | uniq)
536 fi
537 echo files $files
538 for outputfile in ${files}
539 do
540     merge_dispatcher "${outputfile}" "${force}"
541 done
542
543 echo ""
544 echo "Merging done. Special case for statistical uncertainty"
545
546 unamestr=`uname`
547 if [[ "$unamestr" == 'Darwin' ]]; then
548     files=$(find -L "${outputdir}" -type f -regex ".*[hdr|mhd|mha|root|txt]" | awk -F '/' '{ print $NF; }' | sort | uniq)
549 else
550     files=$(find -L "${outputdir}" -regextype 'posix-extended' -type f -regex "${outputdir}/.*\.(hdr|mhd|mha|root|txt)" | awk -F '/' '{ print $NF; }' | sort | uniq)
551 fi
552 echo files = $files
553 for outputfile in ${files}
554 do
555     merge_dispatcher_uncertainty "${outputfile}" "${force}"
556 done
557
558 if [ -f "${rundir}/params.txt" ]
559 then
560     echo "copying params file"
561     cp "${rundir}/params.txt" "${outputdir}/params.txt"
562 fi
563
564 if [ -d "${rundir}/mac" ]
565 then
566     echo "copying mac folder"
567     cp -r "${rundir}/mac" "${outputdir}/mac"
568 fi
569
570 echo "these was ${warning_count} warning(s)"