#!/bin/bash
# 0 1 2 3 4 5 6 7 8 9
databases=( AHA ESTTDB LTSTDB MITBIH-AR MITBIH-AR-DS2 MITBIH-SUP MITBIH-ST INCART Biosigna MITBIH-LT)
db_fmt=( AHA MIT MIT MIT MIT MIT MIT MIT HES MIT)
pidsXrec=( 3 10 30 3 3 3 3 3 3 30 )
db_sizes=( 154 90 86 44 22 78 18 75 56 7 )
db_ext=( ecg dat dat dat dat dat dat dat hes dat )
db_paths=( "/extra/database/bio/ecg/thew/aha/" \
"/extra/database/bio/ecg/thew/European ST-T Database/" \
"/extra/database/bio/ecg/thew/Long-Term ST Database/" \
"/extra/database/bio/ecg/thew/mitbih-ar/" \
"/extra/database/bio/ecg/thew/ds2/" \
"/extra/database/bio/ecg/thew/MIT-BIH Supraventricular Arrhythmia Database/" \
"/extra/database/bio/ecg/thew/MIT-BIH ST Change Database/" \
"/extra/database/bio/ecg/thew/St Petersburg Institute of Cardiological Technics 12-lead Arrhythmia Database/" \
"/extra/database/bio/ecg/thew/biosigna/" \
"/extra/database/bio/ecg/thew/The MIT-BIH Long Term Database/" \
)
#Typical configuration
op_modes=( auto slightly-assisted assisted assisted )
NumOfClusters=( 9 9 9 12 )
ClusteringRepetitions=( 1 1 1 1 )
ClusterPresence=( 50 75 50 50 )
#Long-term configuration
#op_modes=( assisted )
#NumOfClusters=( 15 )
#ClusteringRepetitions=( 5 )
#ClusterPresence=( 50 )
#Esto sirve para que no haya dependencias de hacer algunos JOBS antes que otros.
#finish_preproc_first=1
finish_preproc_first=0
#Esto sirve para limitar la cantidad de procesos que corren en CONDOR.
#limited=1
limited=0
#numero de veces que reintenta un registro cuando ocurre un error. Esto esta
#pensado para cuando falla al acceder a un archivo en el filesystem en red.
retries=1
submit_delay=0
processes_running_total=1200
processes_idle_total=700 #esto es para no encolar mucho trabajo en Condor
databases_running=4
iterations=60
if [ "$limited" -eq "0" ]
then
databases_running=${#databases_idx[@]}
fi
aux_str="*."
tmp_path=/extra/scratch/bio/mllamedo/tmp/
condor_output_path=/home/bio/mllamedo/ecg_classification/tmp/condor_output/
#do_cleanup=1 #SI limpia temporales anteriores
do_cleanup=0 #NO limpia temporales anteriores
#echo "Cleaning previous $condor_output_path_this_work/global.dag files ..."
#rm ./global.*
#echo "done."
global_parent_string="PARENT"
aux=$(echo "scale=0; ${#databases[@]}-1" | bc)
#databases_idx=($(seq 0 $aux) )
#databases_idx=($(seq 0 8) )
databases_idx=(3 4)
if [ "$databases_running" -ge "${#databases_idx[@]}" ]
then
databases_running=${#databases_idx[@]}
fi
timestamp=$(date +%Y_%m_%d-%H_%M_%S)
condor_output_path_this_work="$condor_output_path$timestamp"
if [ ! -d $condor_output_path_this_work ]
then
mkdir $condor_output_path_this_work
fi
echo "# Dag generated by MEC" > $condor_output_path_this_work/global.dag
#Fabrico el archivos de configuracion de cada proceso.
echo "Universe = vanilla" > $condor_output_path_this_work/aux.condor
echo "InitialDir = /home/bio/mllamedo/ecg_classification/a2hbc" >> $condor_output_path_this_work/aux.condor
echo "Executable = \$(InitialDir)/scripts/common/condor/condor_exec.sh" >> $condor_output_path_this_work/aux.condor
echo "GetEnv = false" >> $condor_output_path_this_work/aux.condor
echo "Requirements = ((Arch == \"INTEL\") || (Arch == \"X86_64\")) && (OpSys == \"LINUX\") && (Disk >= DiskUsage) && ((Memory * 1024) >= ImageSize) && (TARGET.FileSystemDomain == MY.FileSystemDomain)" >> $condor_output_path_this_work/aux.condor
echo "+ShortJob = TRUE" >> $condor_output_path_this_work/aux.condor
echo "nice_user = true" >> $condor_output_path_this_work/aux.condor
#lo duplico porque tambien usare esta conf para la recoleccion de resultados.
cp $condor_output_path_this_work/aux.condor $condor_output_path_this_work/aux_recs_results_collector.condor
cp $condor_output_path_this_work/aux.condor $condor_output_path_this_work/aux_db_results_collector.condor
echo "Arguments = a2hbc( \\" >> $condor_output_path_this_work/aux.condor
echo " 'recording_name', '\$(recording_name)', \\" >> $condor_output_path_this_work/aux.condor
echo " 'recording_format', '\$(recording_format)', \\" >> $condor_output_path_this_work/aux.condor
echo " 'op_mode', '\$(op_mode)', \\" >> $condor_output_path_this_work/aux.condor
echo " 'SimulateExpert', true, \\" >> $condor_output_path_this_work/aux.condor
echo " 'Repetitions', \$(Repetitions), \\" >> $condor_output_path_this_work/aux.condor
echo " 'tmp_path', '\$(tmp_path)', \\" >> $condor_output_path_this_work/aux.condor
echo " 'NumOfClusters', \$(NumOfClusters), \\" >> $condor_output_path_this_work/aux.condor
echo " 'ClusteringRepetitions', \$(ClusteringRepetitions), \\" >> $condor_output_path_this_work/aux.condor
echo " 'ClusterPresence', \$(ClusterPresence), \\" >> $condor_output_path_this_work/aux.condor
echo " 'cant_pids', \$(cant_pids), \\" >> $condor_output_path_this_work/aux.condor
echo " 'this_pid', \$(Process)+1 \\" >> $condor_output_path_this_work/aux.condor
echo " );" >> $condor_output_path_this_work/aux.condor
echo "Queue \$(cant_pids)" >> $condor_output_path_this_work/aux.condor
echo "Arguments = CollectResutls('\$(tmp_path)', '\$(op_mode)' );" >> $condor_output_path_this_work/aux_recs_results_collector.condor
echo "Queue 1" >> $condor_output_path_this_work/aux_recs_results_collector.condor
echo "Error = ../tmp/condor_output/$timestamp/global.err" > $condor_output_path_this_work/global_db_results_collector.condor
echo "Log = ../tmp/condor_output/$timestamp/global.log" >> $condor_output_path_this_work/global_db_results_collector.condor
echo "Output = ../tmp/condor_output/$timestamp/global.out" >> $condor_output_path_this_work/global_db_results_collector.condor
cat $condor_output_path_this_work/aux_db_results_collector.condor >> $condor_output_path_this_work/global_db_results_collector.condor
echo "Arguments = ResultsForAllDatabases('\$(tmp_path)', '\$(op_mode)' );" >> $condor_output_path_this_work/global_db_results_collector.condor
echo "Queue 1" >> $condor_output_path_this_work/global_db_results_collector.condor
for ((iii=0; iii<${#databases_idx[@]}; iii++))
do
ii=${databases_idx[$iii]}
#echo ${databases[${ii}]} ${db_paths[${ii}]}
# echo "Cleaning previous ${databases[${ii}]} database logs ..."
# rm $condor_output_path_this_work/${databases[${ii}]}.*
# echo "done."
num_recs_found=$(ls "${db_paths[${ii}]}"*.${db_ext[${ii}]}| wc -l )
cant_pids=${pidsXrec[${ii}]}
repetitions=$(echo "scale=0; $iterations/$cant_pids" | bc)
if [ "$num_recs_found" -eq "${db_sizes[${ii}]}" ]
then
if [ "$do_cleanup" -eq "1" ]
then
log_files=( $(find ${tmp_path}${databases[${ii}]} -name "tmpfile*.*" -print) )
if [ "${#log_files[@]}" -gt "0" ]
then
echo "Cleaning previous ${databases[${ii}]} tmp files ..."
echo ${log_files[@]} | xargs rm
echo "done."
fi
fi
echo "# Dag generated by MEC" > "$condor_output_path_this_work/${databases[${ii}]}.dag"
for ((kk=0; kk<${#op_modes[@]}; kk++))
do
vars_str=()
jj=0
op_mode=${op_modes[${kk}]}
condor_output_path_this_db_this_mode="$condor_output_path_this_work/${databases[${ii}]}_$op_mode${kk}"
echo "Error = ../tmp/condor_output/$timestamp/${databases[${ii}]}_$op_mode${kk}_\$(Cluster)_\$(Process).err" > $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}.condor
echo "Log = ../tmp/condor_output/$timestamp/${databases[${ii}]}_$op_mode${kk}.log" >> $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}.condor
echo "Output = /dev/null" >> $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}.condor
cat $condor_output_path_this_work/aux.condor >> $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}.condor
parent_string="PARENT "
while read rec_name
do
xbase=${rec_name##*/}
rec_name_only=${xbase%.*}
job_name="${rec_name_only}_${op_mode}${kk}"
echo "JOB ${job_name} $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}.condor" >> "$condor_output_path_this_work/${databases[${ii}]}.dag"
#echo "RETRY ${job_name} $retries" >> "$condor_output_path_this_work/${databases[${ii}]}.dag"
parent_string="${parent_string} ${job_name} "
#concurrency_limits = \"'mllamedo:$PIDs_X_DB_runing'\" \
vars_str[$jj]="VARS ${job_name} \
recording_name=\"${rec_name}\" \
recording_format=\"${db_fmt[${ii}]}\" \
op_mode=\"${op_mode}\" \
tmp_path=\"${tmp_path}${databases[${ii}]}/\" \
NumOfClusters=\"${NumOfClusters[${kk}]}\" \
ClusteringRepetitions=\"${ClusteringRepetitions[${kk}]}\" \
ClusterPresence=\"${ClusterPresence[${kk}]}\" \
Repetitions=\"${repetitions}\" \
cant_pids=\"${cant_pids}\" "
let "jj += 1"
#break
#esto es para los paths con espacios, que los pase como un solo string
done < <(find "${db_paths[${ii}]}"*.${db_ext[${ii}]} )
echo "Error = ../tmp/condor_output/$timestamp/${databases[${ii}]}_$op_mode${kk}.err" > $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}_recs_result_collector.condor
echo "Log = ../tmp/condor_output/$timestamp/${databases[${ii}]}_$op_mode${kk}.log" >> $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}_recs_result_collector.condor
echo "Output = /dev/null" >> $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}_recs_result_collector.condor
cat $condor_output_path_this_work/aux_recs_results_collector.condor >> $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}_recs_result_collector.condor
echo "JOB recs_results_collector_${op_mode}${kk} $condor_output_path_this_work/${databases[${ii}]}_${op_mode}${kk}_recs_result_collector.condor" >> "$condor_output_path_this_work/${databases[${ii}]}.dag"
parent_string="${parent_string} CHILD recs_results_collector_${op_mode}${kk}"
echo $parent_string >> "$condor_output_path_this_work/${databases[${ii}]}.dag"
for ((jj=0; jj<${#vars_str[@]}; jj++))
do
echo ${vars_str[$jj]} >> "$condor_output_path_this_work/${databases[${ii}]}.dag"
done
echo "VARS recs_results_collector_${op_mode}${kk} op_mode=\"${op_mode}\" tmp_path=\"${tmp_path}${databases[${ii}]}\" " >> "$condor_output_path_this_work/${databases[${ii}]}.dag"
done
if [ "$finish_preproc_first" -eq "1" ]
then
if [ "${#op_modes[@]}" -ge "2" ]
then
while read rec_name
do
xbase=${rec_name##*/}
rec_name_only=${xbase%.*}
intra_DB_parent_string="PARENT ${rec_name_only}_${op_modes[0]}0 CHILD"
for ((kk=1; kk<${#op_modes[@]}; kk++))
do
intra_DB_parent_string="${intra_DB_parent_string} ${rec_name_only}_${op_modes[$kk]}${kk} "
done
echo $intra_DB_parent_string >> "$condor_output_path_this_work/${databases[${ii}]}.dag"
#break
#esto es para los paths con espacios, que los pase como un solo string
done < <(find "${db_paths[${ii}]}"*.${db_ext[${ii}]} )
fi
fi
echo "CONFIG $condor_output_path_this_work/${databases[${ii}]}.config.dag" >> "$condor_output_path_this_work/${databases[${ii}]}.dag"
recordings_running=$(echo "scale=0; $processes_running_total/$databases_running/$cant_pids" | bc)
if [ "$limited" -eq "1" ]
then
echo "DAGMAN_MAX_JOBS_SUBMITTED=$recordings_running" > $condor_output_path_this_work/${databases[${ii}]}.config.dag
echo "${databases[${ii}]} correra $recordings_running registro/s a la vez."
echo "DAGMAN_SUBMIT_DELAY=$submit_delay" >> $condor_output_path_this_work/${databases[${ii}]}.config.dag
else
recordings_idle=$(echo "scale=0; $processes_idle_total/$databases_running/$cant_pids" | bc)
echo "DAGMAN_MAX_JOBS_IDLE=$recordings_idle" > $condor_output_path_this_work/${databases[${ii}]}.config.dag
echo "${databases[${ii}]} limitara a $recordings_idle registros idle."
fi
echo "DAGMAN_PENDING_REPORT_INTERVAL=300" >> $condor_output_path_this_work/${databases[${ii}]}.config.dag
#cat $condor_output_path_this_work/${databases[${ii}]}.dag
echo "Generating dag files ${databases[${ii}]} work ..."
condor_submit_dag -f -no_submit $condor_output_path_this_work/${databases[${ii}]}.dag
echo "JOB ${databases[${ii}]} $condor_output_path_this_work/${databases[${ii}]}.dag.condor.sub" >> $condor_output_path_this_work/global.dag
global_parent_string="${global_parent_string} ${databases[${ii}]} "
else
echo "Expected ${db_sizes[${ii}]} recs, found ${num_recs_found}. Check path ${db_paths[${ii}]}"
#echo ${db_sizes[${ii}]} $recs_found ${db_paths[${ii}]}
exit
fi
done
global_parent_string="${global_parent_string} CHILD "
for ((kk=0; kk<${#op_modes[@]}; kk++))
do
echo "JOB databases_collector_${op_modes[${kk}]}${kk} $condor_output_path_this_work/global_db_results_collector.condor" >> $condor_output_path_this_work/global.dag
global_parent_string="${global_parent_string} databases_collector_${op_modes[${kk}]}${kk}"
done
echo $global_parent_string >> $condor_output_path_this_work/global.dag
for ((kk=0; kk<${#op_modes[@]}; kk++))
do
echo "VARS databases_collector_${op_modes[${kk}]}${kk} op_mode=\"${op_modes[${kk}]}\" tmp_path=\"${tmp_path}\" " >> $condor_output_path_this_work/global.dag
done
#return
if [ "$limited" -eq "1" ]
then
condor_submit_dag -f -maxjobs $databases_running $condor_output_path_this_work/global.dag
else
condor_submit_dag -f $condor_output_path_this_work/global.dag
fi
echo "Logs en $condor_output_path_this_work"