allow all track sources in global reader request + clean QC-only step in async-reco

Author: shahor02

DATA/common/setenv.sh

@@ -49,8 +49,6 @@ if [[ -z "${WORKFLOW_DETECTORS_RECO+x}" ]] || [[ "0$WORKFLOW_DETECTORS_RECO" ==
 if [[ -z "${WORKFLOW_DETECTORS_CTF+x}" ]] || [[ "0$WORKFLOW_DETECTORS_CTF" == "0ALL" ]]; then export WORKFLOW_DETECTORS_CTF=$WORKFLOW_DETECTORS; fi
 if [[ "0${WORKFLOW_DETECTORS_FLP_PROCESSING:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_FLP_PROCESSING=$WORKFLOW_DETECTORS; fi
 if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER=$WORKFLOW_DETECTORS; else export WORKFLOW_DETECTORS_USE_GLOBAL_READER=${WORKFLOW_DETECTORS_USE_GLOBAL_READER:-}; fi
-if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$WORKFLOW_DETECTORS; else export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=${WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS:-}; fi
-if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$WORKFLOW_DETECTORS; else export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=${WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS:-}; fi
 if [[ -z "${WORKFLOW_PARAMETERS:-}" ]]; then export WORKFLOW_PARAMETERS=; fi
 
 if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_QC:-} ]]; then
@@ -63,16 +61,6 @@ if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_CALIB:-} ]]; then
     export WORKFLOW_DETECTORS_CALIB=$(echo $WORKFLOW_DETECTORS_CALIB | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//")
   done
 fi
-if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS:-} ]]; then
-  for i in ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS//,/ }; do
-    export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$(echo $WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//")
-  done
-fi
-if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_CLUSTERS:-} ]]; then
-  for i in ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_CLUSTERS//,/ }; do
-    export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$(echo $WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//")
-  done
-fi
 
 if [[ -z "${TFLOOP:-}" ]];         then export TFLOOP=0; fi                    # loop over timeframes
 if [[ -z "${NTIMEFRAMES:-}" ]];    then export NTIMEFRAMES=-1; fi              # max number of time frames to process, <=0 : unlimited
@@ -222,13 +210,32 @@ for det in `echo $LIST_OF_DETECTORS | sed "s/,/ /g"`; do
   fi
 done
 
+if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$TRACK_SOURCES;
+elif [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS:-}" == "0ALLSINGLE" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$WORKFLOW_DETECTORS;
+else export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=${WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS:-}; fi
+if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$TRACK_SOURCES;
+elif [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS:-}" == "0ALLSINGLE" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$WORKFLOW_DETECTORS;
+else export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=${WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS:-}; fi
+if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS:-} ]]; then
+  for i in ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS//,/ }; do
+    export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$(echo $WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//")
+  done
+fi
+if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_CLUSTERS:-} ]]; then
+  for i in ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_CLUSTERS//,/ }; do
+    export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$(echo $WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//")
+  done
+fi
+
 : ${VERTEXING_SOURCES:="$TRACK_SOURCES"}
 : ${VERTEX_TRACK_MATCHING_SOURCES:="$TRACK_SOURCES"}
 [[ ! -z $VERTEXING_SOURCES ]] && PVERTEX_CONFIG+=" --vertexing-sources $VERTEXING_SOURCES"
 [[ ! -z $VERTEX_TRACK_MATCHING_SOURCES ]] && PVERTEX_CONFIG+=" --vertex-track-matching-sources $VERTEX_TRACK_MATCHING_SOURCES"
 
 if [[ -z ${SVERTEXING_SOURCES:-} ]]; then
   SVERTEXING_SOURCES="$VERTEXING_SOURCES"
+elif [[ "${SVERTEXING_SOURCES^^}" == "NONE" ]]; then
+  SVERTEXING_SOURCES=
 fi
 
 # this option requires well calibrated timing beween different detectors, at the moment suppress it

DATA/production/configurations/asyncReco/async_pass.sh

@@ -586,6 +586,8 @@ else
     echo "WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS"
     echo "Step 3) matching, calib, AOD, potentially QC"
     echo -e "\nStep 3) matching, calib, AOD, potentially QC" >> workflowconfig.log
+    export ALIEN_JDL_PROCESSITSDEADMAP=  # disable ITS dead map calibration, if it was requested then it was already done at step 2
+    export ALIEN_JDL_PROCESSMFTDEADMAP=  # disable MFT dead map calibration, if it was requested then it was already done at step 2
     # This uses the same time frame rate limiting as in full wf, unless differently specified in the JDL
     export TIMEFRAME_RATE_LIMIT=${ALIEN_JDL_TIMEFRAMERATELIMITSSPLITWF:-${TIMEFRAME_RATE_LIMIT}}
     echo "Removing detectors $DETECTORS_EXCLUDE"
@@ -596,11 +598,13 @@ else
       export ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader+=" --reader-delay $READER_DELAY "
     fi
     echo "extra args are $ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader"
-    env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log
+    READ_TRACKS="ALLSINGLE"
+    READ_CLUSTERS="ALLSINGLE"
+    env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$READ_TRACKS WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$READ_CLUSTERS WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log
     # run it
     if [[ "0$RUN_WORKFLOW" != "00" ]]; then
       timeStart=`date +%s`
-      time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list
+      time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$READ_TRACKS WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$READ_CLUSTERS WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list
       exitcode=$?
       timeEnd=`date +%s`
       timeUsed=$(( $timeUsed+$timeEnd-$timeStart ))
@@ -627,11 +631,16 @@ else
       echo -e "\nStep 4) QC" >> workflowconfig.log
       echo "Removing detectors $DETECTORS_EXCLUDE"
       echo "The rate limiting will be the same as in step 3: TIMEFRAME_RATE_LIMIT = ${TIMEFRAME_RATE_LIMIT}"
-      env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log
+      READ_TRACKS="ALL"
+      READ_CLUSTERS="ALL"
+      export GLOBAL_READER_NEEDS_PV="1"
+      export GLOBAL_READER_NEEDS_SV="0"
+      export SVERTEXING_SOURCES="none"
+      env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$READ_TRACKS WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$READ_CLUSTERS WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS= WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log
       # run it
       if [[ "0$RUN_WORKFLOW" != "00" ]]; then
         timeStart=`date +%s`
-        time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list
+        time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$READ_TRACKS WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$READ_CLUSTERS WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS= WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list
         exitcode=$?
         timeEnd=`date +%s`
         timeUsed=$(( $timeUsed+$timeEnd-$timeStart ))