Add env options GEN_TOPO_EPN_CCDB_SERVER and GEN_TOPO_EPN_CCDB_SERVER + cleanup

Author: davidrohr

DATA/tools/epn/gen_topo.sh

@@ -1,45 +1,41 @@
 #!/bin/bash
-# Settings coming from AliECS via env variables
-if [ -z $GEN_TOPO_PARTITION ]; then echo \$GEN_TOPO_PARTITION missing; exit 1; fi # Partition
-if [ -z $GEN_TOPO_HASH ]; then echo \$GEN_TOPO_HASH missing; exit 1; fi # Flag whether source is a hash or a folder
-if [ -z $GEN_TOPO_SOURCE ]; then echo \$GEN_TOPO_SOURCE missing; exit 1; fi # O2DPG repository source, either a commit hash or a path
-if [ -z $GEN_TOPO_LIBRARY_FILE ]; then echo \$GEN_TOPO_LIBRARY_FILE missing; exit 1; fi # Topology description library file in the DATA path of the O2DPG repository
-if [ -z $GEN_TOPO_WORKFLOW_NAME ]; then echo \$GEN_TOPO_WORKFLOW_NAME missing; exit 1; fi # Workflow name in library file
-if [ -z ${WORKFLOW_DETECTORS+x} ]; then echo \$WORKFLOW_DETECTORS missing; exit 1; fi # Comma-separated list of detectors to run processing for
-if [ -z ${WORKFLOW_DETECTORS_QC+x} ]; then echo \$WORKFLOW_DETECTORS_QC missing; exit 1; fi # Comma-separated list of detectors to run QC for
-if [ -z ${WORKFLOW_DETECTORS_CALIB+x} ]; then echo \$WORKFLOW_DETECTORS_CALIB missing; exit 1; fi # Comma-separated list of detectors to run calibration for
-if [ -z ${WORKFLOW_PARAMETERS+x} ]; then echo \$WORKFLOW_PARAMETERS missing; exit 1; fi # Additional parameters for workflow
-if [ -z ${RECO_NUM_NODES_OVERRIDE+x} ]; then echo \$RECO_NUM_NODES_OVERRIDE missing; exit 1; fi # Override number of nodes
-if [ -z $DDMODE ] && [ -z $DDWORKFLOW ]; then echo Either \$DDMODE or \$DDWORKFLOW must be set; exit 1; fi # Select data distribution workflow
-if [ -z "$MULTIPLICITY_FACTOR_RAWDECODERS" ]; then echo \$MULTIPLICITY_FACTOR_RAWDECODERS missing; exit 1; fi # Process multiplicity scaling parameter
-if [ -z "$MULTIPLICITY_FACTOR_CTFENCODERS" ]; then echo \$MULTIPLICITY_FACTOR_CTFENCODERS missing; exit 1; fi # Process multiplicity scaling parameter
-if [ -z "$MULTIPLICITY_FACTOR_REST" ]; then echo \$MULTIPLICITY_FACTOR_REST missing; exit 1; fi # Process multiplicity scaling parameter
+
+# This is a wrapper script that sets EPN-related env variables needed for the PDP DPL Topology generation.
+# Author: David Rohr
+
+# A reference version of this script is contained in the O2DPG repository: https://github.com/AliceO2Group/O2DPG/blob/master/DATA/tools/epn/gen_topo.sh
+
+# The purpose of this script is to separate the topology generation (which is in O2DPG) from the setting of the EPN-related settings
 
 # Settings for some EPN paths / names / etc.
 [ -z "$FILEWORKDIR" ] && export FILEWORKDIR=/home/epn/odc/files # Path to common grp / geometry / etc files
 [ -z "$INRAWCHANNAME" ] && export INRAWCHANNAME=tf-builder-pipe-0 # Pipe name to get data from TfBuilder
 [ -z "$CTF_DIR" ] && export CTF_DIR=/data/tf/compressed # Output directory for CTFs
 [ -z "$CTF_METAFILES_DIR" ] && [ "0$WORKFLOWMODE" != "0print" ] && export CTF_METAFILES_DIR=/data/epn2eos_tool/epn2eos #CTF Metafiles directory
-[ -z "$GEN_TOPO_WORKDIR" ] && export GEN_TOPO_WORKDIR=$HOME/gen_topo/${GEN_TOPO_PARTITION}_${GEN_TOPO_ONTHEFLY} # Persistent working directory for checkout of O2DPG repository and for XML cache. Must be per partition. This script must not run twice in parallel with the same workdir
-[ -z "$GEN_TOPO_STDERR_LOGGING" ] && export GEN_TOPO_STDERR_LOGGING=1
+[ -z "$GEN_TOPO_WORKDIR" ] && export GEN_TOPO_WORKDIR=$HOME/gen_topo/ # Working directory for checkout of O2DPG repository and for XML cache. If this directory is wiped, gen_topo will recreate all necessary content the next time it runs. The folder should be persistent to cache workflows.
+[ -z "$GEN_TOPO_STDERR_LOGGING" ] && export GEN_TOPO_STDERR_LOGGING=1 # Enable logging of stderr messages
 [ -z "$IS_SIMULATED_DATA" ] && export IS_SIMULATED_DATA=0 # by default we are processing raw data
+[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ] && export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recown 'wn_(?"'!'"online-calib).*_.*' --calibwn 'wn_online-calib_.*'" # Arguments to pass to odc-epn-topo command
+[ -z "$GEN_TOPO_EPN_CCDB_SERVER" ] && export GEN_TOPO_EPN_CCDB_SERVER="http://o2-ccdb.internal" # CCDB server to use
 
-if [[ "0$DDMODE" == "0discard" ]] || [[ "0$DDMODE" == "0disk" ]]; then
-  export GEN_TOPO_LIBRARY_FILE="production/no-processing.desc"
-  export GEN_TOPO_WORKFLOW_NAME="no-processing"
-fi
-
+# GEN_TOPO_RUN_HOME is a debug setting used in some tests. This is not needed for online running.
 if [[ "0$GEN_TOPO_RUN_HOME" == "01" ]]; then
   [[ $WORKFLOWMODE != "print" ]] && { echo "ERROR: GEN_TOPO_RUN_HOME is only supported with WORKFLOWMODE=print!" 1>&2; exit 1; }
 else
   # Load required module and run gen_topo_o2dpg (PDP part of this script)
+  # This makes sure to load ODC only when it is not already in the environment.
+  # The background is that in the online system ODC is already loaded, but for tests it might not be loaded thus we just load the latest available versions for convenience.
   if [[ `module info-loaded ODC 2> /dev/null | grep ODC | wc -l` == 0 ]]; then
     module load ODC 1>&2 || { echo Error loading ODC / O2DPG 1>&2; exit 1; }
   fi
+  # Set O2DPG_ROOT from the latest available O2DPG module, if not already set.
+  # Note that this does not load the module, but just needs an O2DPG path to find, which then does the bulk of the topology generation.
+  # gen_topo_o2dpg.sh is kept compatible between O2DPG versions, thus it doesn't really depend on which O2DPG version we use at this point.
   if [[ -z $O2DPG_ROOT ]]; then
     O2DPG_ROOT=`bash -c "module load O2DPG > /dev/null; echo \\\$O2DPG_ROOT;"`
   fi
 fi
+# Now we know which gen_topo_o2dpg.sh we can use, and all EPN related env variables are set, so we can run the topology generation.
 $O2DPG_ROOT/DATA/tools/epn/gen_topo_o2dpg.sh
 if [ $? != 0 ]; then
   echo topology generation failed 1>&2

DATA/tools/epn/gen_topo_o2dpg.sh

@@ -1,4 +1,40 @@
 #!/bin/bash
+
+# Check settings coming from AliECS via env variables
+if [ -z $GEN_TOPO_PARTITION ]; then echo \$GEN_TOPO_PARTITION missing; exit 1; fi # Partition
+if [ -z $GEN_TOPO_HASH ]; then echo \$GEN_TOPO_HASH missing; exit 1; fi # Flag whether source is a hash or a folder
+if [ -z $GEN_TOPO_SOURCE ]; then echo \$GEN_TOPO_SOURCE missing; exit 1; fi # O2DPG repository source, either a commit hash or a path
+if [ -z $GEN_TOPO_LIBRARY_FILE ]; then echo \$GEN_TOPO_LIBRARY_FILE missing; exit 1; fi # Topology description library file in the DATA path of the O2DPG repository
+if [ -z $GEN_TOPO_WORKFLOW_NAME ]; then echo \$GEN_TOPO_WORKFLOW_NAME missing; exit 1; fi # Workflow name in library file
+if [ -z ${WORKFLOW_DETECTORS+x} ]; then echo \$WORKFLOW_DETECTORS missing; exit 1; fi # Comma-separated list of detectors to run processing for
+if [ -z ${WORKFLOW_DETECTORS_QC+x} ]; then echo \$WORKFLOW_DETECTORS_QC missing; exit 1; fi # Comma-separated list of detectors to run QC for
+if [ -z ${WORKFLOW_DETECTORS_CALIB+x} ]; then echo \$WORKFLOW_DETECTORS_CALIB missing; exit 1; fi # Comma-separated list of detectors to run calibration for
+if [ -z ${WORKFLOW_PARAMETERS+x} ]; then echo \$WORKFLOW_PARAMETERS missing; exit 1; fi # Additional parameters for workflow
+if [ -z ${RECO_NUM_NODES_OVERRIDE+x} ]; then echo \$RECO_NUM_NODES_OVERRIDE missing; exit 1; fi # Override number of nodes
+if [ -z $DDMODE ] && [ -z $DDWORKFLOW ]; then echo Either \$DDMODE or \$DDWORKFLOW must be set; exit 1; fi # Select data distribution workflow
+if [ -z "$MULTIPLICITY_FACTOR_RAWDECODERS" ]; then echo \$MULTIPLICITY_FACTOR_RAWDECODERS missing; exit 1; fi # Process multiplicity scaling parameter
+if [ -z "$MULTIPLICITY_FACTOR_CTFENCODERS" ]; then echo \$MULTIPLICITY_FACTOR_CTFENCODERS missing; exit 1; fi # Process multiplicity scaling parameter
+if [ -z "$MULTIPLICITY_FACTOR_REST" ]; then echo \$MULTIPLICITY_FACTOR_REST missing; exit 1; fi # Process multiplicity scaling parameter
+
+# Check settings coming from the EPN
+if [ -z "$FILEWORKDIR" ]; then echo \$FILEWORKDIR missing; exit 1; fi
+if [ -z "$INRAWCHANNAME" ]; then echo \$INRAWCHANNAME missing; exit 1; fi
+if [ -z "$CTF_DIR" ]; then echo \$CTF_DIR missing; exit 1; fi
+if [ -z "$CTF_METAFILES_DIR" ]; then echo \$CTF_METAFILES_DIR missing; exit 1; fi
+if [ -z "$GEN_TOPO_WORKDIR" ]; then echo \$GEN_TOPO_WORKDIR missing; exit 1; fi
+if [ -z "$GEN_TOPO_STDERR_LOGGING" ]; then echo \$GEN_TOPO_STDERR_LOGGING missing; exit 1; fi
+if [ -z "$IS_SIMULATED_DATA" ]; then echo \$IS_SIMULATED_DATA missing; exit 1; fi
+if [ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]; then echo \$GEN_TOPO_ODC_EPN_TOPO_ARGS missing; exit 1; fi
+if [ -z "$GEN_TOPO_EPN_CCDB_SERVER" ]; then echo \$GEN_TOPO_EPN_CCDB_SERVER missing; exit 1; fi
+
+
+export GEN_TOPO_WORKDIR=$GEN_TOPO_WORKDIR/${GEN_TOPO_PARTITION}_${GEN_TOPO_ONTHEFLY}
+
+if [[ "0$DDMODE" == "0discard" ]] || [[ "0$DDMODE" == "0disk" ]]; then
+  export GEN_TOPO_LIBRARY_FILE="production/no-processing.desc"
+  export GEN_TOPO_WORKFLOW_NAME="no-processing"
+fi
+
 mkdir -p $GEN_TOPO_WORKDIR/cache || { echo Error creating directory 1>&2; exit 1; }
 if [ $GEN_TOPO_HASH == 1 ]; then
   cd $GEN_TOPO_WORKDIR || { echo Cannot enter work dir 1>&2; exit 1; }

DATA/tools/parse

@@ -127,7 +127,7 @@ for line in f:
             if 'RECO_MAX_FAIL_NODES_OVERRIDE' in os.environ and os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'] != "":
                 reconodesmin = max(1, reconodes - int(os.environ['RECO_NUM_NODES_OVERRIDE']))
             if os.environ['WORKFLOWMODE'] == 'dds':
-                odccommand = "odc-epn-topo"
+                odccommand = "odc-epn-topo " + os.environ['GEN_TOPO_ODC_EPN_TOPO_ARGS']
                 if reconodes:
                     replacestring = ""
                     dd_env_variables = ['DD_DISK_FRACTION']
@@ -143,11 +143,11 @@ for line in f:
                     fddout.close()
                     odccommand += " --dd " + filename
                 if len(recoworkflows):
-                    odccommand += " --recown 'wn_(?!online-calib).*_.*' --reco " + " ".join(recoworkflows)
+                    odccommand += " --reco " + " ".join(recoworkflows)
                 odccommand += " --n " + str(reconodes)
                 # odccommand += " --nmin " + str(reconodesmin) # Currently disabled, since odc-epn-topo does not accept --nmin
                 if len(calibworkflows):
-                    odccommand += " --calibwn 'wn_online-calib_.*' --calib " + " ".join(calibworkflows)
+                    odccommand += " --calib " + " ".join(calibworkflows)
                 if "GEN_TOPO_STDERR_LOGGING" in os.environ and int(os.environ['GEN_TOPO_STDERR_LOGGING']):
                     odccommand += " --mon tools/monitoring_workflows/epnstderrlog.xml"
                 if args[1] != "":