Workflow generation: wrapper script purges modules in case of online generation, and sets the odc-epn-topo binary explicitly

Author: davidrohr

DATA/tools/epn/gen_topo.sh

@@ -8,26 +8,31 @@
 # The purpose of this script is to separate the topology generation (which is in O2DPG) from the setting of the EPN-related settings
 
 # Settings for some EPN paths / names / etc.
-[ -z "$FILEWORKDIR" ] && export FILEWORKDIR=/home/epn/odc/files # Path to common grp / geometry / etc files
-[ -z "$INRAWCHANNAME" ] && export INRAWCHANNAME=tf-builder-pipe-0 # Pipe name to get data from TfBuilder
-[ -z "$CTF_DIR" ] && export CTF_DIR=/data/tf/compressed # Output directory for CTFs
-[ -z "$CTF_METAFILES_DIR" ] && [ "0$WORKFLOWMODE" != "0print" ] && export CTF_METAFILES_DIR=/data/epn2eos_tool/epn2eos #CTF Metafiles directory
-[ -z "$GEN_TOPO_WORKDIR" ] && export GEN_TOPO_WORKDIR=$HOME/gen_topo/ # Working directory for checkout of O2DPG repository and for XML cache. If this directory is wiped, gen_topo will recreate all necessary content the next time it runs. The folder should be persistent to cache workflows.
-[ -z "$GEN_TOPO_STDERR_LOGGING" ] && export GEN_TOPO_STDERR_LOGGING=1 # Enable logging of stderr messages
-[ -z "$IS_SIMULATED_DATA" ] && export IS_SIMULATED_DATA=0 # by default we are processing raw data
-[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ] && export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recown 'wn_(?"'!'"online-calib).*_.*' --calibwn 'wn_online-calib_.*'" # Arguments to pass to odc-epn-topo command
-[ -z "$GEN_TOPO_EPN_CCDB_SERVER" ] && export GEN_TOPO_EPN_CCDB_SERVER="http://o2-ccdb.internal" # CCDB server to use
+[[ -z "$FILEWORKDIR" ]] && export FILEWORKDIR=/home/epn/odc/files # Path to common grp / geometry / etc files
+[[ -z "$INRAWCHANNAME" ]] && export INRAWCHANNAME=tf-builder-pipe-0 # Pipe name to get data from TfBuilder
+[[ -z "$CTF_DIR" ]] && export CTF_DIR=/data/tf/compressed # Output directory for CTFs
+[[ -z "$CTF_METAFILES_DIR" ]] && [[ "0$WORKFLOWMODE" != "0print" ]] && export CTF_METAFILES_DIR=/data/epn2eos_tool/epn2eos #CTF Metafiles directory
+[[ -z "$GEN_TOPO_WORKDIR" ]] && export GEN_TOPO_WORKDIR=$HOME/gen_topo/ # Working directory for checkout of O2DPG repository and for XML cache. If this directory is wiped, gen_topo will recreate all necessary content the next time it runs. The folder should be persistent to cache workflows.
+[[ -z "$GEN_TOPO_STDERR_LOGGING" ]] && export GEN_TOPO_STDERR_LOGGING=1 # Enable logging of stderr messages
+[[ -z "$IS_SIMULATED_DATA" ]] && export IS_SIMULATED_DATA=0 # by default we are processing raw data
+[[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]] && export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recown 'wn_(?"'!'"online-calib).*_.*' --calibwn 'wn_online-calib_.*'" # Arguments to pass to odc-epn-topo command
+[[ -z "$GEN_TOPO_EPN_CCDB_SERVER" ]] && export GEN_TOPO_EPN_CCDB_SERVER="http://o2-ccdb.internal" # CCDB server to use
 
 # GEN_TOPO_RUN_HOME is a debug setting used in some tests. This is not needed for online running.
 if [[ "0$GEN_TOPO_RUN_HOME" == "01" ]]; then
   [[ $WORKFLOWMODE != "print" ]] && { echo "ERROR: GEN_TOPO_RUN_HOME is only supported with WORKFLOWMODE=print!" 1>&2; exit 1; }
 else
-  # Load required module and run gen_topo_o2dpg (PDP part of this script)
-  # This makes sure to load ODC only when it is not already in the environment.
-  # The background is that in the online system ODC is already loaded, but for tests it might not be loaded thus we just load the latest available versions for convenience.
-  if [[ `module info-loaded ODC 2> /dev/null | grep ODC | wc -l` == 0 ]]; then
-    module load ODC 1>&2 || { echo Error loading ODC / O2DPG 1>&2; exit 1; }
+  if [ "0$GEN_TOPO_ONTHEFLY" == "01" ]; then
+    # In case we run the on the fly generation on the EPN, we define which odc-epn-topo binary to use.
+    # Then we purge the modules, since the topology generation will load the O2PDPSuite with the O2 version that shall run, and that includes ODC.
+    # If there is already something of ODC or O2PDPSuite in the environment, we should remove it to avoid collisions.
+    # We set the odc-epn-topo command to be used explicitly though.
+    # Note this happens only in case of on the fly generation when we run online, in case of tests this is not needed.
+    export GEN_TOPO_ODC_EPN_TOPO_CMD=`which odc-epn-topo`
+    [[ -z $GEN_TOPO_ODC_EPN_TOPO_CMD ]] && { echo "ERROR: no odc-epn-topo in the path" 1>&2; exit 1; }
+    module purge &> /dev/null
   fi
+
   # Set O2DPG_ROOT from the latest available O2DPG module, if not already set.
   # Note that this does not load the module, but just needs an O2DPG path to find, which then does the bulk of the topology generation.
   # gen_topo_o2dpg.sh is kept compatible between O2DPG versions, thus it doesn't really depend on which O2DPG version we use at this point.

DATA/tools/epn/gen_topo_o2dpg.sh

@@ -27,7 +27,6 @@ if [ -z "$IS_SIMULATED_DATA" ]; then echo \$IS_SIMULATED_DATA missing; exit 1; f
 if [ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]; then echo \$GEN_TOPO_ODC_EPN_TOPO_ARGS missing; exit 1; fi
 if [ -z "$GEN_TOPO_EPN_CCDB_SERVER" ]; then echo \$GEN_TOPO_EPN_CCDB_SERVER missing; exit 1; fi
 
-
 export GEN_TOPO_WORKDIR=$GEN_TOPO_WORKDIR/${GEN_TOPO_PARTITION}_${GEN_TOPO_ONTHEFLY}
 
 if [[ "0$DDMODE" == "0discard" ]] || [[ "0$DDMODE" == "0disk" ]]; then

DATA/tools/parse

@@ -127,7 +127,11 @@ for line in f:
             if 'RECO_MAX_FAIL_NODES_OVERRIDE' in os.environ and os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'] != "":
                 reconodesmin = max(1, reconodes - int(os.environ['RECO_NUM_NODES_OVERRIDE']))
             if os.environ['WORKFLOWMODE'] == 'dds':
-                odccommand = "odc-epn-topo " + os.environ['GEN_TOPO_ODC_EPN_TOPO_ARGS']
+                if 'GEN_TOPO_ODC_EPN_TOPO_CMD' in os.environ:
+                  odccommand = os.environ['GEN_TOPO_ODC_EPN_TOPO_CMD']
+                else:
+                  odccommand = 'odc-epn-topo'
+                odccommand += ' ' + os.environ['GEN_TOPO_ODC_EPN_TOPO_ARGS']
                 if reconodes:
                     replacestring = ""
                     dd_env_variables = ['DD_DISK_FRACTION']