o2-aliecs-shmcleaner

#!/bin/bash

# === This file is part of ALICE O² ===
#
#  Copyright 2021-2022 CERN and copyright holders of ALICE O².
#  Author: Teo Mrnjavac <teo.mrnjavac@cern.ch>
#          Kostas Alexopoulos <kostas.alexopoulos@cern.ch>
#          Roberto Divià <Roberto.Divia@cern.ch>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#  In applying this license CERN does not waive the privileges and
#  immunities granted to it by virtue of its status as an
#  Intergovernmental Organization or submit itself to any jurisdiction.

O2_INFOLOGGER_PATH=/opt/o2-InfoLogger
export O2_SYSTEM=ECS
export O2_FACILITY=core/shmcleaner
export O2_PARTITION=${O2_PARTITION:-}
export O2_ROLE=${O2_ROLE:-}

waitFor=5   # Number of seconds to wait before force-killing

memStart="`grep MemAvailable /proc/meminfo | sed 's/.*: *//'`"

# ------ Beginning extra cleanup ------

# Try to kill leftover processes gracefully
n=0
for p in `pgrep -u flp`; do
   c="`ps -ho %c $p`"
   [ "$c" = "o2-alf" ] && continue
   [ "$c" ] || continue
   echo "Killing '$c' PID:$p" | $O2_INFOLOGGER_PATH/bin/o2-infologger-log -x -s Debug -o Facility=$O2_FACILITY -o Partition=$O2_PARTITION
   kill $p
   n=$(( n + 1 ))
done

# Check if all cleaned
w=""
for p in `pgrep -u flp`; do
   c="`ps -ho %c $p`"
   [ "$c" = "o2-alf" ] && continue
   [ "$c" ] || continue
   w="T"
done

# Force kill anything remaining
nn=0
if [ "$w" = "T" ]; then
   sleep ${waitFor}
   for p in `pgrep -u flp`; do
      c="`ps -ho %c $p`"
      [ "$c" = "o2-alf" ] && continue
      [ "$c" ] || continue
      echo "Killing -9 '$c' PID:$p" | $O2_INFOLOGGER_PATH/bin/o2-infologger-log -x -s Debug -o Facility=$O2_FACILITY -o Partition=$O2_PARTITION
      kill -9 $p
      nn=$(( nn + 1 ))
   done
fi

# Ensure that no flp-owned processes remain before returning
t=0
while true; do
   l=""
   x=0
   for p in `pgrep -u flp`; do
      c="`ps -ho %c $p`"
      [ "$c" = "o2-alf" ] && continue
      [ "$c" ] || continue
      l="$l'$c':$p "
      x=$(( x + 1 ))
   done
   [ "$l" ] || break
   if (( t++ > 50 )); then
     echo "Unkillable process(es):$l" | $O2_INFOLOGGER_PATH/bin/o2-infologger-log -x -s Debug -o Facility=$O2_FACILITY -o Partition=$O2_PARTITION
     break
   fi
   sleep 0.1
done

# Cleanup all SHM (unused as all processes should have been stopped)
rm -f /dev/shm/*fmq*

# Cleanup any ROC-allocated memory
if true; then
   # This is the standard cleanup procedure
   yes | roc-cleanup --light > /dev/null 3>&1
else
   # This is the deep cleanup procedure, it can break ALF/FRED and should not be used unless really necessary
   systemctl stop o2-alf
   yes | roc-cleanup > /dev/null 3>&1
   systemctl start o2-alf
fi

memEnd="`grep MemAvailable /proc/meminfo | sed 's/.*: *//'`"
memDelta=$(( `echo ${memEnd} | awk '{print $1;}'`- `echo ${memStart} | awk '{print $1;}'` ))
memDelta="${memDelta} `echo ${memEnd} | awk '{print $2;}'`"

echo "Cleanup completed killed:$n killed-9:$nn leftover:$x memStart:${memStart} memEnd:${memEnd} memDelta:${memDelta}" | $O2_INFOLOGGER_PATH/bin/o2-infologger-log -x -s Debug -o Facility=$O2_FACILITY -o Partition=$O2_PARTITION

exit 0

# ------ End extra cleanup -------

ANY_FILES_CLEANED=false

for SHM_FILE in /dev/shm/*; do
  if [ -f $SHM_FILE ] ; then
    SIZE=$(ls -lah $SHM_FILE | awk -F " " {'print $5'})
    if ! fuser -s $SHM_FILE ; then
      rm -f $SHM_FILE
      echo "freed unused shared memory $SHM_FILE ($SIZE)" | $O2_INFOLOGGER_PATH/bin/o2-infologger-log -x -o Facility=$O2_FACILITY
      ANY_FILES_CLEANED=true
    else
      echo "could not free shared memory in use $SHM_FILE ($SIZE)" | $O2_INFOLOGGER_PATH/bin/o2-infologger-log -x -s Warning -o Facility=$O2_FACILITY
    fi
  fi
done

if [ "$ANY_FILES_CLEANED" = false ] ; then
  echo "no shared memory freed" | $O2_INFOLOGGER_PATH/bin/o2-infologger-log -x -o Facility=$O2_FACILITY
fi