#!/bin/bash
#
# test_procmon
#
# ULS agent for details processinformations
# konfiguration in /etc/uls/procmon.conf
#####################################################################
#
unset TMP_PATH SETULSHOSTNAME SECTION TESTSTEP SAVESTATEPATH CONFIGFILE CONFIGDIR
#
while getopts h:s:t:d:T:c:C: op
 do
  case "$op" in
   d) [[ -n "$OPTARG" ]] && TMP_PATH="-d $OPTARG";;
   h) [[ -n "$OPTARG" ]] && SETULSHOSTNAME="$OPTARG";;
   s) [[ -n "$OPTARG" ]] && SECTION="$OPTARG";;
   t) [[ -n "$OPTARG" ]] && TESTSTEP="$OPTARG";;
   T) [[ -n "$OPTARG" ]] && SAVESTATEPATH="$OPTARG";;
   c) [[ -n "$OPTARG" ]] && CONFIGFILE="$OPTARG";;
   C) [[ -n "$OPTARG" ]] && CONFIGDIR="$OPTARG";;
   *) echo "usage: `basename $0` [-C <path>] [-d <path>] [-h <ulshostname>] [-s <section>] [-t <teststep>] [-T <savestatepath>] [-c <configfile>] [<date> [<time>]]"
      exit 1;;
  esac
done
shift $(( $OPTIND - 1))
#
. `dirname $0`/uls_header
#
if [[ -z "$CONFIGFILE" ]]
 then
  CONFIGFILE=${CONFIGDIR:-/etc/uls}/procmon.conf
fi
#
if [[ ! -f "$CONFIGFILE" ]]
 then
  exit 0
fi
#
SSP="${SAVESTATEPATH:-/dev/shm}"
#
if [[ ! -d "$SSP" ]]
 then
  echo "Error: Savestatepatch '$SSP' not found"
  exit 1
fi
#
export LANG=C
#
( PROCS=`awk 'BEGIN{n=0}
$1 !~ "#" && $1 != "" { if(n++)
                          printf("|[(]%s[)]", $1)
                        else
                          printf("[(]%s[)]", $1)}' "$CONFIGFILE"`
  IOPROCS=`awk 'BEGIN{n=0}
$1 !~ "#" && $1 != "" && ($2 ~ /[dDeEfF]/ || $3 ~ /[dDeErw]/) {
                        if(n++)
                          printf("|[(]%s[)]", $1)
                        else
                          printf("[(]%s[)]", $1)}' "$CONFIGFILE"`

  if [[ -z "$PROCS" ]]
   then
    exit 0
  fi

  # /proc/stat
  #    1      2     3     4     5     6    7     8
  # cpu[<n>] user nice system idle iowait irq softirq
  #
  # /proc/<pid>/stat
  #  1    2    3  .. 14    15  .. 19    20   ..   22      23
  # pid comm state  utime stime  nice num_thr  starttime vsize
  # ->
  #   1    2    3   ...
  # P[AN] pid state ...
  #
  # /proc/<pid>/statm in pages
  #  1      2       3     4    5   6   7
  # size resident shared text lib data dt
  #
  # /proc/<pid>/io
  #   rchar: n                    - characters read            - f
  #   wchar: n                    - characters written         - F
  #   syscr: n                    - read syscalls              - e
  #   syscw: n                    - write syscalls             - E
  #   read_bytes: n               - bytes read (from storage)  - d
  #   write_bytes: n              - bytes written (to storage) - D
  #   cancelled_write_bytes: n    - bytes not written because file is deleted
  #
  # /proc/<pid>/fd
  #   count links                 - number of open files        - o

  if [[ ! -f $SSP/ulsprocmon.pstat.last ]]
   then
    fgrep 'cpu ' /proc/stat >$SSP/ulsprocmon.stat.last
    egrep -h "$PROCS" /proc/*/stat >$SSP/ulsprocmon.pstat.last 2>/dev/null
    exit 0
  fi

  awk -F '[ \t]*#' '$1 != ""{print $1}' "$CONFIGFILE" | sed 's/^/CF /'
  sed 's/^/SA /' $SSP/ulsprocmon.stat.last
  sed 's/^\([^( ]*\) [(]\([^)]*\)[)] \(.*$\)/PA \1 \3 ( \2/' $SSP/ulsprocmon.pstat.last
  if [[ -n "$IOPROCS" && -f $SSP/ulsprocmon.io.last ]]
   then
    sed 's/^/IA /' $SSP/ulsprocmon.io.last
  fi
  if [[ -n "$IOPROCS" ]]
   then
    egrep -h "$IOPROCS" $SSP/ulsprocmon.pstat.last | while read p x
     do
      awk -v P=$p -F ': *' '
        { w[$1] = $2 }
        END { print P, w["rchar"]+0, w["wchar"]+0, w["syscr"]+0, w["syscw"]+0, w["read_bytes"]+0, w["write_bytes"]+0, w["cancelled_write_bytes"]+0 }' /proc/$p/io 2>/dev/null
    done | tee $SSP/ulsprocmon.io.last | sed 's/^/IN /'
  fi
  (date '+SECS %s.%N'; fgrep 'cpu ' /proc/stat) | tee $SSP/ulsprocmon.stat.last | sed 's/^/SN /'
  egrep -h "$PROCS" /proc/*/stat  2>/dev/null | tee $SSP/ulsprocmon.pstat.last | sed 's/^\([^( ]*\) [(]\([^)]*\)[)] \(.*$\)/PN \1 \3 ( \2/'
) | awk -v H="${SETULSHOSTNAME:-$ULSHOSTNAME}" -v SECTION="${SECTION:-System}" -v TESTSTEP="${TESTSTEP:-Process}" -v DT="$DT" -v CPUS=`grep '^cpu[0-9]' /proc/stat | wc -l` -v MEMTOT=`awk '/MemTotal:/{print $2}' /proc/meminfo` -v NOW=`date '+%s.%N'` -v UPTIME=`cut -f 1 -d ' ' /proc/uptime` '

## Arrays:
##  Konfiguration Zeile "i"
##  pn[i]: erste Spalte - Regexp der Prozesse
##  pf[i]: zweite Spalte - Summen-Flags
##  pp[i]: dritte Spalte - Flags pro Prozess
##  cmdlf[i]: vierte Spalte - CMD-Line Zeilenfilter
##  tst[i]: fünfte Spalte - ULS-Teststep
##  sec[i]: sechste Spalte - ULS-Section
## 
##  pti[i]: Zeiger pti[i] -> tti der sec[tti]
##
##
function flformat(f)
{ if( f < 0.0000001 )
    d = 0
  else
  { d = 4 - log(f)/2.3
    if( d < 0 )
     d = 0
  }
  return sprintf("%0.*f", d, f)
}

function getcmdline(a)
{ cmdfile = "/proc/" a "/cmdline"
  if( (getline longcmd[a] < cmdfile) > 0 )
  { shortcmd[a] = longcmd[a]
    gsub("\0.*", "", shortcmd[a])
    gsub("\0", " ", longcmd[a])
  }
  close(cmdfile)
}

BEGIN   { print "D;" DT ";" H ";"
          cmdlflag = 0
          n = 1
          ttposn = 0
        }

# Configfile
/^CF /  { pn[n] = $2
          pf[n] = $3
          if( $4 != "" && $4 != "." )
            pp[n] = $4
          if( $5 != "" && $5 != "." )
          { cmdlf[n] = $5
            cmdlflag = 1
          }
          if( $6 != "" && $6 != "." )
            tst[n] = $6
          sec[n++] = $7
        }

# /proc/<pid>/io - alt
/^IA / { ioarc[$2] = $3
         ioawc[$2] = $4
         ioasr[$2] = $5
         ioasw[$2] = $6
         ioarb[$2] = $7
         ioawb[$2] = $8
         ioacw[$2] = $9
       }

# /proc/<pid>/io - neu
/^IN / { if( ioarc[$2] != "" )
         { ionrc[$2] = $3
           ionwc[$2] = $4
           ionsr[$2] = $5
           ionsw[$2] = $6
           ionrb[$2] = $7
           ionwb[$2] = $8
           ioncw[$2] = $9
         }
       }

# /proc/<pid>/stat - alt
/^PA /  { cua[$2] = $14
          csa[$2] = $15
        }

# /proc/<pid>/stat - neu
/^PN /  { if( cua[$2] != "" )
          { nn = $NF
            pi = $2
            if( $(NF-1) == "(" )
              nn = $NF
            else
            { k = index($0, "(")+2
              nn = substr($0, k)
            }
### print "nn: " nn ", pi: " pi ", cmdlf: " cmdlf[i]
            for( i = 1; i < n; i++ )
            { if( nn ~ pn[i] )
              { if( cmdlf[i] )
                { if( !longcmd[pi] )
                    getcmdline(pi)
### print "  longcmd[" pi "]: " longcmd[pi]
                  if( longcmd[pi] !~ cmdlf[i] )
                    continue
                }
                if( tst[i] )
                  tt = tst[i]
                else
                  tt = TESTSTEP ":" nn
                if( sec[i] )
                  ttsec = sec[i]
                else
                  ttsec = SECTION
                if( !(tti = ttposttsec[ttsec":"tt]) )
                { tti = ++ttposn
                  ttpostt[tti] = tt
                  ttpossec[tti] = ttsec
                  ttposttsec[ttsec":"tt] = tti
                }
                ttii[tti":"i] = 1
                pii[pi":"i] = tti
                if( !done[pi] )
                { cu[pi] =  $14 - cua[pi]
                  cs[pi] = $15 - csa[pi]
                  pt[pi] = $14 + $15
                  pr[pi] = $22
                  pv[pi] = $23 / 1000
                  pnam[pi] = nn
                  pnic[pi] = $19
                  nthr[pi] = $20
                  ps[pi] = $3
                  done[pi] = 1
                }
                if( (pf[i]  pp[i]) ~ "M|R|S|W" )
                { if( !donem[pi] )
                  { statmfile = "/proc/" pi "/statm"
                    if( (getline statm < statmfile) > 0 )
                    { split(statm, stm)
                      if( stm[1] > 0 )
                      { k = pv[pi]/stm[1]
                        res[pi] = stm[2] * k
                        shrd[pi] = stm[3] * k
                      }
                      close(statmfile)
                    }
                    donem[pi] = 1
                  }
                }
                if( (pf[i]  pp[i]) ~ "o" )
                { if( nof[pi] == "" )
                  { countopenfiles = "ls /proc/" pi "/fd | wc -l"
                    countopenfiles | getline nof[pi]
                    close(countopenfiles)
                  }
                }
                if( !donesp[tti":"pi] )
                { pti[i] = tti
                  scc[tti]++;
                  scu[tti] += cu[pi]
                  scs[tti] += cs[pi]
                  spt[tti] += pt[pi]
                  spv[tti] += pv[pi]
                  snthr[tti] += nthr[pi]
                  if( ioarc[pi] != "" && ionrc[pi] != "" )
                  { siorc[tti] += iorc[pi] = ionrc[pi] - ioarc[pi];
                    siowc[tti] += iowc[pi] = ionwc[pi] - ioawc[pi];
                    siosr[tti] += iosr[pi] = ionsr[pi] - ioasr[pi];
                    siosw[tti] += iosw[pi] = ionsw[pi] - ioasw[pi];
                    siorb[tti] += iorb[pi] = ionrb[pi] - ioarb[pi];
                    siowb[tti] += iowb[pi] = ionwb[pi] - ioawb[pi];
                    siocw[tti] += iocw[pi] = ioncw[pi] - ioacw[pi];
                  }
                  donesp[tti":"pi] = 1
                }
                if( pf[i] ~ "M|R|S|W" )
                { if( !donespm[tti":"pi] )
                  { sres[tti] += res[pi]
                    sshrd[tti] += shrd[pi]
                    donespm[tti":"pi] = 1
                  }
                }
                if( pf[i] ~ "o" )
                { if( !donesof[tti":"pi] )
                  { snof[tti] += nof[pi]
                    donesof[tti":"pi] = 1
                  }
                }
              }
            }
          }
        }

# /proc/stat - alt
/^SA /  { if( $2 == "SECS" )
            atime = $3
          { ua = $3; na = $4; sa = $5; ida = $6; ioa = $7
            hirqa = $8; sirqa = $9
          }
        }

# /proc/stat - neu
/^SN /  { if( $2 == "SECS" )
            ntime = $3
          else
          { su = $3-ua; sn = $4-na; ss = $5-sa; sid = $6-ida; sio = $7-ioa
            shirq = $8-hirqa ; ssirq = $9-sirqa
          }
        }

END {
  t = su + sn + ss + sid + sio + shirq + ssirq
  if( t > 0 )
    t = CPUS*100.0/t
  else
    t = 0;

  if( ntime > atime )
    tsd = ntime - atime
  else
    tsd = 0
  for( i in pf )
  { if( pf[i] ~ "a" && !pti[i] )
    { if( tst[i] )
        tt = tst[i]
      else
        tt = TESTSTEP ":"pn[i]
      if( sec[i] )
        section = sec[i]
      else
        section = SECTION
      printf("V;;;\"%s\";\"%s\";Count;0;#;\n", section, tt)
    }
  }

  for( tti in scc )
  { pfi = ""
    for( i in pf )
    { if( ttii[tti":"i] )
        pfi = pfi pf[i]
    }

    section = ttpossec[tti]
    teststep = ttpostt[tti]
    if( pfi ~ "c" )
      printf("V;;;\"%s\";\"%s\";Count;%d;#;\n", section, teststep, scc[tti])
    if( pfi ~ "C" )
      printf("V;;;\"%s\";\"%s\";Cpu;%.2f;%%;\n", section, teststep, (scu[tti]+scs[tti])*t)
    if( pfi ~ "u" )
      printf("V;;;\"%s\";\"%s\";Cpu-User;%.2f;%%;\n", section, teststep, scu[tti]*t)
    if( pfi ~ "y" )
      printf("V;;;\"%s\";\"%s\";Cpu-Sys;%.2f;%%;\n", section, teststep, scs[tti]*t)
    if( pfi ~ "T" )
      printf("V;;;\"%s\";\"%s\";Time+;%d;s;\n", section, teststep, spt[tti]/100)
    if( pfi ~ "M" )
      printf("V;;;\"%s\";\"%s\";Mem;%.2f;%%;\n", section, teststep, sres[tti]/MEMTOT*100.0)
    if( pfi ~ "R" )
      print "V;;;\"" section  "\";\"" teststep "\";RES;" flformat(sres[tti]/1000) ";MB;"
    if( pfi ~ "S" )
      print "V;;;\"" section "\";\"" teststep "\";Shared-Mem;" flformat(sshrd[tti]/1000) ";MB;"
    if( pfi ~ "V" )
      print "V;;;\"" section "\";\"" teststep "\";Virt;" flformat(spv[tti]/1000) ";MB;"
    if( pfi ~ "W" )
      print "V;;;\"" section "\";\"" teststep "\";Mem-Swaped;" flformat((spv[tti]-sres[tti])/1000) ";MB;"
    if( pfi ~ "h" )
      print "V;;;\"" section "\";\"" teststep "\";Num-Threads;" snthr[tti] ";#;"
    if( pfi ~ "o" )
      print "V;;;\"" section "\";\"" teststep "\";Num-Open-Files;" snof[tti] ";#;"
    if( tsd > 0 )
    { if( pfi ~ "d" )
        print "V;;;\"" section "\";\"" teststep "\";io-Read;" flformat(siorb[tti]/1000/tsd) ";kB/s;"
      if( pfi ~ "D" )
        print "V;;;\"" section "\";\"" teststep "\";io-Write;" flformat(siowb[tti]/1000/tsd) ";kB/s;"
      if( pfi ~ "e" )
        print "V;;;\"" section "\";\"" teststep "\";Sys-Read;" flformat(siosr[tti]/1000/tsd) ";#/s;"
      if( pfi ~ "E" )
        print "V;;;\"" section "\";\"" teststep "\";Sys-Write;" flformat(siosw[tti]/1000/tsd) ";#/s;"
      if( pfi ~ "f" )
        print "V;;;\"" section "\";\"" teststep "\";Chars-Read;" flformat(siorc[tti]/1000/tsd) ";kB/s;"
      if( pfi ~ "F" )
        print "V;;;\"" section "\";\"" teststep "\";Chars-Write;" flformat(siowc[tti]/1000/tsd) ";kB/s;"
    }
  }
  for( pi in done )
  { delete ppa
    for( i in pp )
    { if( (tti = pii[pi":"i]) )
        ppa[tti] = ppa[tti] pp[i]
    }

    for( tti in ppa )
    { section = ttpossec[tti]
      teststep = ttpostt[tti] ":" pi
      pfi = ppa[tti]
      if( pfi ~ "b" )
        printf("V;;;\"%s\";\"%s\";Start-Time;%s;{DT};\n", section, teststep, strftime("%F %T",int(NOW-UPTIME+0.5)+int(pr[pi]/100+0.5)))
      if( pfi ~ "B" )
        printf("V;;;\"%s\";\"%s\";Start-Time;%d;{TT};\n", section, teststep, int(NOW-UPTIME+0.5)+int(pr[pi]/100+0.5))
      if( pfi ~ "C" )
        printf("V;;;\"%s\";\"%s\";Cpu;%.2f;%%;\n", section, teststep, (cu[pi]+cs[pi])*t)
      if( pfi ~ "u" )
        printf("V;;;\"%s\";\"%s\";Cpu-User;%.2f;%%;\n", section, teststep, cu[pi]*t)
      if( pfi ~ "y" )
        printf("V;;;\"%s\";\"%s\";Cpu-Sys;%.2f;%%;\n", section, teststep, cs[pi]*t)
      if( pfi ~ "N" )
        printf("V;;;\"%s\";\"%s\";Nice;%d;#;\n", section, teststep, pnic[pi])
      if( pfi ~ "s" )
        printf("V;;;\"%s\";\"%s\";State;%s;[ ];\n", section, teststep, ps[pi])
      if( pfi ~ "T" )
        printf("V;;;\"%s\";\"%s\";Time+;%d;s;\n", section, teststep, pt[pi]/100)
      if( pfi ~ "m" )
        printf("V;;;\"%s\";\"%s\";Uptime;%d;min;\n", section, teststep, (UPTIME-pr[pi]/100)/60)
      if( pfi ~ "t" )
        printf("V;;;\"%s\";\"%s\";Uptime;%d;s;\n", section, teststep, UPTIME-pr[pi]/100)
      if( pfi ~ "M" )
        printf("V;;;\"%s\";\"%s\";Mem;%.2f;%%;\n", section, teststep, res[pi]/MEMTOT*100.0)
      if( pfi ~ "R" )
        print "V;;;\"" section "\";\"" teststep "\";Res;" flformat(res[pi]/1000) ";MB;"
      if( pfi ~ "S" )
        print "V;;;\"" section "\";\"" teststep "\";Shared-Mem;" flformat(shrd[pi]/1000) ";MB;"
      if( pfi ~ "V" )
        print "V;;;\"" section "\";\"" teststep "\";Virt;" flformat(pv[pi]/1000) ";MB;"
      if( pfi ~ "W" )
        print "V;;;\"" section "\";\"" teststep "\";Mem-Swaped;" flformat((pv[pi]-res[pi])/1000) ";MB;"
      if( pfi ~ "h" )
        print "V;;;\"" section "\";\"" teststep "\";Num-Threads;" nthr[pi] ";#;"
      if( pfi ~ "o" )
        print "V;;;\"" section "\";\"" teststep "\";Num-Open-Files;" nof[pi] ";#;"
      if( tsd > 0 )
      { if( pfi ~ "d" )
          print "V;;;\"" section "\";\"" teststep "\";io-Read;" flformat(iorb[pi]/1000/tsd) ";kB/s;"
        if( pfi ~ "D" )
          print "V;;;\"" section "\";\"" teststep "\";io-Write;" flformat(iowb[pi]/1004/tsd) ";kB/s;"
        if( pfi ~ "e" )
          print "V;;;\"" section "\";\"" teststep "\";Sys-Read;" flformat(iosr[pi]/1000/tsd) ";#/s;"
        if( pfi ~ "E" )
          print "V;;;\"" section "\";\"" teststep "\";Sys-Write;" flformat(iosw[pi]/1000/tsd) ";#/s;"
        if( pfi ~ "f" )
          print "V;;;\"" section "\";\"" teststep "\";Chars-Read;" flformat(iorc[pi]/1000/tsd) ";kB/s;"
        if( pfi ~ "F" )
          print "V;;;\"" section "\";\"" teststep "\";Chars-Write;" flformat(iowc[pi]/1000/tsd) ";kB/s;"
      }
      if( pfi ~ "U" )
      { usercmd = "ls -ld /proc/" pi
        usercmd | getline
        print "V;;;\"" section "\";\"" teststep "\";User;" $3 ";[ ];"
        close(usercmd)
      }
      if( pfi ~ "l" )
      { if( !shortcmd[pi] )
          getcmdline(pi)
        print "V;;;\"" section "\";\"" teststep "\";Command;\"" shortcmd[pi] "\";;"
      }
      if( pfi ~ "L" )
      { if( !longcmd[pi] )
          getcmdline(pi)
        print "V;;;\"" section "\";\"" teststep "\";Commandline;\"" longcmd[pi] "\";;"
      }
    }
  }
}' 2>/dev/null | send_test_tab $TMP_PATH
