#!/bin/sh #---------------------------------------------------------------------------- # /var/install/bin/smartmon-plot - logging, plotting, warning script for smartmon # # Creation: 01.01.2004 jb # Last Update: $Id$ # # Copyright (c) 2004-2008 Jens Berger # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. #---------------------------------------------------------------------------- #exec 2>/tmp/smartmon-plot-$$.log #set -x plot_dir=/var/smartmon/plot log_dir=/var/smartmon/log tmp_dir=/var/smartmon smartmon_device_file=/var/smartmon/smartmon_devices_watch smartmon_plot_file=/var/smartmon/smartmon_devices_plot smartmon_warn_attrib=/var/smartmon/smartmon_warn_attributes x_value=1 # 8=PowerOnHours date=`date +%d.%m.%Y` time=`date +%H:%M` timestamp="$date $time" indexhtml=$plot_dir/index.html menuhtml=$plot_dir/menu.html homehtml=$plot_dir/home.html # read some config files . /etc/config.d/base . /etc/config.d/smartmon # delete old tmp files rm -f $tmp_dir/*.tmp # delete old .plot and .html files rm -f $plot_dir/*.plot rm -f $plot_dir/*.html write_smart_plot () { # write the plot file ( echo "# Plot file of the smartmon package" echo "set grid" if [ $x_value = 1 ]; then echo "set xdata time" echo "set timefmt '%d%m%Y%H%M'" else echo "set xlabel 'PowerOnHours'" fi echo "set size 0.85,0.65" echo "set key below" echo "set terminal png" #echo "set terminal png notransparent color x000000 xccccff xffffff" echo "set output '$smart_png_file'" if [ $smartmon_device_type = "3ware" ]; then echo -n "set title \"3ware ATA-RAID /dev/$dev, port $port\n" else echo -n "set title \"ATA Drive /dev/$dev\n" fi echo -n "Device Model: $Device_Model\n" if [ "$SMARTMON_PLOT_SHOW_SN" = "yes" ]; then echo -n "Serial Number: $Serial_Number\n" fi echo "$HOSTNAME.$DOMAIN_NAME @ $timestamp\"" echo -n "plot" echo -n " '$logfile' u $x_value:3 t '$j (SMART)' w lp" echo -n ", '$logfile' u $x_value:4 t '$j (WORST)' w l" echo -n ", '$logfile' u $x_value:5 t '$j (THRESHOLD)' w l" echo "" echo "replot" ) >$plot_dir/$smart_plot_file } write_large_smart_plot () { # write the plot file ( echo "# Plot file of the smartmon package" echo "set grid" if [ $x_value = 1 ]; then echo "set xdata time" echo "set timefmt '%d%m%Y%H%M'" else echo "set xlabel 'PowerOnHours'" fi echo "set size 1,1" echo "set key below" echo "set terminal png" #echo "set terminal png notransparent color x000000 xccccff xffffff" echo "set output '$large_smart_png_file'" if [ $smartmon_device_type = "3ware" ]; then echo -n "set title \"3ware ATA-RAID /dev/$dev, port $port\n" else echo -n "set title \"ATA Drive /dev/$dev\n" fi echo -n "Device Model: $Device_Model\n" if [ "$SMARTMON_PLOT_SHOW_SN" = "yes" ]; then echo -n "Serial Number: $Serial_Number\n" fi echo "$HOSTNAME.$DOMAIN_NAME @ $timestamp\"" echo -n "plot" echo -n " '$last_values_log' u $x_value:3 t '$j (SMART)' w lp" echo -n ", '$last_values_log' u $x_value:4 t '$j (WORST)' w l" echo -n ", '$last_values_log' u $x_value:5 t '$j (THRESHOLD)' w l" echo "" echo "replot" ) >$plot_dir/$large_smart_plot_file } write_raw_plot () { # write the plot file ( echo "# Plot file of the smartmon package" echo "set grid" if [ $x_value = 1 ]; then echo "set xdata time" echo "set timefmt '%d%m%Y%H%M'" else echo "set xlabel 'PowerOnHours'" fi echo "set size 0.85,0.65" echo "set key below" echo "set terminal png" #echo "set terminal png notransparent color x000000 xccccff xffffff" echo "set output '$raw_png_file'" if [ $smartmon_device_type = "3ware" ]; then echo -n "set title \"3ware ATA-RAID /dev/$dev, port $port\n" else echo -n "set title \"ATA Drive /dev/$dev\n" fi echo -n "Device Model: $Device_Model\n" if [ "$SMARTMON_PLOT_SHOW_SN" = "yes" ]; then echo -n "Serial Number: $Serial_Number\n" fi echo "$HOSTNAME.$DOMAIN_NAME @ $timestamp\"" echo -n "plot" echo -n " '$logfile' u $x_value:7 t '$j (RAW)' w lp" echo "" echo "replot" ) >$plot_dir/$raw_plot_file } write_large_raw_plot () { # write the plot file ( echo "# Plot file of the smartmon package" echo "set grid" if [ $x_value = 1 ]; then echo "set xdata time" echo "set timefmt '%d%m%Y%H%M'" else echo "set xlabel 'PowerOnHours'" fi echo "set size 1,1" echo "set key below" echo "set terminal png" #echo "set terminal png notransparent color x000000 xccccff xffffff" echo "set output '$large_raw_png_file'" if [ $smartmon_device_type = "3ware" ]; then echo -n "set title \"3ware ATA-RAID /dev/$dev, port $port\n" else echo -n "set title \"ATA Drive /dev/$dev\n" fi echo -n "Device Model: $Device_Model\n" if [ "$SMARTMON_PLOT_SHOW_SN" = "yes" ]; then echo -n "Serial Number: $Serial_Number\n" fi echo "$HOSTNAME.$DOMAIN_NAME @ $timestamp\"" echo -n "plot" echo -n " '$last_values_log' u $x_value:7 t '$j (RAW)' w lp" echo "" echo "replot" ) >$plot_dir/$large_raw_plot_file } write_log () { if [ -f $tmpfile ]; then Device_Model=`grep "Device Model" $tmpfile | awk '{print $3" "$4}' | sed 's/ $//'` Serial_Number=`grep "Serial Number" $tmpfile | awk '{print $3" "$4}' | sed 's/ $//'` # check if the table contains 9 or 10 columns, # sometimes the 'UPDATED' column is missing. # we want to get the raw value too. # We are using 'tail -n1' to filter the last line of perhaps more than one lines # due to possibly doubled output of some attributes! Seen for 'Temperature_Celcius' # for at least one SAMSUNG SP0842N if [ "$contains_updated" = "" ]; then k=`grep "$j" $tmpfile | tail -n1 | awk '{print $1" "$4" "$5" "$6" "$8" "$9}'` # get Power_On_Hours/Power_On_Seconds additionally, here column 9 p=`grep "Power_On_" $tmpfile | tail -n1 | awk '{print $9}' | cut -dh -f1` else k=`grep "$j" $tmpfile | tail -n1 | awk '{print $1" "$4" "$5" "$6" "$9" "$10}'` # get Power_On_Hours/Power_On_Seconds additionally, here column 10 p=`grep "Power_On_" $tmpfile | tail -n1 | awk '{print $10}' | cut -dh -f1` fi # write the log file: # 1 2 3 4 5 6 7 8 #---------- --- ----- ----- --------- ----------- --- ---------------------- # Timestamp ID# Value Worst Threshold When_Failed RAW PowerOn[Hours|Seconds] echo -n "$(date +%d%m%Y%H%M) " >> $logfile # Timestamp echo -n "$k " >> $logfile # column 2-7 echo -n "$p" >> $logfile # 8, PowerOn*, could be missed echo "" >> $logfile fi tail -n 60 $logfile > $last_values_log } send_warn_mail () { ( echo "From: S.M.A.R.T. Daemon " echo "To: $SMARTMON_FAILURE_MAIL_TO" echo -n "Subject: '$j' on " if [ $smartmon_device_type = "3ware" ]; then echo -n "/dev/$dev, port $port " else echo -n "/dev/$dev " fi echo "is $relation threshold" echo "" echo "Dear Admin," echo "" echo -n "the S.M.A.R.T. attribute \"$j\"" echo "" echo -n "of device " if [ $smartmon_device_type = "3ware" ]; then echo -n "'3ware-RAID /dev/$dev, port $port'" else echo -n "'/dev/$dev'" fi echo " is with a current value of" echo "$recent_value $relation threshold of $threshold." echo "" echo "The EIS/FAIR S.M.A.R.T. Daemon" ) | /usr/lib/sendmail $SMARTMON_FAILURE_MAIL_TO } check_warn () { if [ -f $smartmon_warn_attrib ]; then should_be_watched=`grep "$j" $smartmon_warn_attrib | grep "$dev"` if [ "$should_be_watched" != "" ]; then # smart or raw? kind=`grep "$dev:$j:" $smartmon_warn_attrib | cut -d: -f3` if [ "$kind" = "smart" ]; then # get smart value from logfile recent_value=`grep $(date +%d%m%Y%H%M) $logfile | cut -d\ -f3` elif [ "$kind" = "raw" ]; then # get raw value from logfile recent_value=`grep $(date +%d%m%Y%H%M) $logfile | cut -d\ -f7` fi # over or under threshold? relation=`grep "$dev:$j:" $smartmon_warn_attrib | cut -d: -f4` # the threshold itself threshold=`grep "$dev:$j:" $smartmon_warn_attrib | cut -d: -f5` if [ "$relation" = "over" ]; then if [ "$recent_value" -gt "$threshold" ]; then send_warn_mail fi elif [ "$relation" = "under" ]; then if [ "$recent_value" -lt "$threshold" ]; then send_warn_mail fi fi fi fi } write_all () { write_log write_raw_plot write_large_raw_plot write_smart_plot write_large_smart_plot check_warn } naming_files () { logfile=$log_dir/plot_$value_device.log last_values_log=$tmp_dir/plot_large_$value_device.log smart_plot_file=plot_smart_$value_device.plot large_smart_plot_file=plot_large_smart_$value_device.plot raw_plot_file=plot_raw_$value_device.plot large_raw_plot_file=plot_large_raw_$value_device.plot smart_png_file=plot_smart_$value_device.png raw_png_file=plot_raw_$value_device.png large_smart_png_file=plot_large_smart_$value_device.png large_raw_png_file=plot_large_raw_$value_device.png # remove underscores from value names val=`echo $j | sed 's/_/ /g'` } naming_html_files () { smart_device_html=plot_smart_$devport.html smart_device_html_abs=$plot_dir/$smart_device_html raw_device_html=plot_raw_$devport.html raw_device_html_abs=$plot_dir/$raw_device_html if [ ! -f $smart_device_html_abs ]; then echo "" >> $smart_device_html_abs echo "" >> $smart_device_html_abs echo " " >> $smart_device_html_abs echo " " >> $smart_device_html_abs echo " " >> $smart_device_html_abs echo " " >> $smart_device_html_abs echo " " >> $smart_device_html_abs echo " " >> $smart_device_html_abs fi if [ ! -f $raw_device_html_abs ]; then echo "" >> $raw_device_html_abs echo "" >> $smart_device_html_abs echo " " >> $raw_device_html_abs echo " " >> $raw_device_html_abs echo " " >> $raw_device_html_abs echo " " >> $raw_device_html_abs echo " " >> $raw_device_html_abs echo " " >> $raw_device_html_abs fi } # write the index.html file ( echo "" echo "" echo " " echo " S.M.A.R.T. History for $HOSTNAME.$DOMAIN_NAME" echo " " echo " " echo " " echo " " echo " " echo " " echo " " echo "" ) > $indexhtml # write the home.html file ( echo "" echo "" echo " " echo " S.M.A.R.T. History for $HOSTNAME.$DOMAIN_NAME" echo " " echo " " echo " " echo " " echo "

S.M.A.R.T. History for $HOSTNAME.$DOMAIN_NAME

" echo " " echo "" ) > $homehtml # create menu html file echo "" >> $menuhtml echo "" >> $menuhtml echo " " >> $menuhtml echo " " >> $menuhtml echo " " >> $menuhtml echo " " >> $menuhtml echo " " >> $menuhtml echo " " >> $menuhtml devices=`cat $smartmon_plot_file | sed -e 's/\/dev\///g' | cut -d: -f1 | sort` for dev in $devices; do # get device type smartmon_device_type=`grep "$dev:" $smartmon_plot_file | cut -d: -f2` # get port list ports=`grep "$dev:" $smartmon_plot_file | cut -d: -f3` # if ports not 0, then assume a RAID controller :-) if [ "$ports" != "0" ]; then portlist=`echo $ports | sed 's/,/ /g'` for port in `echo $portlist`; do # we need to compress this until 'plot_smart_$dev_$port.html' does not work devport=''$dev'_'$port'' naming_html_files echo " /dev/$dev, port $port
" >> $menuhtml echo "   [Pre-failure Attributes]
" >> $menuhtml tmpfile=$tmp_dir/plot_$devport.tmp # write attribute table into temp file /usr/sbin/smartctl -A -d $smartmon_device_type,$port /dev/$dev > $tmpfile # add device model and serial (-A1!) number to temp file /usr/sbin/smartctl -a -d $smartmon_device_type,$port /dev/$dev | grep -A1 "Device Model" >> $tmpfile # collect pre-failure values, using 'uniq' to filter doubled attributes prefail_values=`cat $tmpfile | grep 'Pre-fail' | grep -v 'Unknown_Attribute' | awk '{print $2}' | uniq -i` # collect old-age values, using 'uniq' to filter doubled attributes oldage_values=`cat $tmpfile | grep 'Old_age' | grep -v 'Unknown_Attribute' | awk '{print $2}' | uniq -i` # check for the 'UPDATED' column contains_updated=`cat $tmpfile | grep '^ID# ' | grep 'UPDATED'` for j in $prefail_values; do eval value_device=''$dev'_'$port'_Pre_Fail_$j' naming_files echo " $val - [Pre-Failure] (SMART)
" >> $smart_device_html_abs echo " \"$smart_png_file\"

" >> $smart_device_html_abs echo " $val - [Pre-Failure] (Raw)
" >> $raw_device_html_abs echo " \"$smart_png_file\"

" >> $raw_device_html_abs echo "     $val (Raw)
" >> $menuhtml if [ "`grep $smart_device_html $menuhtml`" = "" ]; then echo " /dev/$dev, port $port
" >> $menuhtml fi write_all done echo "   [Old-Age Attributes]
" >> $menuhtml for j in $oldage_values; do eval value_device=''$dev'_'$port'_Old_Age_$j' naming_files echo " $val - [Old Age] (SMART)
" >> $smart_device_html_abs echo " \"$smart_png_file\"

" >> $smart_device_html_abs echo " $val - [Old Age] (Raw)
" >> $raw_device_html_abs echo " \"$smart_png_file\"

" >> $raw_device_html_abs echo "     $val (Raw)
" >> $menuhtml if [ "`grep $smart_device_html $menuhtml`" = "" ]; then echo " /dev/$dev, port $port
" >> $menuhtml fi write_all done done else devport=''$dev'' naming_html_files # ports == "0" ==> (s|p)ata echo " /dev/$dev
" >> $menuhtml echo "   [Pre-failure Attributes]
" >> $menuhtml tmpfile=$tmp_dir/plot_$dev.tmp /usr/sbin/smartctl -A -d $smartmon_device_type /dev/$dev > $tmpfile /usr/sbin/smartctl -a -d $smartmon_device_type /dev/$dev | grep -A1 "Device Model" >> $tmpfile # collect pre-failure values, using 'uniq' to filter doubled attributes prefail_values=`cat $tmpfile | grep 'Pre-fail' | grep -v 'Unknown_Attribute' | awk '{print $2}' | uniq -i` # collect old-age values, using 'uniq' to filter doubled attributes oldage_values=`cat $tmpfile | grep 'Old_age' | grep -v 'Unknown_Attribute' | awk '{print $2}' | uniq -i` # check for the 'UPDATED' column contains_updated=`cat $tmpfile | grep '^ID# ' | grep 'UPDATED'` for j in $prefail_values; do eval value_device=''$dev'_Pre_Fail_$j' naming_files echo " $val - [Pre-Failure] (SMART)
" >> $smart_device_html_abs echo " \"$smart_png_file\"

" >> $smart_device_html_abs echo " $val - [Pre-Failure] (Raw)
" >> $raw_device_html_abs echo " \"$smart_png_file\"

" >> $raw_device_html_abs echo "     $val (Raw)
" >> $menuhtml if [ "`grep $smart_device_html $menuhtml`" = "" ]; then echo " /dev/$dev
" >> $menuhtml fi write_all done echo "   [Old-Age Attributes]
" >> $menuhtml for j in $oldage_values; do eval value_device=''$dev'_Old_Age_$j' naming_files echo " $val - [Old Age] (SMART)
" >> $smart_device_html_abs echo " \"$smart_png_file\"

" >> $smart_device_html_abs echo " $val - [Old Age] (Raw)
" >> $raw_device_html_abs echo " \"$smart_png_file\"

" >> $raw_device_html_abs echo "     $val (Raw)
" >> $menuhtml if [ "`grep $smart_device_html $menuhtml`" = "" ]; then echo " /dev/$dev
" >> $menuhtml fi write_all done fi done # close the files echo " " >> $smart_device_html_abs echo "" >> $smart_device_html_abs echo " " >> $raw_device_html_abs echo "" >> $raw_device_html_abs # close the menu.html file echo " " >> $menuhtml echo "" >> $menuhtml # do the job cd $plot_dir/ for plotfile in '*.plot'; do nice -n 19 /usr/sbin/gnuplot $plotfile 2>/dev/null done # remove temporary files rm -f *.plot rm -f $tmp_dir/plot_* find $plot_dir -type d | xargs chmod 755 find $plot_dir -type f | xargs chmod 644 exit 0