#!/bin/sh
#----------------------------------------------------------------------------
# /var/install/bin/smartmon-plot - logging, plotting, warning script for smartmon
#
# Creation: 01.01.2004 jb
# Last Update: $Id$
#
# Copyright (c) 2004-2008 Jens Berger <jberger@gmx.net>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#----------------------------------------------------------------------------
#exec 2>/tmp/smartmon-plot-$$.log
#set -x
plot_dir=/var/smartmon/plot
log_dir=/var/smartmon/log
tmp_dir=/var/smartmon
smartmon_device_file=/var/smartmon/smartmon_devices_watch
smartmon_plot_file=/var/smartmon/smartmon_devices_plot
smartmon_warn_attrib=/var/smartmon/smartmon_warn_attributes
x_value=1 # 8=PowerOnHours
date=`date +%d.%m.%Y`
time=`date +%H:%M`
timestamp="$date $time"
indexhtml=$plot_dir/index.html
menuhtml=$plot_dir/menu.html
homehtml=$plot_dir/home.html
# read some config files
. /etc/config.d/base
. /etc/config.d/smartmon
# delete old tmp files
rm -f $tmp_dir/*.tmp
# delete old .plot and .html files
rm -f $plot_dir/*.plot
rm -f $plot_dir/*.html
write_smart_plot ()
{
# write the plot file
( echo "# Plot file of the smartmon package"
echo "set grid"
if [ $x_value = 1 ]; then
echo "set xdata time"
echo "set timefmt '%d%m%Y%H%M'"
else
echo "set xlabel 'PowerOnHours'"
fi
echo "set size 0.85,0.65"
echo "set key below"
echo "set terminal png"
#echo "set terminal png notransparent color x000000 xccccff xffffff"
echo "set output '$smart_png_file'"
if [ $smartmon_device_type = "3ware" ]; then
echo -n "set title \"3ware ATA-RAID /dev/$dev, port $port\n"
else
echo -n "set title \"ATA Drive /dev/$dev\n"
fi
echo -n "Device Model: $Device_Model\n"
if [ "$SMARTMON_PLOT_SHOW_SN" = "yes" ]; then
echo -n "Serial Number: $Serial_Number\n"
fi
echo "$HOSTNAME.$DOMAIN_NAME @ $timestamp\""
echo -n "plot"
echo -n " '$logfile' u $x_value:3 t '$j (SMART)' w lp"
echo -n ", '$logfile' u $x_value:4 t '$j (WORST)' w l"
echo -n ", '$logfile' u $x_value:5 t '$j (THRESHOLD)' w l"
echo ""
echo "replot"
) >$plot_dir/$smart_plot_file
}
write_large_smart_plot ()
{
# write the plot file
( echo "# Plot file of the smartmon package"
echo "set grid"
if [ $x_value = 1 ]; then
echo "set xdata time"
echo "set timefmt '%d%m%Y%H%M'"
else
echo "set xlabel 'PowerOnHours'"
fi
echo "set size 1,1"
echo "set key below"
echo "set terminal png"
#echo "set terminal png notransparent color x000000 xccccff xffffff"
echo "set output '$large_smart_png_file'"
if [ $smartmon_device_type = "3ware" ]; then
echo -n "set title \"3ware ATA-RAID /dev/$dev, port $port\n"
else
echo -n "set title \"ATA Drive /dev/$dev\n"
fi
echo -n "Device Model: $Device_Model\n"
if [ "$SMARTMON_PLOT_SHOW_SN" = "yes" ]; then
echo -n "Serial Number: $Serial_Number\n"
fi
echo "$HOSTNAME.$DOMAIN_NAME @ $timestamp\""
echo -n "plot"
echo -n " '$last_values_log' u $x_value:3 t '$j (SMART)' w lp"
echo -n ", '$last_values_log' u $x_value:4 t '$j (WORST)' w l"
echo -n ", '$last_values_log' u $x_value:5 t '$j (THRESHOLD)' w l"
echo ""
echo "replot"
) >$plot_dir/$large_smart_plot_file
}
write_raw_plot ()
{
# write the plot file
( echo "# Plot file of the smartmon package"
echo "set grid"
if [ $x_value = 1 ]; then
echo "set xdata time"
echo "set timefmt '%d%m%Y%H%M'"
else
echo "set xlabel 'PowerOnHours'"
fi
echo "set size 0.85,0.65"
echo "set key below"
echo "set terminal png"
#echo "set terminal png notransparent color x000000 xccccff xffffff"
echo "set output '$raw_png_file'"
if [ $smartmon_device_type = "3ware" ]; then
echo -n "set title \"3ware ATA-RAID /dev/$dev, port $port\n"
else
echo -n "set title \"ATA Drive /dev/$dev\n"
fi
echo -n "Device Model: $Device_Model\n"
if [ "$SMARTMON_PLOT_SHOW_SN" = "yes" ]; then
echo -n "Serial Number: $Serial_Number\n"
fi
echo "$HOSTNAME.$DOMAIN_NAME @ $timestamp\""
echo -n "plot"
echo -n " '$logfile' u $x_value:7 t '$j (RAW)' w lp"
echo ""
echo "replot"
) >$plot_dir/$raw_plot_file
}
write_large_raw_plot ()
{
# write the plot file
( echo "# Plot file of the smartmon package"
echo "set grid"
if [ $x_value = 1 ]; then
echo "set xdata time"
echo "set timefmt '%d%m%Y%H%M'"
else
echo "set xlabel 'PowerOnHours'"
fi
echo "set size 1,1"
echo "set key below"
echo "set terminal png"
#echo "set terminal png notransparent color x000000 xccccff xffffff"
echo "set output '$large_raw_png_file'"
if [ $smartmon_device_type = "3ware" ]; then
echo -n "set title \"3ware ATA-RAID /dev/$dev, port $port\n"
else
echo -n "set title \"ATA Drive /dev/$dev\n"
fi
echo -n "Device Model: $Device_Model\n"
if [ "$SMARTMON_PLOT_SHOW_SN" = "yes" ]; then
echo -n "Serial Number: $Serial_Number\n"
fi
echo "$HOSTNAME.$DOMAIN_NAME @ $timestamp\""
echo -n "plot"
echo -n " '$last_values_log' u $x_value:7 t '$j (RAW)' w lp"
echo ""
echo "replot"
) >$plot_dir/$large_raw_plot_file
}
write_log ()
{
if [ -f $tmpfile ]; then
Device_Model=`grep "Device Model" $tmpfile | awk '{print $3" "$4}' | sed 's/ $//'`
Serial_Number=`grep "Serial Number" $tmpfile | awk '{print $3" "$4}' | sed 's/ $//'`
# check if the table contains 9 or 10 columns,
# sometimes the 'UPDATED' column is missing.
# we want to get the raw value too.
# We are using 'tail -n1' to filter the last line of perhaps more than one lines
# due to possibly doubled output of some attributes! Seen for 'Temperature_Celcius'
# for at least one SAMSUNG SP0842N
if [ "$contains_updated" = "" ]; then
k=`grep "$j" $tmpfile | tail -n1 | awk '{print $1" "$4" "$5" "$6" "$8" "$9}'`
# get Power_On_Hours/Power_On_Seconds additionally, here column 9
p=`grep "Power_On_" $tmpfile | tail -n1 | awk '{print $9}' | cut -dh -f1`
else
k=`grep "$j" $tmpfile | tail -n1 | awk '{print $1" "$4" "$5" "$6" "$9" "$10}'`
# get Power_On_Hours/Power_On_Seconds additionally, here column 10
p=`grep "Power_On_" $tmpfile | tail -n1 | awk '{print $10}' | cut -dh -f1`
fi
# write the log file:
# 1 2 3 4 5 6 7 8
#---------- --- ----- ----- --------- ----------- --- ----------------------
# Timestamp ID# Value Worst Threshold When_Failed RAW PowerOn[Hours|Seconds]
echo -n "$(date +%d%m%Y%H%M) " >> $logfile # Timestamp
echo -n "$k " >> $logfile # column 2-7
echo -n "$p" >> $logfile # 8, PowerOn*, could be missed
echo "" >> $logfile
fi
tail -n 60 $logfile > $last_values_log
}
send_warn_mail ()
{
( echo "From: S.M.A.R.T. Daemon <root@$HOSTNAME.$DOMAIN_NAME>"
echo "To: $SMARTMON_FAILURE_MAIL_TO"
echo -n "Subject: '$j' on "
if [ $smartmon_device_type = "3ware" ]; then
echo -n "/dev/$dev, port $port "
else
echo -n "/dev/$dev "
fi
echo "is $relation threshold"
echo ""
echo "Dear Admin,"
echo ""
echo -n "the S.M.A.R.T. attribute \"$j\""
echo ""
echo -n "of device "
if [ $smartmon_device_type = "3ware" ]; then
echo -n "'3ware-RAID /dev/$dev, port $port'"
else
echo -n "'/dev/$dev'"
fi
echo " is with a current value of"
echo "$recent_value $relation threshold of $threshold."
echo ""
echo "The EIS/FAIR S.M.A.R.T. Daemon"
) | /usr/lib/sendmail $SMARTMON_FAILURE_MAIL_TO
}
check_warn ()
{
if [ -f $smartmon_warn_attrib ]; then
should_be_watched=`grep "$j" $smartmon_warn_attrib | grep "$dev"`
if [ "$should_be_watched" != "" ]; then
# smart or raw?
kind=`grep "$dev:$j:" $smartmon_warn_attrib | cut -d: -f3`
if [ "$kind" = "smart" ]; then
# get smart value from logfile
recent_value=`grep $(date +%d%m%Y%H%M) $logfile | cut -d\ -f3`
elif [ "$kind" = "raw" ]; then
# get raw value from logfile
recent_value=`grep $(date +%d%m%Y%H%M) $logfile | cut -d\ -f7`
fi
# over or under threshold?
relation=`grep "$dev:$j:" $smartmon_warn_attrib | cut -d: -f4`
# the threshold itself
threshold=`grep "$dev:$j:" $smartmon_warn_attrib | cut -d: -f5`
if [ "$relation" = "over" ]; then
if [ "$recent_value" -gt "$threshold" ]; then
send_warn_mail
fi
elif [ "$relation" = "under" ]; then
if [ "$recent_value" -lt "$threshold" ]; then
send_warn_mail
fi
fi
fi
fi
}
write_all ()
{
write_log
write_raw_plot
write_large_raw_plot
write_smart_plot
write_large_smart_plot
check_warn
}
naming_files ()
{
logfile=$log_dir/plot_$value_device.log
last_values_log=$tmp_dir/plot_large_$value_device.log
smart_plot_file=plot_smart_$value_device.plot
large_smart_plot_file=plot_large_smart_$value_device.plot
raw_plot_file=plot_raw_$value_device.plot
large_raw_plot_file=plot_large_raw_$value_device.plot
smart_png_file=plot_smart_$value_device.png
raw_png_file=plot_raw_$value_device.png
large_smart_png_file=plot_large_smart_$value_device.png
large_raw_png_file=plot_large_raw_$value_device.png
# remove underscores from value names
val=`echo $j | sed 's/_/ /g'`
}
naming_html_files ()
{
smart_device_html=plot_smart_$devport.html
smart_device_html_abs=$plot_dir/$smart_device_html
raw_device_html=plot_raw_$devport.html
raw_device_html_abs=$plot_dir/$raw_device_html
if [ ! -f $smart_device_html_abs ]; then
echo "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">" >> $smart_device_html_abs
echo "<HTML>" >> $smart_device_html_abs
echo " <HEAD>" >> $smart_device_html_abs
echo " <TITLE></TITLE>" >> $smart_device_html_abs
echo " <META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; charset=ISO-8859-1\">" >> $smart_device_html_abs
echo " <META HTTP-EQUIV=\"EXPIRES\" CONTENT=\"0\">" >> $smart_device_html_abs
echo " </HEAD>" >> $smart_device_html_abs
echo " <BODY>" >> $smart_device_html_abs
fi
if [ ! -f $raw_device_html_abs ]; then
echo "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">" >> $raw_device_html_abs
echo "<HTML>" >> $smart_device_html_abs
echo " <HEAD>" >> $raw_device_html_abs
echo " <TITLE></TITLE>" >> $raw_device_html_abs
echo " <META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; charset=ISO-8859-1\">" >> $raw_device_html_abs
echo " <META HTTP-EQUIV=\"EXPIRES\" CONTENT=\"0\">" >> $raw_device_html_abs
echo " </HEAD>" >> $raw_device_html_abs
echo " <BODY>" >> $raw_device_html_abs
fi
}
# write the index.html file
( echo "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">"
echo "<HTML>"
echo " <HEAD>"
echo " <TITLE>S.M.A.R.T. History for $HOSTNAME.$DOMAIN_NAME</TITLE>"
echo " <META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; charset=ISO-8859-1\">"
echo " <META HTTP-EQUIV=\"EXPIRES\" CONTENT=\"0\">"
echo " </HEAD>"
echo " <frameset cols=\"300,*\" border=0 frameborder=0 framespacing=0>"
echo " <frame src=\"menu.html\" name=\"menu\" marginwidth=5 marginheight=5>"
echo " <frame src=\"home.html\" name=\"home\" marginwidth=5 marginheight=5>"
echo " </frameset>"
echo "</HTML>"
) > $indexhtml
# write the home.html file
( echo "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">"
echo "<HTML>"
echo " <HEAD>"
echo " <TITLE>S.M.A.R.T. History for $HOSTNAME.$DOMAIN_NAME</TITLE>"
echo " <META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; charset=ISO-8859-1\">"
echo " <META HTTP-EQUIV=\"EXPIRES\" CONTENT=\"0\">"
echo " </HEAD>"
echo " <BODY>"
echo " <div align=\"center\"><h1>S.M.A.R.T. History for $HOSTNAME.$DOMAIN_NAME</div></h1>"
echo " </BODY>"
echo "</HTML>"
) > $homehtml
# create menu html file
echo "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">" >> $menuhtml
echo "<HTML>" >> $menuhtml
echo " <HEAD>" >> $menuhtml
echo " <TITLE></TITLE>" >> $menuhtml
echo " <META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; charset=ISO-8859-1\">" >> $menuhtml
echo " <META HTTP-EQUIV=\"EXPIRES\" CONTENT=\"0\">" >> $menuhtml
echo " </HEAD>" >> $menuhtml
echo " <BODY>" >> $menuhtml
devices=`cat $smartmon_plot_file | sed -e 's/\/dev\///g' | cut -d: -f1 | sort`
for dev in $devices; do
# get device type
smartmon_device_type=`grep "$dev:" $smartmon_plot_file | cut -d: -f2`
# get port list
ports=`grep "$dev:" $smartmon_plot_file | cut -d: -f3`
# if ports not 0, then assume a RAID controller :-)
if [ "$ports" != "0" ]; then
portlist=`echo $ports | sed 's/,/ /g'`
for port in `echo $portlist`; do
# we need to compress this until 'plot_smart_$dev_$port.html' does not work
devport=''$dev'_'$port''
naming_html_files
echo " <a href=\"$smart_device_html\" target=\"home\">/dev/$dev, port $port</a><br>" >> $menuhtml
echo "  [Pre-failure Attributes]<br>" >> $menuhtml
tmpfile=$tmp_dir/plot_$devport.tmp
# write attribute table into temp file
/usr/sbin/smartctl -A -d $smartmon_device_type,$port /dev/$dev > $tmpfile
# add device model and serial (-A1!) number to temp file
/usr/sbin/smartctl -a -d $smartmon_device_type,$port /dev/$dev | grep -A1 "Device Model" >> $tmpfile
# collect pre-failure values, using 'uniq' to filter doubled attributes
prefail_values=`cat $tmpfile | grep 'Pre-fail' | grep -v 'Unknown_Attribute' | awk '{print $2}' | uniq -i`
# collect old-age values, using 'uniq' to filter doubled attributes
oldage_values=`cat $tmpfile | grep 'Old_age' | grep -v 'Unknown_Attribute' | awk '{print $2}' | uniq -i`
# check for the 'UPDATED' column
contains_updated=`cat $tmpfile | grep '^ID# ' | grep 'UPDATED'`
for j in $prefail_values; do
eval value_device=''$dev'_'$port'_Pre_Fail_$j'
naming_files
echo " <a name=\"$j\">$val</a> - [Pre-Failure] (SMART)<br>" >> $smart_device_html_abs
echo " <a href="$large_smart_png_file"><img border=\"0\" src=\"$smart_png_file\" alt=\"$smart_png_file\"></a><p><hr>" >> $smart_device_html_abs
echo " <a name=\"$j\">$val</a> - [Pre-Failure] (Raw)<br>" >> $raw_device_html_abs
echo " <a href="$large_raw_png_file"><img border=\"0\" src=\"$raw_png_file\" alt=\"$smart_png_file\"></a><p><hr>" >> $raw_device_html_abs
echo " <a href=\"$smart_device_html#$j\" target=\"home\">$val</a> (<a href=\"$raw_device_html#$j\" target=\"home\">Raw</a>)<br>" >> $menuhtml
if [ "`grep $smart_device_html $menuhtml`" = "" ]; then
echo " <a href=\"$smart_device_html\" target=\"home\">/dev/$dev, port $port</a><br>" >> $menuhtml
fi
write_all
done
echo " [Old-Age Attributes]<br>" >> $menuhtml
for j in $oldage_values; do
eval value_device=''$dev'_'$port'_Old_Age_$j'
naming_files
echo " <a name=\"$j\">$val</a> - [Old Age] (SMART)<br>" >> $smart_device_html_abs
echo " <a href="$large_smart_png_file"><img border=\"0\" src=\"$smart_png_file\" alt=\"$smart_png_file\"></a><p><hr>" >> $smart_device_html_abs
echo " <a name=\"$j\">$val</a> - [Old Age] (Raw)<br>" >> $raw_device_html_abs
echo " <a href="$large_raw_png_file"><img border=\"0\" src=\"$raw_png_file\" alt=\"$smart_png_file\"></a><p><hr>" >> $raw_device_html_abs
echo " <a href=\"$smart_device_html#$j\" target=\"home\">$val</a> (<a href=\"$raw_device_html#$j\" target=\"home\">Raw</a>)<br>" >> $menuhtml
if [ "`grep $smart_device_html $menuhtml`" = "" ]; then
echo " <a href=\"$smart_device_html\" target=\"home\">/dev/$dev, port $port</a><br>" >> $menuhtml
fi
write_all
done
done
else
devport=''$dev''
naming_html_files
# ports == "0" ==> (s|p)ata
echo " <a href=\"plot_smart_$dev.html\" target=\"home\">/dev/$dev</a><br>" >> $menuhtml
echo " [Pre-failure Attributes]<br>" >> $menuhtml
tmpfile=$tmp_dir/plot_$dev.tmp
/usr/sbin/smartctl -A -d $smartmon_device_type /dev/$dev > $tmpfile
/usr/sbin/smartctl -a -d $smartmon_device_type /dev/$dev | grep -A1 "Device Model" >> $tmpfile
# collect pre-failure values, using 'uniq' to filter doubled attributes
prefail_values=`cat $tmpfile | grep 'Pre-fail' | grep -v 'Unknown_Attribute' | awk '{print $2}' | uniq -i`
# collect old-age values, using 'uniq' to filter doubled attributes
oldage_values=`cat $tmpfile | grep 'Old_age' | grep -v 'Unknown_Attribute' | awk '{print $2}' | uniq -i`
# check for the 'UPDATED' column
contains_updated=`cat $tmpfile | grep '^ID# ' | grep 'UPDATED'`
for j in $prefail_values; do
eval value_device=''$dev'_Pre_Fail_$j'
naming_files
echo " <a name=\"$j\">$val</a> - [Pre-Failure] (SMART)<br>" >> $smart_device_html_abs
echo " <a href="$large_smart_png_file"><img border=\"0\" src=\"$smart_png_file\" alt=\"$smart_png_file\"></a><p><hr>" >> $smart_device_html_abs
echo " <a name=\"$j\">$val</a> - [Pre-Failure] (Raw)<br>" >> $raw_device_html_abs
echo " <a href="$large_raw_png_file"><img border=\"0\" src=\"$raw_png_file\" alt=\"$smart_png_file\"></a><p><hr>" >> $raw_device_html_abs
echo " <a href=\"$smart_device_html#$j\" target=\"home\">$val</a> (<a href=\"$raw_device_html#$j\" target=\"home\">Raw</a>)<br>" >> $menuhtml
if [ "`grep $smart_device_html $menuhtml`" = "" ]; then
echo " <a href=\"$smart_device_html\" target=\"home\"><strong>/dev/$dev</strong></a><br>" >> $menuhtml
fi
write_all
done
echo " [Old-Age Attributes]<br>" >> $menuhtml
for j in $oldage_values; do
eval value_device=''$dev'_Old_Age_$j'
naming_files
echo " <a name=\"$j\">$val</a> - [Old Age] (SMART)<br>" >> $smart_device_html_abs
echo " <a href="$large_smart_png_file"><img border=\"0\" src=\"$smart_png_file\" alt=\"$smart_png_file\"></a><p><hr>" >> $smart_device_html_abs
echo " <a name=\"$j\">$val</a> - [Old Age] (Raw)<br>" >> $raw_device_html_abs
echo " <a href="$large_raw_png_file"><img border=\"0\" src=\"$raw_png_file\" alt=\"$smart_png_file\"></a><p><hr>" >> $raw_device_html_abs
echo " <a href=\"$smart_device_html#$j\" target=\"home\">$val</a> (<a href=\"$raw_device_html#$j\" target=\"home\">Raw</a>)<br>" >> $menuhtml
if [ "`grep $smart_device_html $menuhtml`" = "" ]; then
echo " <a href=\"$smart_device_html\" target=\"home\" alt=\"/dev/$dev\">/dev/$dev</a><br>" >> $menuhtml
fi
write_all
done
fi
done
# close the files
echo " </BODY>" >> $smart_device_html_abs
echo "</HTML>" >> $smart_device_html_abs
echo " </BODY>" >> $raw_device_html_abs
echo "</HTML>" >> $raw_device_html_abs
# close the menu.html file
echo " </BODY>" >> $menuhtml
echo "</HTML>" >> $menuhtml
# do the job
cd $plot_dir/
for plotfile in '*.plot'; do
nice -n 19 /usr/sbin/gnuplot $plotfile 2>/dev/null
done
# remove temporary files
rm -f *.plot
rm -f $tmp_dir/plot_*
find $plot_dir -type d | xargs chmod 755
find $plot_dir -type f | xargs chmod 644
exit 0