Service Downage Macro

The service_downage_alarms_macros.cfg is a script macro for reporting when services go down (or, optionally, come back up) on remote systems.

service_downage(L, S, I, T, A, M)
 
	init
		status =piktstatus
		level (L)
		task "Report (S) service downages on remote systems"
		input proc "(I)"
		dat $host 1
		keys $host
	begin
		set $missioncritical = $command("=piktc -L +H missioncritical | =oneline")
#ifdef debug
		output $missioncritical
#endifdef

	rule
		set $dnmsg = "$host's (S) services are down ((M))"
		set $upmsg = "$host's (S) services are back up"

	rule
		if $alert() =~~ "red|emergency|server"
			if " $missioncritical " !~ " $host "
				set $state = %state
				next
			fi
		fi

	rule
		if $alert() =~~ "client"
			if " $missioncritical " =~ " $host "
				set $state = %state
				next
			fi
		fi

#ifdef debug
	rule
		output $host
#elsedef
	rule
		if $alert() =~ "(A)"
			output $host
			output $newline()
		fi
#endifdef

	rule	// initially, assume the service is up
		set $state = "+"

	rule
		=bypass_server_reboots

	rule
		if (T)
			if $alert() =~ "(A)"
				output $dnmsg
				output =newline
			else
				set $state = "-"
				// for all systems, always report new downages
				if $state ne %state
					output mail $dnmsg
				// but for missioncritical systems, report
				// continuing downages only periodically
				elsif " $missioncritical " =~ " $host "
					=every_four_hours(output mail $dnmsg, )
				fi
			fi
			next
		fi

	rule	// for missioncritical systems, if state was "-",
		// is now "+", then report change
		if    " $missioncritical " =~ " $host "
		   && $state ne %state
			output mail $upmsg
		fi

	end
		quit

You might invoke the =service_downage() macro in your alarms.cfg file thusly:

///////////////////////////////////////////////////////////////////////////////
//
// network_alarms.cfg
//
///////////////////////////////////////////////////////////////////////////////
 
#if piktmaster
 
RpcDown
	=service_downage(warning, RPC, =piktc -L +H pikt -H down sick, =rpcfail($host),
                         DownRpc|DownRpcServers|DownRpcClients, rpcinfo -p failure)

#endif

///////////////////////////////////////////////////////////////////////////////

#if piktmaster

SshDown

	=service_downage(warning, SSH, =piktc -L +H pikt -H down sick, =sshfail($host),
                         DownSsh, telnet to port 22 failure)

#endif

///////////////////////////////////////////////////////////////////////////////

#if piktmaster

SmtpDown

	=service_downage(warning, SMTP, =piktc -L +H pikt -H nosmtp down sick, =smtpfail($host),
                         DownSmtp, telnet to port 25 failure)

#endif

///////////////////////////////////////////////////////////////////////////////

#if piktmaster

HttpDown

	=service_downage(info, HTTP, =piktc -L +H webserver -H down sick, =httpfail($host),
                         DownHttp, telnet to port 80 failure)

#endif
 
///////////////////////////////////////////////////////////////////////////////

where 'down' is a host group of known down systems (specified in down_systems.cfg) and 'sick' is a host group of known "sick" systems (systems up and running but somehow impaired).

Output from this script might look like, for example:

URGENT:
    RpcDownUrgent
        Report RPC downages on remote systems

        helsinki's RPC services are down (rpcinfo -p failure)

URGENT:
    SshDownUrgent
        Report ssh service downages on remote systems

        helsinki's SSH services are down (telnet to port 22 failure)
        rouen's SSH services are back up

For more examples, see Samples.