Syslog Scan Macro

The syslog_scan_alarms_macros.cfg is a script macro to scan syslog output for noteworthy entries.

syslog_scan(F, f, BYPASS1, LOGONLY, PAGE, BYPASS2)
 
	init
		status =piktstatus
		level =piktlevel
		task "Scan the system (f) log for noteworthy entries"
		// if the error log reports something like "variable
		// mismatch in line , this probably signifies
		// that the last line in the log file doesn't end with
		// linefeed; the solution here is to hand edit out that
		// last offending line
		input logfile "/var/log/(f)/current"
 
	begin
		if $alarm() !~~ "critical|kernel"
			=checkpoint(=lalim)
		fi
	rule	// automatic bypasses
		if    $inlin =~~ "==> /var/log/.+/current <=="
		   || $inlin =~~ "last (message|output) repeated"
		   || $inlin =~~ "metalog.+died with signal"
#if piktmaster
		   || $inlin =~~ "/diffing/|/staging/"
#endif
		   || $inlin =~ "^[[:space:]]*$"
			next
		endif

	rule	// special alarm-specific bypasses, first bypass
		if $inlin =~~ "(BYPASS1)"
			next
		endif

	rule	// make any desired substitutions here before assigning to $il
		set $il = $inlin

	rule	// reverse resolve any ipaddrs
		=resolveipaddr($il)

	rule	// log anything not bypassed
		if $il =~~ "(LOGONLY)"
			=output_alarm_log($il)
			next
		endif

	rule	// for flagged stuff, report and log (and possibly page)
		if (    $il =~~ "=redflags"
		     || $il =~~ "=yellowflags"
		   )
			output mail $il
			=output_alarm_log($il)
			// if $il =~~ "(PAGE)"
			// 	=page()
			// endif
			next
		endif

	rule	// report and log any root-related stuff
		if $alarm() =~~ "critical|kernel"
			if $il =~ "root"
				output mail $il
				=output_alarm_log($il)
				next
			endif
		// but for cron, just bypass root stuff
		elsif $alarm() =~~ "cron"
			if $il =~ "root"
				next
			endif
		endif

//#ifndef paranoid
//	rule	// bypass everything else
//		next
//#endifdef  // paranoid

	rule	// log anything not bypassed
		if $alarm() =~~ "critical|kernel"
			=output_alarm_log($il)
		endif

	rule	// special alarm-specific bypasses, second bypass
		if $il =~~ "(BYPASS2)"
			next
		endif

	rule	// report anything not bypassed
		if $alarm() =~~ "critical"
			output mail $il
		endif

	rule	// report anything not bypassed, if in verbose mode
		if $alarm() =~~ "kernel"
			=outputmail $il
		endif

	end
		quit

You might invoke the =syslog_scan() macro in your alarms.cfg file thusly:

///////////////////////////////////////////////////////////////////////////////
//
// logs_system_alarms.cfg
//
///////////////////////////////////////////////////////////////////////////////
 
SyslogCriticalScan
        =syslog_scan(Critical, critical, =nonesuch, =nonesuch, =nonesuch, =nonesuch)

///////////////////////////////////////////////////////////////////////////////

SyslogKernelScan

#if munich
        =syslog_scan(Kernel, kernel,
                     hub 2-1|hub_port_status failed|reset low speed USB device|
                     =syslogkernelbypasses, =nonesuch, =\nonesuch, =nonesuch)
#elsif codersys
        =syslog_scan(Kernel, kernel,
                     segfault|=syslogkernelbypasses, =nonesuch, =nonesuch, =nonesuch)
#else
        =syslog_scan(Kernel, kernel,
                     =syslogkernelbypasses, =nonesuch, =nonesuch, =nonesuch)
#endif

///////////////////////////////////////////////////////////////////////////////

SyslogCrondScan

        =syslog_scan(Crond, crond, download, =nonesuch, =nonesuch, =nonesuch)

///////////////////////////////////////////////////////////////////////////////

// this alarm is not particularly useful, as mainly it reports simple
// login failures by legitimate system owners (from flubbed password
// entry, which happens all the time)

SyslogPwdfailScan

        =syslog_scan(PwdFail, pwdfail, couperin|boyce|machaut|gibbons,
                     =nonesuch, =nonesuch, =nonesuch)

///////////////////////////////////////////////////////////////////////////////

SyslogSshdScan

        =syslog_scan(Sshd, sshd, =nonesuch, =nonesuch, =nonesuch, =nonesuch)

///////////////////////////////////////////////////////////////////////////////

 
///////////////////////////////////////////////////////////////////////////////

where '=syslogkernelbypasses' is a macro (defined in macros.cfg) of uninteresting syslog kernel output:

syslogkernelbypasses    [email protected]|root=/dev/|exception support|exception polling|
                        mounted root|obsolete setsockopt|assume root bridge|
                        reset.+speed usb|set_dentry_child_flags|
                        pcie_portdrv_probe->dev.+ has invalid irq|
                        write protect|too many iterations.+nv_nic_irq|
                        analog subsections not ready|
                        changing to secondary root

Output from this script might look like, for example:

CRITICAL:
    SyslogKernelScan
        Scan the system kernel log for noteworthy entries

        Apr  1 20:23:25 [kernel] task.prod[19726] general protection
                                 rip:80cb09f rsp:fff33780 error:2a
        Apr  1 20:26:02 [kernel] task.prod[19826] general protection
                                 rip:80cb09f rsp:ffb10b60 error:2a

WARNING:
    SyslogSshdScan
        Scan the system sshd log for noteworthy entries

        Apr  1 18:56:34 [sshd] channel 2: open failed:
                               administratively prohibited: open failed

Note how, on the munich & coder systems, we add special bypasses.

For more examples, see Samples.