Package Products :: Package ZenEvents :: Module SyslogProcessing
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenEvents.SyslogProcessing

  1  ########################################################################### 
  2  # 
  3  # This program is part of Zenoss Core, an open source monitoring platform. 
  4  # Copyright (C) 2007, Zenoss Inc. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify it 
  7  # under the terms of the GNU General Public License version 2 or (at your 
  8  # option) any later version as published by the Free Software Foundation. 
  9  # 
 10  # For complete information please visit: http://www.zenoss.com/oss/ 
 11  # 
 12  ########################################################################### 
 13   
 14  __doc__ = """SyslogProcessing 
 15  Class for turning syslog events into Zenoss Events 
 16  """ 
 17   
 18  import re 
 19  import logging 
 20  slog = logging.getLogger("zen.Syslog") 
 21  import socket 
 22   
 23  import Globals 
 24  from Products.ZenEvents.syslog_h import * 
 25  from Products.ZenUtils.IpUtil import isip 
 26   
 27   
 28  # Regular expressions that parse syslog tags from different sources 
 29  # A tuple can also be specified, in which case the second item in the 
 30  # tuple is a boolean which tells whether or not to keep the entry (default) 
 31  # or to discard the entry and not create an event. 
 32  parsers = ( 
 33  # generic mark 
 34  r"^(?P<summary>-- (?P<eventClassKey>MARK) --)", 
 35   
 36  # Cisco UCS 
 37  # : 2010 Oct 19 15:47:45 CDT: snmpd: SNMP Operation (GET) failed. Reason:2 reqId (257790979) errno (42) error index (1) 
 38  r'^: \d{4} \w{3}\s+\d{1,2}\s+\d{1,2}:\d\d:\d\d \w{3}: (?P<eventClassKey>[^:]+): (?P<summary>.*)', 
 39   
 40  # ntsyslog windows msg 
 41  r"^(?P<component>.+)\[(?P<ntseverity>\D+)\] (?P<ntevid>\d+) (?P<summary>.*)", 
 42   
 43  # cisco msg with card indicator 
 44  r"%CARD-\S+:(SLOT\d+) %(?P<eventClassKey>\S+): (?P<summary>.*)", 
 45   
 46  # cisco standard msg 
 47  r"%(?P<eventClassKey>(?P<component>\S+)-\d-\S+): *(?P<summary>.*)", 
 48   
 49  # Cisco ACS 
 50  r"^(?P<ipAddress>\S+)\s+(?P<summary>(?P<eventClassKey>CisACS_\d\d_\S+)\s+(?P<eventKey>\S+)\s.*)", 
 51   
 52  # netscreen device msg 
 53  r"device_id=\S+\s+\[\S+\](?P<eventClassKey>\S+\d+):\s+(?P<summary>.*)\s+\((?P<originalTime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\)", 
 54   
 55  # NetApp 
 56  # [deviceName: 10/100/1000/e1a:warning]: Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available]) 
 57  r"^\[[^:]+: (?P<component>[^:]+)[^\]]+\]: (?P<summary>.*)", 
 58   
 59  # unix syslog with pid 
 60  r"(?P<component>\S+)\[(?P<pid>\d+)\]:\s*(?P<summary>.*)", 
 61   
 62  # unix syslog without pid 
 63  r"(?P<component>\S+): (?P<summary>.*)", 
 64   
 65  # adtran devices 
 66  r"^(?P<deviceModel>[^\[]+)\[(?P<deviceManufacturer>ADTRAN)\]:(?P<component>[^\|]+\|\d+\|\d+)\|(?P<summary>.*)", 
 67   
 68  r"^date=.+ (?P<summary>devname=.+ log_id=(?P<eventClassKey>\d+) type=(?P<component>\S+).+)", 
 69   
 70  # proprietary message passing system 
 71  r"^(?P<component>\S+)(\.|\s)[A-Z]{3} \d \S+ \d\d:\d\d:\d\d-\d\d:\d\d:\d\d \d{5} \d{2} \d{5} \S+ \d{4} \d{3,5} (- )*(?P<summary>.*) \d{4} \d{4}", 
 72   
 73  # Cisco port state logging info 
 74  r"^Process (?P<process_id>\d+), Nbr (?P<device>\d+\.\d+\.\d+\.\d+) on (?P<interface>\w+/\d+) from (?P<start_state>\w+) to (?P<end_state>\w+), (?P<summary>.+)", 
 75   
 76  # Cisco VPN Concentrator 
 77  # 54884 05/25/2009 13:41:14.060 SEV=3 HTTP/42 RPT=4623 Error on socket accept. 
 78  r"^\d+ \d+\/\d+\/\d+ \d+:\d+:\d+\.\d+ SEV=\d+ (?P<eventClassKey>\S+) RPT=\d+ (?P<summary>.*)", 
 79   
 80  # Dell Storage Array 
 81  # 2626:48:VolExec:27-Aug-2009 13:15:58.072049:VE_VolSetWorker.hh:75:WARNING:43.3.2:Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB. 
 82  r'^\d+:\d+:(?P<component>[^:]+):\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)', 
 83   
 84  # 1-Oct-2009 23:00:00.383809:snapshotDelete.cc:290:INFO:8.2.5:Successfully deleted snapshot 'UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563'. 
 85  r'^\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)', 
 86  )  
 87   
 88  # compile regex parsers on load 
 89  compiledParsers = [] 
 90  for regex in parsers: 
 91      keepEntry = True 
 92      if isinstance(regex, tuple): 
 93          regex, keepEntry = regex 
 94      try: 
 95          compiled = re.compile(regex) 
 96          compiledParsers.append((compiled, keepEntry))  
 97      except: 
 98          pass 
 99   
100   
101 -class SyslogProcessor(object):
102 """ 103 Class to process syslog messages and convert them into events viewable 104 in the Zenoss event console. 105 """ 106
107 - def __init__(self,sendEvent,minpriority,parsehost,monitor,defaultPriority):
108 """ 109 Initializer 110 111 @param sendEvent: message from a remote host 112 @type sendEvent: string 113 @param minpriority: ignore anything under this priority 114 @type minpriority: integer 115 @param parsehost: hostname where this parser is running 116 @type parsehost: string 117 @param monitor: name of the distributed collector monitor 118 @type monitor: string 119 @param defaultPriority: priority to use if it can't be understood from the received packet 120 @type defaultPriority: integer 121 """ 122 self.minpriority = minpriority 123 self.parsehost = parsehost 124 self.sendEvent = sendEvent 125 self.monitor = monitor 126 self.defaultPriority = defaultPriority
127 128
129 - def process(self, msg, ipaddr, host, rtime):
130 """ 131 Process an event from syslog and convert to a Zenoss event 132 133 @param msg: message from a remote host 134 @type msg: string 135 @param ipaddr: IP address of the remote host 136 @type ipaddr: string 137 @param host: remote host's name 138 @type host: string 139 @param rtime: time as reported by the remote host 140 @type rtime: string 141 """ 142 evt = dict(device=host, 143 ipAddress=ipaddr, 144 firstTime=rtime, 145 lastTime=rtime, 146 eventGroup='syslog') 147 slog.debug("host=%s, ip=%s", host, ipaddr) 148 slog.debug(msg) 149 150 evt, msg = self.parsePRI(evt, msg) 151 if evt['priority'] > self.minpriority: return 152 153 evt, msg = self.parseHEADER(evt, msg) 154 evt = self.parseTag(evt, msg) 155 if evt: 156 #rest of msg now in summary of event 157 evt = self.buildEventClassKey(evt) 158 evt['monitor'] = self.monitor 159 self.sendEvent(evt)
160 161
162 - def parsePRI(self, evt, msg):
163 """ 164 Parse RFC-3164 PRI part of syslog message to get facility and priority. 165 166 @param evt: dictionary of event properties 167 @type evt: dictionary 168 @param msg: message from host 169 @type msg: string 170 @return: tuple of dictionary of event properties and the message 171 @type: (dictionary, string) 172 """ 173 pri = self.defaultPriority 174 fac = None 175 if msg[:1] == '<': 176 pos = msg.find('>') 177 fac, pri = LOG_UNPACK(int(msg[1:pos])) 178 msg = msg[pos+1:] 179 elif msg and msg[0] < ' ': 180 fac, pri = LOG_KERN, ord(msg[0]) 181 msg = msg[1:] 182 evt['facility'] = fac 183 evt['priority'] = pri 184 evt['severity'] = self.defaultSeverityMap(pri) 185 slog.debug("fac=%s pri=%s", fac, pri) 186 slog.debug("facility=%s severity=%s", evt['facility'], evt['severity']) 187 return evt, msg
188 189
190 - def defaultSeverityMap(self, pri):
191 """ 192 Default mapping from syslog priority to severity. 193 194 @param pri: syslog priority from host 195 @type pri: integer 196 @return: numeric severity 197 @type: integer 198 """ 199 sev = 1 200 if pri < 3: sev = 5 201 elif pri == 3: sev = 4 202 elif pri == 4: sev = 3 203 elif pri == 5 or pri == 6: sev = 2 204 return sev
205 206 207 timeParse = \ 208 re.compile("^(\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[\d ]{2}) (.*)").search 209 notHostSearch = re.compile("[\[:]").search
210 - def parseHEADER(self, evt, msg):
211 """ 212 Parse RFC-3164 HEADER part of syslog message. TIMESTAMP format is: 213 MMM HH:MM:SS and host is next token without the characters '[' or ':'. 214 215 @param evt: dictionary of event properties 216 @type evt: dictionary 217 @param msg: message from host 218 @type msg: string 219 @return: tuple of dictionary of event properties and the message 220 @type: (dictionary, string) 221 """ 222 slog.debug(msg) 223 m = re.sub("Kiwi_Syslog_Daemon \d+: \d+: " 224 "\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[^:]+: ", "", msg) 225 m = self.timeParse(msg) 226 if m: 227 slog.debug("parseHEADER timestamp=%s", m.group(1)) 228 evt['originalTime'] = m.group(1) 229 msg = m.group(2).strip() 230 msglist = msg.split() 231 if self.parsehost and not self.notHostSearch(msglist[0]): 232 device = msglist[0] 233 if device.find('@') >= 0: 234 device = device.split('@', 1)[1] 235 slog.debug("parseHEADER hostname=%s", evt['device']) 236 msg = " ".join(msglist[1:]) 237 evt['device'] = device 238 if isip(device): 239 evt['ipAddress'] = device 240 else: 241 if 'ipAddress' in evt: 242 del(evt['ipAddress']) 243 return evt, msg
244 245
246 - def parseTag(self, evt, msg):
247 """ 248 Parse the RFC-3164 tag of the syslog message using the regex defined 249 at the top of this module. 250 251 @param evt: dictionary of event properties 252 @type evt: dictionary 253 @param msg: message from host 254 @type msg: string 255 @return: dictionary of event properties 256 @type: dictionary 257 """ 258 slog.debug(msg) 259 for parser, keepEntry in compiledParsers: 260 slog.debug("tag regex: %s", parser.pattern) 261 m = parser.search(msg) 262 if not m: 263 continue 264 elif not keepEntry: 265 slog.debug("Dropping syslog message due to parser rule.") 266 return None 267 slog.debug("tag match: %s", m.groupdict()) 268 evt.update(m.groupdict()) 269 break 270 else: 271 slog.info("No matching parser: '%s'", msg) 272 evt['summary'] = msg 273 return evt
274 275
276 - def buildEventClassKey(self, evt):
277 """ 278 Build the key used to find an events dictionary record. If eventClass 279 is defined it is used. For NT events "Source_Evid" is used. For other 280 syslog events we use the summary of the event to perform a full text 281 or'ed search. 282 283 @param evt: dictionary of event properties 284 @type evt: dictionary 285 @return: dictionary of event properties 286 @type: dictionary 287 """ 288 if 'eventClassKey' in evt or 'eventClass' in evt: 289 return evt 290 elif 'ntevid' in evt: 291 evt['eventClassKey'] = "%s_%s" % (evt['component'],evt['ntevid']) 292 elif 'component' in evt: 293 evt['eventClassKey'] = evt['component'] 294 if 'eventClassKey' in evt: 295 slog.debug("eventClassKey=%s", evt['eventClassKey']) 296 try: 297 evt['eventClassKey'] = evt['eventClassKey'].decode('latin-1') 298 except: 299 evt['eventClassKey'] = evt['eventClassKey'].decode('utf-8') 300 else: 301 slog.debug("No eventClassKey assigned") 302 return evt
303