1
2
3
4
5
6
7
8
9
10
11
12
13 __doc__ = """zenbatchload
14
15 zenbatchload loads a list of devices read from a file.
16 """
17
18 import sys
19 import re
20 from traceback import format_exc
21 import socket
22
23 import Globals
24 from ZODB.POSException import ConflictError
25 from ZODB.transact import transact
26 from zope.component import getUtility
27
28 from zExceptions import BadRequest
29
30 from Products.ZenModel.interfaces import IDeviceLoader
31 from Products.ZenUtils.ZCmdBase import ZCmdBase
32 from Products.ZenModel.Device import Device
33 from Products.ZenRelations.ZenPropertyManager import iszprop
34 from Products.ZenModel.ZenModelBase import iscustprop
35 from Products.ZenEvents.ZenEventClasses import Change_Add
36
37 from zenoss.protocols.protobufs.zep_pb2 import SEVERITY_INFO, SEVERITY_ERROR
41 """
42 Base class wrapping around dmd.DeviceLoader
43 """
44
45 sample_configs = """#
46 # Example zenbatchloader device file
47 #
48 # This file is formatted with one entry per line, like this:
49 #
50 # /Devices/device_class_name Python-expression
51 # hostname Python-expression
52 #
53 # For organizers (ie the /Devices path), the Python-expression
54 # is used to define defaults to be used for devices listed
55 # after the organizer. The defaults that can be specified are:
56 #
57 # * loader arguments (use the --show_options flag to show these)
58 # * zPropertie (from a device, use the More -> zProperties
59 # menu option to see the available ones.)
60 #
61 # NOTE: new zProperties *cannot* be created through this file
62 #
63 # The Python-expression is used to create a dictionary of settings.
64 # device_settings = eval( 'dict(' + python-expression + ')' )
65 #
66
67
68 # If no organizer is specified at the beginning of the file,
69 # defaults to the /Devices/Discovered device class.
70 device0 comments="A simple device"
71 # All settings must be seperated by a comma.
72 device1 comments="A simple device", zSnmpCommunity='blue', zSnmpVer='v1'
73
74 # Notes for this file:
75 # * Oraganizer names *must* start with '/'
76 #
77 /Devices/Server/Linux zSnmpPort=1543
78 # Python strings can use either ' or " -- there's no difference.
79 # As a special case, it is also possible to specify the IP address
80 linux_device1 setManageIp='10.10.10.77', zSnmpCommunity='blue', zSnmpVer="v2c"
81 # A '\' at the end of the line allows you to place more
82 # expressions on a new line. Don't forget the comma...
83 linux_device2 zLinks="<a href='http://example.org'>Support site</a>", \
84 zTelnetEnable=True, \
85 zTelnetPromptTimeout=15.3
86
87 # A new organizer drops all previous settings, and allows
88 # for new ones to be used. Settings do not span files.
89 /Devices/Server/Windows zWinUser="administrator", zWinPassword='fred'
90 # Bind templates
91 windows_device1 zDeviceTemplates=[ 'Device', 'myTemplate' ]
92 # Override the default from the organizer setting.
93 windows_device2 zWinUser="administrator", zWinPassword='thomas'
94
95 # Apply other settings to the device
96 settingsDevice setManageIp='10.10.10.77', setLocation="123 Elm Street", \
97 setSystems=['/mySystems'], setPerformanceMonitor='remoteCollector1', \
98 setHWSerialNumber="abc123456789", setGroups=['/myGroup'], \
99 setHWProduct=('myproductName','manufacturer'), setOSProduct=('OS Name','manufacturer')
100
101 # If the device or device class contains a space, then it must be quoted (either ' or ")
102 "/Server/Windows/WMI/Active Directory/2008"
103
104 # Now, what if we have a device that isn't really a device, and requires
105 # a special loader?
106 # The 'loader' setting requires a registered utility, and 'loader_arg_keys' is
107 # a list from which any other settings will be passed into the loader callable.
108 #
109 # Here is a commmented-out example of how a VMware endpoint might be added:
110 #
111 #/Devices/VMware loader='vmware', loader_arg_keys=['host', 'username', 'password', 'useSsl', 'id']
112 #esxwin2 id='esxwin2', host='esxwin2.zenoss.loc', username='testuser', password='password', useSsl=True
113
114 # Apply custom schema properties (c-properties) to a device
115 windows_device7 cDateTest='2010/02/28'
116
117 """
118
120 ZCmdBase.__init__(self, *args, **kwargs)
121 self.defaults = {}
122
123 self.loader = self.dmd.DeviceLoader.loadDevice
124
125 self.fqdn = socket.getfqdn()
126 self.baseEvent = dict(
127 device=self.fqdn,
128 component='',
129 agent='zenbatchload',
130 monitor='localhost',
131 manager=self.fqdn,
132 severity=SEVERITY_ERROR,
133
134
135
136 eventClass=Change_Add,
137 )
138
139
140 self.loader_args = dict.fromkeys( self.loader.func_code.co_varnames )
141 unsupportable_args = [
142 'REQUEST', 'device', 'self', 'xmlrpc', 'e', 'handler',
143 ]
144 for opt in unsupportable_args:
145 if opt in self.loader_args:
146 del self.loader_args[opt]
147
149 """
150 Read through all of the files listed as arguments and
151 return a list of device entries.
152
153 @parameter args: list of filenames (uses self.args is this is None)
154 @type args: list of strings
155 @return: list of device specifications
156 @rtype: list of dictionaries
157 """
158 if args is None:
159 args = self.args
160
161 device_list = []
162 for filename in args:
163 try:
164 data = open(filename,'r').readlines()
165 except IOError:
166 msg = "Unable to open the file '%s'" % filename
167 self.reportException(msg)
168 continue
169
170 temp_dev_list = self.parseDevices(data)
171 if temp_dev_list:
172 device_list += temp_dev_list
173
174 return device_list
175
177 """
178 Apply zProperty settings (if any) to the device.
179
180 @parameter device: device to modify
181 @type device: DMD device object
182 @parameter device_specs: device creation dictionary
183 @type device_specs: dictionary
184 """
185 self.log.debug( "Applying zProperties..." )
186
187
188 dev_zprops = dict( device.zenPropertyItems() )
189
190 for zprop, value in device_specs.items():
191 if not iszprop(zprop):
192 continue
193
194 if zprop in dev_zprops:
195 try:
196 device.setZenProperty(zprop, value)
197 except BadRequest:
198 self.log.warn( "Object %s zproperty %s is invalid or duplicate" % (
199 device.titleOrId(), zprop) )
200 else:
201 self.log.warn( "The zproperty %s doesn't exist in %s" % (
202 zprop, device_specs.get('deviceName', device.id)))
203
205 """
206 Custom schema properties
207 """
208 self.log.debug( "Applying custom schema properties..." )
209 dev_cprops = device.custPropertyIds()
210
211 for cprop, value in device_specs.items():
212 if not iscustprop(cprop):
213 continue
214
215 if cprop in dev_cprops:
216 setattr(device, cprop, value)
217 else:
218 self.log.warn( "The cproperty %s doesn't exist in %s" % (
219 cprop, device_specs.get('deviceName', device.id)))
220
222 location = device_specs.get('setLocation')
223 if location:
224 self.addLGSOrganizer('Locations', (location,) )
225
226 systems = device_specs.get('setSystems')
227 if systems:
228 if not isinstance(systems, list) and not isinstance(systems, tuple):
229 systems = (systems,)
230 self.addLGSOrganizer('Systems', systems)
231
232 groups = device_specs.get('setGroups')
233 if groups:
234 if not isinstance(groups, list) and not isinstance(groups, tuple):
235 groups = (groups,)
236 self.addLGSOrganizer('Groups', groups)
237
255
257 """
258 Add any organizers as required, and apply zproperties to them.
259 """
260 path = device_specs.get('devicePath')
261 baseOrg = path.split('/', 2)[1]
262 base = getattr(self.dmd, baseOrg, None)
263 if base is None:
264 self.log.error("The base of path %s (%s) does not exist -- skipping",
265 baseOrg, path)
266 return
267
268 try:
269 org = base.getDmdObj(path)
270 except KeyError:
271 self.log.info("Creating organizer %s", path)
272 @transact
273 def inner():
274 base.manage_addOrganizer(path)
275 inner()
276 org = base.getDmdObj(path)
277 self.applyZProps(org, device_specs)
278 self.applyOtherProps(org, device_specs)
279
281 """
282 Apply non-zProperty settings (if any) to the device.
283
284 @parameter device: device to modify
285 @type device: DMD device object
286 @parameter device_specs: device creation dictionary
287 @type device_specs: dictionary
288 """
289 self.log.debug( "Applying other properties..." )
290 internalVars = [
291 'deviceName', 'devicePath', 'comments', 'loader', 'loader_arg_keys',
292 ]
293 @transact
294 def setDescription(org, description):
295 setattr(org, 'description', description)
296
297 for functor, value in device_specs.items():
298 if iszprop(functor) or iscustprop(functor) or functor in internalVars:
299 continue
300
301
302 if functor == 'description':
303 if hasattr(device, functor):
304 setDescription(device, value)
305 continue
306
307 try:
308 self.log.debug("For %s, calling device.%s(%s)",
309 device.id, functor, value)
310 func = getattr(device, functor, None)
311 if func is None:
312 self.log.warn("The function '%s' for device %s is not found.",
313 functor, device.id)
314 elif isinstance(value, (list, tuple)):
315
316 try:
317 func(*value)
318 except TypeError:
319 func(value)
320 else:
321 func(value)
322 except ConflictError:
323 raise
324 except Exception:
325 msg = "Device %s device.%s(%s) failed" % (device.id, functor, value)
326 self.reportException(msg, device.id)
327
329 """
330 It's up to the loader now to figure out what's going on.
331
332 @parameter loader: device loader
333 @type loader: callable
334 @parameter device_specs: device entries
335 @type device_specs: dictionary
336 """
337 argKeys = device_specs.get('loader_arg_keys', [])
338 loader_args = {}
339 for key in argKeys:
340 if key in device_specs:
341 loader_args[key] = device_specs[key]
342
343 result = loader().load_device(self.dmd, **loader_args)
344
345
346
347 if isinstance(result, Device):
348 return result
349 return None
350
352 """
353 Read the input and process the devices
354 * create the device entry
355 * set zproperties
356 * set custom schema properties
357 * model the device
358
359 @parameter device_list: list of device entries
360 @type device_list: list of dictionaries
361 """
362
363 def transactional(f):
364 return f if self.options.nocommit else transact(f)
365
366 processed = {'total':0, 'errors':0}
367
368 @transactional
369 def _process(device_specs):
370
371 self.dmd.zport._p_jar.sync()
372
373 loaderName = device_specs.get('loader')
374 if loaderName is not None:
375 try:
376 orgName = device_specs['devicePath']
377 organizer = self.dmd.getObjByPath('dmd' + orgName)
378 deviceLoader = getUtility(IDeviceLoader, loaderName, organizer)
379 devobj = self.runLoader(deviceLoader, device_specs)
380 except ConflictError:
381 raise
382 except Exception:
383 devName = device_specs.get('device_specs', 'Unkown Device')
384 msg = "Ignoring device loader issue for %s" % devName
385 self.reportException(msg, devName, {'specs':str(device_specs)})
386 processed['errors'] += 1
387 return
388 else:
389 devobj = self.getDevice(device_specs)
390 deviceLoader = None
391
392 if devobj is None:
393 if deviceLoader is not None:
394 processed['total'] += 1
395 else:
396 self.addAllLGSOrganizers(device_specs)
397 self.applyZProps(devobj, device_specs)
398 self.applyCustProps(devobj, device_specs)
399 self.applyOtherProps(devobj, device_specs)
400
401 return devobj
402
403 @transactional
404 def _snmp_community(device_specs, devobj):
405
406 if 'zSnmpCommunity' not in device_specs:
407 self.log.debug('Discovering SNMP version and community')
408 devobj.manage_snmpCommunity()
409
410 @transactional
411 def _model(devobj):
412 try:
413 devobj.collectDevice(setlog=self.options.showModelOutput)
414 except ConflictError:
415 raise
416 except Exception, ex:
417 msg = "Modeling error for %s" % devobj.id
418 self.reportException(msg, devobj.id, ex)
419 processed['errors'] += 1
420 processed['total'] += 1
421
422 for device_specs in device_list:
423 devobj = _process(device_specs)
424
425
426
427 if devobj and not self.options.nocommit and not self.options.nomodel:
428 _snmp_community(device_specs, devobj)
429 _model(devobj)
430
431 self.reportResults(processed, len(device_list))
432
434 """
435 Report exceptions back to the the event console
436 """
437 self.log.exception(msg)
438 if not self.options.nocommit:
439 evt = self.baseEvent.copy()
440 evt.update(dict(
441 summary=msg,
442 traceback=format_exc()
443 ))
444 evt.update(kwargs)
445 if devName:
446 evt['device'] = devName
447 self.dmd.ZenEventManager.sendEvent(evt)
448
450 """
451 Report the success + total counts from loading devices.
452 """
453 msg = "Modeled %d of %d devices, with %d errors" % (
454 processed['total'], totalDevices, processed['errors'] )
455 self.log.info(msg)
456
457 if not self.options.nocommit:
458 evt = self.baseEvent.copy()
459 evt.update(dict(
460 severity=SEVERITY_INFO,
461 summary=msg,
462 modeled=processed['total'],
463 errors=processed['errors'],
464 total=totalDevices,
465 ))
466 self.dmd.ZenEventManager.sendEvent(evt)
467
469 """
470 Report that we added a new device.
471 """
472 if not self.options.nocommit:
473 evt = self.baseEvent.copy()
474 evt.update(dict(
475 severity=SEVERITY_INFO,
476 summary= "Added new device %s" % deviceName
477 ))
478 self.dmd.ZenEventManager.sendEvent(evt)
479
481 """
482 Find or create the specified device
483
484 @parameter device_specs: device creation dictionary
485 @type device_specs: dictionary
486 @return: device or None
487 @rtype: DMD device object
488 """
489 if 'deviceName' not in device_specs:
490 return None
491 name = device_specs['deviceName']
492 devobj = self.dmd.Devices.findDevice(name)
493 if devobj is not None:
494 self.log.info("Found existing device %s" % name)
495 return devobj
496
497 specs = {}
498 for key in self.loader_args:
499 if key in device_specs:
500 specs[key] = device_specs[key]
501
502 try:
503 self.log.info("Creating device %s" % name)
504
505
506 specs['discoverProto'] = 'none'
507
508 self.loader(**specs)
509 devobj = self.dmd.Devices.findDevice(name)
510 if devobj is None:
511 self.log.error("Unable to find newly created device %s -- skipping" \
512 % name)
513 else:
514 self.notifyNewDeviceCreated(name)
515
516 except Exception:
517 msg = "Unable to load %s -- skipping" % name
518 self.reportException(msg, name)
519
520 return devobj
521
523 """
524 Add our command-line options to the basics
525 """
526 ZCmdBase.buildOptions(self)
527
528 self.parser.add_option('--show_options',
529 dest="show_options", default=False,
530 action="store_true",
531 help="Show the various options understood by the loader")
532
533 self.parser.add_option('--sample_configs',
534 dest="sample_configs", default=False,
535 action="store_true",
536 help="Show an example configuration file.")
537
538 self.parser.add_option('--showModelOutput',
539 dest="showModelOutput", default=True,
540 action="store_false",
541 help="Show modelling activity")
542
543 self.parser.add_option('--nocommit',
544 dest="nocommit", default=False,
545 action="store_true",
546 help="Don't commit changes to the ZODB. Use for verifying config file.")
547
548 self.parser.add_option('--nomodel',
549 dest="nomodel", default=False,
550 action="store_true",
551 help="Don't model the remote devices. Must be able to commit changes.")
552
554 """
555 From the list of strings in rawDevices, construct a list
556 of device dictionaries, ready to load into Zenoss.
557
558 @parameter data: list of strings representing device entries
559 @type data: list of strings
560 @return: list of parsed device entries
561 @rtype: list of dictionaries
562 """
563 if not data:
564 return []
565
566 comment = re.compile(r'^\s*#.*')
567
568 defaults = {'devicePath':"/Discovered" }
569 finalList = []
570 i = 0
571 while i < len(data):
572 line = data[i]
573 line = re.sub(comment, '', line).strip()
574 if line == '':
575 i += 1
576 continue
577
578
579 while line[-1] == '\\' and i < len(data):
580 i += 1
581 line = line[:-1] + data[i]
582 line = re.sub(comment, '', line).strip()
583
584 if line[0] == '/' or line[1] == '/':
585 defaults = self.parseDeviceEntry(line, {})
586 if defaults is None:
587 defaults = {'devicePath':"/Discovered" }
588 else:
589 defaults['devicePath'] = defaults['deviceName']
590 del defaults['deviceName']
591 self.addOrganizer(defaults)
592
593 else:
594 configs = self.parseDeviceEntry(line, defaults)
595 if configs:
596 finalList.append(configs)
597 i += 1
598
599 return finalList
600
601 - def parseDeviceEntry(self, line, defaults):
602 """
603 Build a dictionary of properties from one line's input
604
605 @parameter line: string containing one device's info
606 @type line: string
607 @parameter defaults: dictionary of default settings
608 @type defaults: dictionary
609 @return: parsed device entry
610 @rtype: dictionary
611 """
612 options = []
613
614 if line[0] in ["'", '"']:
615 delim = line[0]
616 eom = line.find(delim, 1)
617 if eom == -1:
618 self.log.error("While reading name, unable to parse" \
619 " the entry for %s -- skipping", line )
620 return None
621 name = line[1:eom]
622 options = line[eom+1:]
623
624 else:
625 options = line.split(None, 1)
626 name = options.pop(0)
627 if options:
628 options = options.pop(0)
629
630 configs = defaults.copy()
631 configs['deviceName'] = name
632
633 if options:
634 try:
635
636 evalString = 'dict(' + options + '\n)'
637 configs.update(eval(evalString))
638 except:
639 self.log.error( "Unable to parse the entry for %s -- skipping" % name )
640 self.log.error( "Raw string: %s" % options )
641 return None
642
643 return configs
644
645
646 if __name__=='__main__':
647 batchLoader = BatchDeviceLoader()
648
649 if batchLoader.options.show_options:
650 print "Options = %s" % sorted( batchLoader.loader_args.keys() )
651 help(batchLoader.loader)
652 sys.exit(0)
653
654 if batchLoader.options.sample_configs:
655 print batchLoader.sample_configs
656 sys.exit(0)
657
658 device_list = batchLoader.loadDeviceList()
659 if not device_list:
660 batchLoader.log.warn("No device entries found to load.")
661 sys.exit(1)
662
663 batchLoader.processDevices(device_list)
664 sys.exit(0)
665