Package Products :: Package ZenModel :: Module BatchDeviceLoader
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenModel.BatchDeviceLoader

  1  ########################################################################### 
  2  # 
  3  # This program is part of Zenoss Core, an open source monitoring platform. 
  4  # Copyright (C) 2009, 2011 Zenoss Inc. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify it 
  7  # under the terms of the GNU General Public License version 2 or (at your 
  8  # option) any later version as published by the Free Software Foundation. 
  9  # 
 10  # For complete information please visit: http://www.zenoss.com/oss/ 
 11  # 
 12  ########################################################################### 
 13  __doc__ = """zenbatchload 
 14   
 15  zenbatchload loads a list of devices read from a file. 
 16  """ 
 17   
 18  import sys 
 19  import re 
 20  from traceback import format_exc 
 21  import socket 
 22   
 23  import Globals 
 24  from ZODB.POSException import ConflictError 
 25  from ZODB.transact import transact 
 26  from zope.component import getUtility 
 27   
 28  from zExceptions import BadRequest 
 29   
 30  from Products.ZenModel.interfaces import IDeviceLoader 
 31  from Products.ZenUtils.ZCmdBase import ZCmdBase 
 32  from Products.ZenModel.Device import Device 
 33  from Products.ZenRelations.ZenPropertyManager import iszprop 
 34  from Products.ZenModel.ZenModelBase import iscustprop 
 35  from Products.ZenEvents.ZenEventClasses import Change_Add 
 36   
 37  from zenoss.protocols.protobufs.zep_pb2 import SEVERITY_INFO, SEVERITY_ERROR 
38 39 40 -class BatchDeviceLoader(ZCmdBase):
41 """ 42 Base class wrapping around dmd.DeviceLoader 43 """ 44 45 sample_configs = """# 46 # Example zenbatchloader device file 47 # 48 # This file is formatted with one entry per line, like this: 49 # 50 # /Devices/device_class_name Python-expression 51 # hostname Python-expression 52 # 53 # For organizers (ie the /Devices path), the Python-expression 54 # is used to define defaults to be used for devices listed 55 # after the organizer. The defaults that can be specified are: 56 # 57 # * loader arguments (use the --show_options flag to show these) 58 # * zPropertie (from a device, use the More -> zProperties 59 # menu option to see the available ones.) 60 # 61 # NOTE: new zProperties *cannot* be created through this file 62 # 63 # The Python-expression is used to create a dictionary of settings. 64 # device_settings = eval( 'dict(' + python-expression + ')' ) 65 # 66 67 68 # If no organizer is specified at the beginning of the file, 69 # defaults to the /Devices/Discovered device class. 70 device0 comments="A simple device" 71 # All settings must be seperated by a comma. 72 device1 comments="A simple device", zSnmpCommunity='blue', zSnmpVer='v1' 73 74 # Notes for this file: 75 # * Oraganizer names *must* start with '/' 76 # 77 /Devices/Server/Linux zSnmpPort=1543 78 # Python strings can use either ' or " -- there's no difference. 79 # As a special case, it is also possible to specify the IP address 80 linux_device1 setManageIp='10.10.10.77', zSnmpCommunity='blue', zSnmpVer="v2c" 81 # A '\' at the end of the line allows you to place more 82 # expressions on a new line. Don't forget the comma... 83 linux_device2 zLinks="<a href='http://example.org'>Support site</a>", \ 84 zTelnetEnable=True, \ 85 zTelnetPromptTimeout=15.3 86 87 # A new organizer drops all previous settings, and allows 88 # for new ones to be used. Settings do not span files. 89 /Devices/Server/Windows zWinUser="administrator", zWinPassword='fred' 90 # Bind templates 91 windows_device1 zDeviceTemplates=[ 'Device', 'myTemplate' ] 92 # Override the default from the organizer setting. 93 windows_device2 zWinUser="administrator", zWinPassword='thomas' 94 95 # Apply other settings to the device 96 settingsDevice setManageIp='10.10.10.77', setLocation="123 Elm Street", \ 97 setSystems=['/mySystems'], setPerformanceMonitor='remoteCollector1', \ 98 setHWSerialNumber="abc123456789", setGroups=['/myGroup'], \ 99 setHWProduct=('myproductName','manufacturer'), setOSProduct=('OS Name','manufacturer') 100 101 # If the device or device class contains a space, then it must be quoted (either ' or ") 102 "/Server/Windows/WMI/Active Directory/2008" 103 104 # Now, what if we have a device that isn't really a device, and requires 105 # a special loader? 106 # The 'loader' setting requires a registered utility, and 'loader_arg_keys' is 107 # a list from which any other settings will be passed into the loader callable. 108 # 109 # Here is a commmented-out example of how a VMware endpoint might be added: 110 # 111 #/Devices/VMware loader='vmware', loader_arg_keys=['host', 'username', 'password', 'useSsl', 'id'] 112 #esxwin2 id='esxwin2', host='esxwin2.zenoss.loc', username='testuser', password='password', useSsl=True 113 114 # Apply custom schema properties (c-properties) to a device 115 windows_device7 cDateTest='2010/02/28' 116 117 """ 118
119 - def __init__(self, *args, **kwargs):
120 ZCmdBase.__init__(self, *args, **kwargs) 121 self.defaults = {} 122 123 self.loader = self.dmd.DeviceLoader.loadDevice 124 125 self.fqdn = socket.getfqdn() 126 self.baseEvent = dict( 127 device=self.fqdn, 128 component='', 129 agent='zenbatchload', 130 monitor='localhost', 131 manager=self.fqdn, 132 severity=SEVERITY_ERROR, 133 # Note: Change_Add is probably a better event class, but these 134 # events get sent to history by the default Zen property stuff 135 # on the event class. zendisc uses Status_Snmp, so so will we. 136 eventClass=Change_Add, 137 ) 138 139 # Create the list of options we want people to know about 140 self.loader_args = dict.fromkeys( self.loader.func_code.co_varnames ) 141 unsupportable_args = [ 142 'REQUEST', 'device', 'self', 'xmlrpc', 'e', 'handler', 143 ] 144 for opt in unsupportable_args: 145 if opt in self.loader_args: 146 del self.loader_args[opt]
147
148 - def loadDeviceList(self, args=None):
149 """ 150 Read through all of the files listed as arguments and 151 return a list of device entries. 152 153 @parameter args: list of filenames (uses self.args is this is None) 154 @type args: list of strings 155 @return: list of device specifications 156 @rtype: list of dictionaries 157 """ 158 if args is None: 159 args = self.args 160 161 device_list = [] 162 for filename in args: 163 try: 164 data = open(filename,'r').readlines() 165 except IOError: 166 msg = "Unable to open the file '%s'" % filename 167 self.reportException(msg) 168 continue 169 170 temp_dev_list = self.parseDevices(data) 171 if temp_dev_list: 172 device_list += temp_dev_list 173 174 return device_list
175
176 - def applyZProps(self, device, device_specs):
177 """ 178 Apply zProperty settings (if any) to the device. 179 180 @parameter device: device to modify 181 @type device: DMD device object 182 @parameter device_specs: device creation dictionary 183 @type device_specs: dictionary 184 """ 185 self.log.debug( "Applying zProperties..." ) 186 # Returns a list of (key, value) pairs. 187 # Convert it to a dictionary. 188 dev_zprops = dict( device.zenPropertyItems() ) 189 190 for zprop, value in device_specs.items(): 191 if not iszprop(zprop): 192 continue 193 194 if zprop in dev_zprops: 195 try: 196 device.setZenProperty(zprop, value) 197 except BadRequest: 198 self.log.warn( "Object %s zproperty %s is invalid or duplicate" % ( 199 device.titleOrId(), zprop) ) 200 else: 201 self.log.warn( "The zproperty %s doesn't exist in %s" % ( 202 zprop, device_specs.get('deviceName', device.id)))
203
204 - def applyCustProps(self, device, device_specs):
205 """ 206 Custom schema properties 207 """ 208 self.log.debug( "Applying custom schema properties..." ) 209 dev_cprops = device.custPropertyIds() 210 211 for cprop, value in device_specs.items(): 212 if not iscustprop(cprop): 213 continue 214 215 if cprop in dev_cprops: 216 setattr(device, cprop, value) 217 else: 218 self.log.warn( "The cproperty %s doesn't exist in %s" % ( 219 cprop, device_specs.get('deviceName', device.id)))
220
221 - def addAllLGSOrganizers(self, device_specs):
222 location = device_specs.get('setLocation') 223 if location: 224 self.addLGSOrganizer('Locations', (location,) ) 225 226 systems = device_specs.get('setSystems') 227 if systems: 228 if not isinstance(systems, list) and not isinstance(systems, tuple): 229 systems = (systems,) 230 self.addLGSOrganizer('Systems', systems) 231 232 groups = device_specs.get('setGroups') 233 if groups: 234 if not isinstance(groups, list) and not isinstance(groups, tuple): 235 groups = (groups,) 236 self.addLGSOrganizer('Groups', groups)
237
238 - def addLGSOrganizer(self, lgsType, paths=[]):
239 """ 240 Add any new locations, groups or organizers 241 """ 242 prefix = '/zport/dmd/' + lgsType 243 base = getattr(self.dmd, lgsType) 244 if hasattr(base, 'sync'): 245 base.sync() 246 existing = [x.getPrimaryUrlPath().replace(prefix, '') \ 247 for x in base.getSubOrganizers()] 248 for path in paths: 249 if path in existing: 250 continue 251 try: 252 base.manage_addOrganizer(path) 253 except BadRequest: 254 pass
255
256 - def addOrganizer(self, device_specs):
257 """ 258 Add any organizers as required, and apply zproperties to them. 259 """ 260 path = device_specs.get('devicePath') 261 baseOrg = path.split('/', 2)[1] 262 base = getattr(self.dmd, baseOrg, None) 263 if base is None: 264 self.log.error("The base of path %s (%s) does not exist -- skipping", 265 baseOrg, path) 266 return 267 268 try: 269 org = base.getDmdObj(path) 270 except KeyError: 271 self.log.info("Creating organizer %s", path) 272 @transact 273 def inner(): 274 base.manage_addOrganizer(path)
275 inner() 276 org = base.getDmdObj(path) 277 self.applyZProps(org, device_specs) 278 self.applyOtherProps(org, device_specs)
279
280 - def applyOtherProps(self, device, device_specs):
281 """ 282 Apply non-zProperty settings (if any) to the device. 283 284 @parameter device: device to modify 285 @type device: DMD device object 286 @parameter device_specs: device creation dictionary 287 @type device_specs: dictionary 288 """ 289 self.log.debug( "Applying other properties..." ) 290 internalVars = [ 291 'deviceName', 'devicePath', 'comments', 'loader', 'loader_arg_keys', 292 ] 293 @transact 294 def setDescription(org, description): 295 setattr(org, 'description', description)
296 297 for functor, value in device_specs.items(): 298 if iszprop(functor) or iscustprop(functor) or functor in internalVars: 299 continue 300 301 # Special case for organizers which can take a description 302 if functor == 'description': 303 if hasattr(device, functor): 304 setDescription(device, value) 305 continue 306 307 try: 308 self.log.debug("For %s, calling device.%s(%s)", 309 device.id, functor, value) 310 func = getattr(device, functor, None) 311 if func is None: 312 self.log.warn("The function '%s' for device %s is not found.", 313 functor, device.id) 314 elif isinstance(value, (list, tuple)): 315 # The function either expects a list or arguments 316 try: # arguments 317 func(*value) 318 except TypeError: # Try as a list 319 func(value) 320 else: 321 func(value) 322 except ConflictError: 323 raise 324 except Exception: 325 msg = "Device %s device.%s(%s) failed" % (device.id, functor, value) 326 self.reportException(msg, device.id) 327
328 - def runLoader(self, loader, device_specs):
329 """ 330 It's up to the loader now to figure out what's going on. 331 332 @parameter loader: device loader 333 @type loader: callable 334 @parameter device_specs: device entries 335 @type device_specs: dictionary 336 """ 337 argKeys = device_specs.get('loader_arg_keys', []) 338 loader_args = {} 339 for key in argKeys: 340 if key in device_specs: 341 loader_args[key] = device_specs[key] 342 343 result = loader().load_device(self.dmd, **loader_args) 344 345 # If the loader returns back a device object, carry 346 # on processing 347 if isinstance(result, Device): 348 return result 349 return None
350
351 - def processDevices(self, device_list):
352 """ 353 Read the input and process the devices 354 * create the device entry 355 * set zproperties 356 * set custom schema properties 357 * model the device 358 359 @parameter device_list: list of device entries 360 @type device_list: list of dictionaries 361 """ 362 363 def transactional(f): 364 return f if self.options.nocommit else transact(f)
365 366 processed = {'total':0, 'errors':0} 367 368 @transactional 369 def _process(device_specs): 370 # Get the latest bits 371 self.dmd.zport._p_jar.sync() 372 373 loaderName = device_specs.get('loader') 374 if loaderName is not None: 375 try: 376 orgName = device_specs['devicePath'] 377 organizer = self.dmd.getObjByPath('dmd' + orgName) 378 deviceLoader = getUtility(IDeviceLoader, loaderName, organizer) 379 devobj = self.runLoader(deviceLoader, device_specs) 380 except ConflictError: 381 raise 382 except Exception: 383 devName = device_specs.get('device_specs', 'Unkown Device') 384 msg = "Ignoring device loader issue for %s" % devName 385 self.reportException(msg, devName, {'specs':str(device_specs)}) 386 processed['errors'] += 1 387 return 388 else: 389 devobj = self.getDevice(device_specs) 390 deviceLoader = None 391 392 if devobj is None: 393 if deviceLoader is not None: 394 processed['total'] += 1 395 else: 396 self.addAllLGSOrganizers(device_specs) 397 self.applyZProps(devobj, device_specs) 398 self.applyCustProps(devobj, device_specs) 399 self.applyOtherProps(devobj, device_specs) 400 401 return devobj 402 403 @transactional 404 def _snmp_community(device_specs, devobj): 405 # Discover the SNMP community if it isn't explicitly set. 406 if 'zSnmpCommunity' not in device_specs: 407 self.log.debug('Discovering SNMP version and community') 408 devobj.manage_snmpCommunity() 409 410 @transactional 411 def _model(devobj): 412 try: 413 devobj.collectDevice(setlog=self.options.showModelOutput) 414 except ConflictError: 415 raise 416 except Exception, ex: 417 msg = "Modeling error for %s" % devobj.id 418 self.reportException(msg, devobj.id, ex) 419 processed['errors'] += 1 420 processed['total'] += 1 421 422 for device_specs in device_list: 423 devobj = _process(device_specs) 424 425 # We need to commit in order to model, so don't bother 426 # trying to model unless we can do both 427 if devobj and not self.options.nocommit and not self.options.nomodel: 428 _snmp_community(device_specs, devobj) 429 _model(devobj) 430 431 self.reportResults(processed, len(device_list)) 432
433 - def reportException(self, msg, devName='', **kwargs):
434 """ 435 Report exceptions back to the the event console 436 """ 437 self.log.exception(msg) 438 if not self.options.nocommit: 439 evt = self.baseEvent.copy() 440 evt.update(dict( 441 summary=msg, 442 traceback=format_exc() 443 )) 444 evt.update(kwargs) 445 if devName: 446 evt['device'] = devName 447 self.dmd.ZenEventManager.sendEvent(evt)
448
449 - def reportResults(self, processed, totalDevices):
450 """ 451 Report the success + total counts from loading devices. 452 """ 453 msg = "Modeled %d of %d devices, with %d errors" % ( 454 processed['total'], totalDevices, processed['errors'] ) 455 self.log.info(msg) 456 457 if not self.options.nocommit: 458 evt = self.baseEvent.copy() 459 evt.update(dict( 460 severity=SEVERITY_INFO, 461 summary=msg, 462 modeled=processed['total'], 463 errors=processed['errors'], 464 total=totalDevices, 465 )) 466 self.dmd.ZenEventManager.sendEvent(evt)
467
468 - def notifyNewDeviceCreated(self, deviceName):
469 """ 470 Report that we added a new device. 471 """ 472 if not self.options.nocommit: 473 evt = self.baseEvent.copy() 474 evt.update(dict( 475 severity=SEVERITY_INFO, 476 summary= "Added new device %s" % deviceName 477 )) 478 self.dmd.ZenEventManager.sendEvent(evt)
479
480 - def getDevice(self, device_specs):
481 """ 482 Find or create the specified device 483 484 @parameter device_specs: device creation dictionary 485 @type device_specs: dictionary 486 @return: device or None 487 @rtype: DMD device object 488 """ 489 if 'deviceName' not in device_specs: 490 return None 491 name = device_specs['deviceName'] 492 devobj = self.dmd.Devices.findDevice(name) 493 if devobj is not None: 494 self.log.info("Found existing device %s" % name) 495 return devobj 496 497 specs = {} 498 for key in self.loader_args: 499 if key in device_specs: 500 specs[key] = device_specs[key] 501 502 try: 503 self.log.info("Creating device %s" % name) 504 505 # Do NOT model at this time 506 specs['discoverProto'] = 'none' 507 508 self.loader(**specs) 509 devobj = self.dmd.Devices.findDevice(name) 510 if devobj is None: 511 self.log.error("Unable to find newly created device %s -- skipping" \ 512 % name) 513 else: 514 self.notifyNewDeviceCreated(name) 515 516 except Exception: 517 msg = "Unable to load %s -- skipping" % name 518 self.reportException(msg, name) 519 520 return devobj
521
522 - def buildOptions(self):
523 """ 524 Add our command-line options to the basics 525 """ 526 ZCmdBase.buildOptions(self) 527 528 self.parser.add_option('--show_options', 529 dest="show_options", default=False, 530 action="store_true", 531 help="Show the various options understood by the loader") 532 533 self.parser.add_option('--sample_configs', 534 dest="sample_configs", default=False, 535 action="store_true", 536 help="Show an example configuration file.") 537 538 self.parser.add_option('--showModelOutput', 539 dest="showModelOutput", default=True, 540 action="store_false", 541 help="Show modelling activity") 542 543 self.parser.add_option('--nocommit', 544 dest="nocommit", default=False, 545 action="store_true", 546 help="Don't commit changes to the ZODB. Use for verifying config file.") 547 548 self.parser.add_option('--nomodel', 549 dest="nomodel", default=False, 550 action="store_true", 551 help="Don't model the remote devices. Must be able to commit changes.")
552
553 - def parseDevices(self, data):
554 """ 555 From the list of strings in rawDevices, construct a list 556 of device dictionaries, ready to load into Zenoss. 557 558 @parameter data: list of strings representing device entries 559 @type data: list of strings 560 @return: list of parsed device entries 561 @rtype: list of dictionaries 562 """ 563 if not data: 564 return [] 565 566 comment = re.compile(r'^\s*#.*') 567 568 defaults = {'devicePath':"/Discovered" } 569 finalList = [] 570 i = 0 571 while i < len(data): 572 line = data[i] 573 line = re.sub(comment, '', line).strip() 574 if line == '': 575 i += 1 576 continue 577 578 # Check for line continuation character '\' 579 while line[-1] == '\\' and i < len(data): 580 i += 1 581 line = line[:-1] + data[i] 582 line = re.sub(comment, '', line).strip() 583 584 if line[0] == '/' or line[1] == '/': # Found an organizer 585 defaults = self.parseDeviceEntry(line, {}) 586 if defaults is None: 587 defaults = {'devicePath':"/Discovered" } 588 else: 589 defaults['devicePath'] = defaults['deviceName'] 590 del defaults['deviceName'] 591 self.addOrganizer(defaults) 592 593 else: 594 configs = self.parseDeviceEntry(line, defaults) 595 if configs: 596 finalList.append(configs) 597 i += 1 598 599 return finalList
600
601 - def parseDeviceEntry(self, line, defaults):
602 """ 603 Build a dictionary of properties from one line's input 604 605 @parameter line: string containing one device's info 606 @type line: string 607 @parameter defaults: dictionary of default settings 608 @type defaults: dictionary 609 @return: parsed device entry 610 @rtype: dictionary 611 """ 612 options = [] 613 # Note: organizers and device names can have spaces in them 614 if line[0] in ["'", '"']: 615 delim = line[0] 616 eom = line.find(delim, 1) 617 if eom == -1: 618 self.log.error("While reading name, unable to parse" \ 619 " the entry for %s -- skipping", line ) 620 return None 621 name = line[1:eom] 622 options = line[eom+1:] 623 624 else: 625 options = line.split(None, 1) 626 name = options.pop(0) 627 if options: 628 options = options.pop(0) 629 630 configs = defaults.copy() 631 configs['deviceName'] = name 632 633 if options: 634 try: 635 # Add a newline to allow for trailing comments 636 evalString = 'dict(' + options + '\n)' 637 configs.update(eval(evalString)) 638 except: 639 self.log.error( "Unable to parse the entry for %s -- skipping" % name ) 640 self.log.error( "Raw string: %s" % options ) 641 return None 642 643 return configs
644 645 646 if __name__=='__main__': 647 batchLoader = BatchDeviceLoader() 648 649 if batchLoader.options.show_options: 650 print "Options = %s" % sorted( batchLoader.loader_args.keys() ) 651 help(batchLoader.loader) 652 sys.exit(0) 653 654 if batchLoader.options.sample_configs: 655 print batchLoader.sample_configs 656 sys.exit(0) 657 658 device_list = batchLoader.loadDeviceList() 659 if not device_list: 660 batchLoader.log.warn("No device entries found to load.") 661 sys.exit(1) 662 663 batchLoader.processDevices(device_list) 664 sys.exit(0) 665