Package Products :: Package ZenRelations :: Module ImportRM
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenRelations.ImportRM

  1  #! /usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  # ########################################################################## 
  4  # 
  5  # This program is part of Zenoss Core, an open source monitoring platform. 
  6  # Copyright (C) 2009, Zenoss Inc. 
  7  # 
  8  # This program is free software; you can redistribute it and/or modify it 
  9  # under the terms of the GNU General Public License version 2 or (at your 
 10  # option) any later version as published by the Free Software Foundation. 
 11  # 
 12  # For complete information please visit: http://www.zenoss.com/oss/ 
 13  # 
 14  # ########################################################################## 
 15   
 16  __doc__ = """ImportRM 
 17  Import RelationshipManager objects into a Zope database 
 18  This provides support methods used by the Python xml.sax library to 
 19  parse and construct an XML document. 
 20   
 21  Descriptions of the XML document format can be found in the 
 22  Developers Guide. 
 23  """ 
 24  import Globals 
 25  import sys 
 26  import os 
 27  import transaction 
 28  import zope.component 
 29  from zope.event import notify 
 30  from DateTime import DateTime 
 31  from xml.sax import make_parser, saxutils, SAXParseException 
 32  from xml.sax.handler import ContentHandler 
 33   
 34  from Acquisition import aq_base 
 35  from zExceptions import NotFound 
 36  from OFS.PropertyManager import PropertyManager 
 37   
 38  from Products.ZenUtils.ZCmdBase import ZCmdBase 
 39  from Products.ZenUtils.Utils import importClass 
 40  from Products.ZenUtils.Utils import getObjByPath 
 41   
 42  from Products.ZenModel.interfaces import IZenDocProvider 
 43  from Products.ZenRelations.Exceptions import * 
 44  from Products.Zuul.catalog.events import IndexingEvent 
 45   
 46  _STRING_PROPERTY_TYPES = ( 'string', 'text', 'password' ) 
 47   
 48   
 49   
50 -class ImportRM(ZCmdBase, ContentHandler):
51 """ 52 Wrapper module to interface between Zope and the Python SAX XML library. 53 The xml.sax.parse() calls different routines depending on what it finds. 54 55 A simple example of a valid XML file can be found in the objects.xml file 56 for a ZenPack. 57 58 <?xml version="1.0"?> 59 <objects> 60 <!-- ('', 'zport', 'dmd', 'Devices', 'rrdTemplates', 'HelloWorld') --> 61 <object id='/zport/dmd/Devices/rrdTemplates/HelloWorld' module='Products.ZenModel.RRDTemplate' class='RRDTemplate'> 62 <property type="text" id="description" mode="w" > This is the glorious description that shows up when we click on our RRD template </property> 63 <tomanycont id='datasources'> 64 <object id='hello' module='Products.ZenModel.BasicDataSource' class='BasicDataSource'> 65 <property select_variable="sourcetypes" type="selection" id="sourcetype" mode="w" > SNMP </property> 66 <property type="boolean" id="enabled" mode="w" > True </property> 67 <property type="string" id="eventClass" mode="w" > /Cmd/Fail </property> 68 <property type="int" id="severity" mode="w" > 3 </property> 69 <property type="int" id="cycletime" mode="w" > 300 </property> 70 <property type="boolean" id="usessh" mode="w" > False </property> 71 <tomanycont id='datapoints'> 72 <object id='hello' module='Products.ZenModel.RRDDataPoint' class='RRDDataPoint'> 73 <property select_variable="rrdtypes" type="selection" id="rrdtype" mode="w" > GAUGE </property> 74 <property type="boolean" id="isrow" mode="w" > True </property> 75 </object> 76 </tomanycont> 77 </object> 78 79 <!-- snip --> 80 81 </objects> 82 """ 83 rootpath = '' 84 skipobj = 0 85
86 - def __init__(self, noopts=0, app=None, keeproot=False):
87 """ 88 Initializer 89 90 @param noopts: don't use sys.argv for command-line options 91 @type noopts: boolean 92 @param app: app 93 @type app: object 94 @param keeproot: keeproot 95 @type keeproot: boolean 96 """ 97 ZCmdBase.__init__(self, noopts, app, keeproot) 98 ContentHandler.__init__(self)
99
100 - def context(self):
101 """ 102 Return the bottom object in the stack 103 104 @return: 105 @rtype: object 106 """ 107 return self.objstack[-1]
108
109 - def cleanattrs(self, attrs):
110 """ 111 Convert all attributes to string values 112 113 @param attrs: (key,val) pairs 114 @type attrs: list 115 @return: same list, but with all values as strings 116 @rtype: list 117 """ 118 myattrs = {} 119 for (key, val) in attrs.items(): 120 myattrs[key] = str(val) 121 return myattrs
122
123 - def startElement(self, name, attrs):
124 """ 125 Function called when the parser finds the starting element 126 in the XML file. 127 128 @param name: name of the element 129 @type name: string 130 @param attrs: list of (key, value) tuples 131 @type attrs: list 132 """ 133 ignoredElements = [ 'objects' ] 134 attrs = self.cleanattrs(attrs) 135 if self.skipobj > 0: 136 self.skipobj += 1 137 return 138 139 self.log.debug('tag %s, context %s, line %s' % ( 140 name, self.context().id, self._locator.getLineNumber() )) 141 142 if name == 'object': 143 if attrs.get('class') == 'Device': 144 devId = attrs['id'].split('/')[-1] 145 dev = self.dmd.Devices.findDeviceByIdOrIp(devId) 146 if dev: 147 msg = 'The device %s already exists on this system! (Line %s)' % \ 148 (devId, self._locator.getLineNumber()) 149 raise Exception(msg) 150 151 if attrs.get('class') == 'IpAddress': 152 ipAddress = attrs['id'] 153 dev = self.dmd.Devices.findDeviceByIdOrIp(ipAddress) 154 if dev: 155 msg = 'The IP address %s already exists on this system! (Line %s)' % \ 156 (ipAddress, self._locator.getLineNumber()) 157 raise Exception(msg) 158 159 obj = self.createObject(attrs) 160 if obj is None: 161 formattedAttrs = '' 162 for key, value in attrs.items(): 163 formattedAttrs += ' * %s: %s\n' % (key, value) 164 raise Exception('Unable to create object using the following ' 165 'attributes:\n%s' % formattedAttrs) 166 167 if not self.options.noindex and hasattr(aq_base(obj), 168 'reIndex') and not self.rootpath: 169 self.rootpath = obj.getPrimaryId() 170 171 self.objstack.append(obj) 172 173 elif name == 'tomanycont' or name == 'tomany': 174 nextobj = self.context()._getOb(attrs['id'], None) 175 if nextobj is None: 176 self.skipobj = 1 177 return 178 else: 179 self.objstack.append(nextobj) 180 elif name == 'toone': 181 relname = attrs.get('id') 182 self.log.debug('toone %s, on object %s', relname, 183 self.context().id) 184 rel = getattr(aq_base(self.context()), relname, None) 185 if rel is None: 186 return 187 objid = attrs.get('objid') 188 self.addLink(rel, objid) 189 elif name == 'link': 190 self.addLink(self.context(), attrs['objid']) 191 elif name == 'property': 192 self.curattrs = attrs 193 elif name in ignoredElements: 194 pass 195 else: 196 self.log.warning( "Ignoring an unknown XML element type: %s" % name )
197
198 - def endElement(self, name):
199 """ 200 Function called when the parser finds the starting element 201 in the XML file. 202 203 @param name: name of the ending element 204 @type name: string 205 """ 206 ignoredElements = [ 'toone', 'link' ] 207 if self.skipobj > 0: 208 self.skipobj -= 1 209 return 210 211 noIncrementalCommit = self.options.noCommit or self.options.chunk_size==0 212 213 if name in ('object', 'tomany', 'tomanycont'): 214 obj = self.objstack.pop() 215 notify(IndexingEvent(obj)) 216 if hasattr(aq_base(obj), 'index_object'): 217 obj.index_object() 218 if self.rootpath == obj.getPrimaryId(): 219 self.log.info('Calling reIndex %s', obj.getPrimaryId()) 220 obj.reIndex() 221 self.rootpath = '' 222 if (not noIncrementalCommit and 223 not self.objectnumber % self.options.chunk_size): 224 self.log.debug("Committing a batch of %s objects" % 225 self.options.chunk_size) 226 self.commit() 227 228 elif name == 'objects': # ie end of the file 229 self.log.info('End loading objects') 230 self.log.info('Processing links') 231 self.processLinks() 232 if not self.options.noCommit: 233 self.commit() 234 self.log.info('Loaded %d objects into the ZODB database' 235 % self.objectnumber) 236 else: 237 self.log.info('Would have created %d objects in the ZODB database' 238 % self.objectnumber) 239 240 elif name == 'property': 241 self.setProperty(self.context(), self.curattrs, 242 self.charvalue) 243 # We've closed off a tag, so now we need to re-initialize 244 # the area that stores the contents of elements 245 self.charvalue = '' 246 247 elif name in ignoredElements: 248 pass 249 else: 250 self.log.warning( "Ignoring an unknown XML element type: %s" % name )
251
252 - def characters(self, chars):
253 """ 254 Called by xml.sax.parse() with data found in an element 255 eg <object>my characters stuff</object> 256 257 Note that this can be called character by character. 258 259 @param chars: chars 260 @type chars: string 261 """ 262 self.charvalue += saxutils.unescape(chars)
263
264 - def createObject(self, attrs):
265 """ 266 Create an object and set it into its container 267 268 @param attrs: attrs 269 @type attrs: string 270 @return: newly created object 271 @rtype: object 272 """ 273 # Does the object exist already? 274 id = attrs.get('id') 275 obj = None 276 try: 277 if id.startswith('/'): 278 obj = getObjByPath(self.app, id) 279 else: 280 obj = self.context()._getOb(id) 281 except (KeyError, AttributeError, NotFound): 282 pass 283 284 if obj is None: 285 klass = importClass(attrs.get('module'), attrs.get('class')) 286 if id.find('/') > -1: 287 (contextpath, id) = os.path.split(id) 288 try: 289 pathobj = getObjByPath(self.context(), contextpath) 290 except (KeyError, AttributeError, NotFound): 291 self.log.warn( "Unable to find context path %s (line %s ?) for %s" % ( 292 contextpath, self._locator.getLineNumber(), id )) 293 if not self.options.noCommit: 294 self.log.warn( "Not committing any changes" ) 295 self.options.noCommit = True 296 return None 297 self.objstack.append(pathobj) 298 self.log.debug('Building instance %s of class %s',id,klass.__name__) 299 try: 300 if klass.__name__ == 'AdministrativeRole': 301 user = [x for x in self.dmd.ZenUsers.objectValues() if x.id == id] 302 if user: 303 obj = klass(user[0], self.context().device()) 304 else: 305 msg = "No AdminRole user %s exists (line %s)" % ( 306 id, self._locator.getLineNumber()) 307 self.log.error(msg) 308 raise Exception(msg) 309 else: 310 obj = klass(id) 311 except TypeError, ex: 312 # This happens when the constructor expects more arguments 313 self.log.exception("Unable to build %s instance of class %s (line %s)", 314 id, klass.__name__, self._locator.getLineNumber()) 315 raise 316 self.context()._setObject(obj.id, obj) 317 obj = self.context()._getOb(obj.id) 318 self.objectnumber += 1 319 self.log.debug('Added object %s to database' 320 % obj.getPrimaryId()) 321 else: 322 self.log.debug('Object %s already exists -- skipping' % id) 323 return obj
324
325 - def setZendoc(self, obj, value):
326 zendocObj = zope.component.queryAdapter(obj, IZenDocProvider) 327 if zendocObj is not None: 328 zendocObj.setZendoc( value ) 329 elif value: 330 self.log.warn('zendoc property could not be set to' + 331 ' %s on object %s' % ( value, obj.id ) )
332
333 - def setProperty(self, obj, attrs, value):
334 """ 335 Set the value of a property on an object. 336 337 @param obj: obj 338 @type obj: string 339 @param attrs: attrs 340 @type attrs: string 341 @param value: value 342 @type value: string 343 @return: 344 @rtype: 345 """ 346 name = attrs.get('id') 347 proptype = attrs.get('type') 348 setter = attrs.get('setter', None) 349 self.log.debug('Setting object %s attr %s type %s value %s' 350 % (obj.id, name, proptype, value)) 351 linenum = self._locator.getLineNumber() 352 353 # Sanity check the value 354 value = value.strip() 355 try: 356 value = str(value) 357 except UnicodeEncodeError: 358 self.log.warn('UnicodeEncodeError at line %s while attempting' % linenum + \ 359 ' str(%s) for object %s attribute %s -- ignoring' % ( 360 obj.id, name, proptype, value)) 361 362 if name == 'zendoc': 363 return self.setZendoc( obj, value ) 364 365 # Guess at how to interpret the value given the property type 366 if proptype == 'selection': 367 try: 368 firstElement = getattr(obj, name)[0] 369 if isinstance(firstElement, basestring): 370 proptype = 'string' 371 except (TypeError, IndexError): 372 self.log.warn("Error at line %s when trying to " % linenum + \ 373 " use (%s) as the value for object %s attribute %s -- assuming a string" 374 % (obj.id, name, proptype, value)) 375 proptype = 'string' 376 377 if proptype == 'date': 378 try: 379 value = float(value) 380 except ValueError: 381 pass 382 value = DateTime(value) 383 384 elif proptype not in _STRING_PROPERTY_TYPES: 385 try: 386 value = eval(value) 387 except NameError: 388 self.log.critical('Error trying to evaluate %s at line %s', 389 value, linenum) 390 raise 391 except SyntaxError: 392 self.log.debug("Non-fatal SyntaxError at line %s while eval'ing '%s'" % ( 393 linenum, value) ) 394 395 # Actually use the value 396 if not obj.hasProperty(name): 397 obj._setProperty(name, value, type=proptype, setter=setter) 398 else: 399 obj._updateProperty(name, value)
400 412 428
429 - def buildOptions(self):
430 """ 431 Command-line options specific to this command 432 """ 433 ZCmdBase.buildOptions(self) 434 self.parser.add_option('-i', '--infile', dest='infile', 435 help='Input file for import. Default is stdin' 436 ) 437 self.parser.add_option('--noindex', dest='noindex', 438 action='store_true', default=False, 439 help='Do not try to index the freshly loaded objects.' 440 ) 441 self.parser.add_option('--chunksize', dest='chunk_size', 442 help='Number of objects to commit at a time.', 443 type='int', 444 default=100 445 ) 446 self.parser.add_option( 447 '-n', 448 '--noCommit', 449 dest='noCommit', 450 action='store_true', 451 default=0, 452 help='Do not store changes to the DMD (ie for debugging purposes)', 453 )
454
455 - def loadObjectFromXML(self, xmlfile=''):
456 """ 457 This method can be used to load data for the root of Zenoss (default 458 behavior) or it can be used to operate on a specific point in the 459 Zenoss hierarchy (ZODB). 460 461 Upon loading the XML file to be processed, the content of the XML file 462 is handled by the methods in this class when called by xml.sax.parse(). 463 464 Reads from a file if xmlfile is specified, otherwise reads 465 from the command-line option --infile. If no files are found from 466 any of these places, read from standard input. 467 468 @param xmlfile: Name of XML file to load, or file-like object 469 @type xmlfile: string or file-like object 470 """ 471 self.objstack = [self.app] 472 self.links = [] 473 self.objectnumber = 0 474 self.charvalue = '' 475 if xmlfile and isinstance(xmlfile, basestring): 476 self.infile = open(xmlfile) 477 elif hasattr(xmlfile, 'read'): 478 self.infile = xmlfile 479 elif self.options.infile: 480 self.infile = open(self.options.infile) 481 else: 482 self.infile = sys.stdin 483 parser = make_parser() 484 parser.setContentHandler(self) 485 try: 486 parser.parse(self.infile) 487 except SAXParseException, ex: 488 self.log.error("XML parse error at line %d column %d: %s", 489 ex.getLineNumber(), ex.getColumnNumber(), 490 ex.getMessage()) 491 finally: 492 self.infile.close()
493
494 - def loadDatabase(self):
495 """ 496 The default behavior of loadObjectFromXML() will be to use the Zope 497 app object, and thus operatate on the whole of Zenoss. 498 """ 499 self.loadObjectFromXML()
500
501 - def commit(self):
502 """ 503 Wrapper around the Zope database commit() 504 """ 505 trans = transaction.get() 506 trans.note('Import from file %s using %s' 507 % (self.options.infile, self.__class__.__name__)) 508 trans.commit() 509 if hasattr(self, 'connection'): 510 # It's safe to call syncdb() 511 self.syncdb()
512
513 -class SpoofedOptions(object):
514 """ 515 SpoofedOptions 516 """ 517
518 - def __init__(self):
519 self.infile = '' 520 self.noCommit = True 521 self.noindex = True 522 self.dataroot = '/zport/dmd'
523 524
525 -class NoLoginImportRM(ImportRM):
526 """ 527 An ImportRM that does not call the __init__ method on ZCmdBase 528 """ 529
530 - def __init__(self, app):
531 """ 532 Initializer 533 534 @param app: app 535 @type app: string 536 """ 537 import Products.ZenossStartup 538 from Products.Five import zcml 539 zcml.load_site() 540 import logging 541 self.log = logging.getLogger('zen.ImportRM') 542 self.app = app 543 ContentHandler.__init__(self) 544 self.options = SpoofedOptions() 545 self.dataroot = None 546 self.getDataRoot()
547 548 549 if __name__ == '__main__': 550 im = ImportRM() 551 im.loadDatabase() 552