1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 __doc__ = """ImportRM
17 Import RelationshipManager objects into a Zope database
18 This provides support methods used by the Python xml.sax library to
19 parse and construct an XML document.
20
21 Descriptions of the XML document format can be found in the
22 Developers Guide.
23 """
24 import Globals
25 import sys
26 import os
27 import transaction
28 import zope.component
29 from zope.event import notify
30 from DateTime import DateTime
31 from xml.sax import make_parser, saxutils, SAXParseException
32 from xml.sax.handler import ContentHandler
33
34 from Acquisition import aq_base
35 from zExceptions import NotFound
36 from OFS.PropertyManager import PropertyManager
37
38 from Products.ZenUtils.ZCmdBase import ZCmdBase
39 from Products.ZenUtils.Utils import importClass
40 from Products.ZenUtils.Utils import getObjByPath
41
42 from Products.ZenModel.interfaces import IZenDocProvider
43 from Products.ZenRelations.Exceptions import *
44 from Products.Zuul.catalog.events import IndexingEvent
45
46 _STRING_PROPERTY_TYPES = ( 'string', 'text', 'password' )
47
48
49
50 -class ImportRM(ZCmdBase, ContentHandler):
51 """
52 Wrapper module to interface between Zope and the Python SAX XML library.
53 The xml.sax.parse() calls different routines depending on what it finds.
54
55 A simple example of a valid XML file can be found in the objects.xml file
56 for a ZenPack.
57
58 <?xml version="1.0"?>
59 <objects>
60 <!-- ('', 'zport', 'dmd', 'Devices', 'rrdTemplates', 'HelloWorld') -->
61 <object id='/zport/dmd/Devices/rrdTemplates/HelloWorld' module='Products.ZenModel.RRDTemplate' class='RRDTemplate'>
62 <property type="text" id="description" mode="w" > This is the glorious description that shows up when we click on our RRD template </property>
63 <tomanycont id='datasources'>
64 <object id='hello' module='Products.ZenModel.BasicDataSource' class='BasicDataSource'>
65 <property select_variable="sourcetypes" type="selection" id="sourcetype" mode="w" > SNMP </property>
66 <property type="boolean" id="enabled" mode="w" > True </property>
67 <property type="string" id="eventClass" mode="w" > /Cmd/Fail </property>
68 <property type="int" id="severity" mode="w" > 3 </property>
69 <property type="int" id="cycletime" mode="w" > 300 </property>
70 <property type="boolean" id="usessh" mode="w" > False </property>
71 <tomanycont id='datapoints'>
72 <object id='hello' module='Products.ZenModel.RRDDataPoint' class='RRDDataPoint'>
73 <property select_variable="rrdtypes" type="selection" id="rrdtype" mode="w" > GAUGE </property>
74 <property type="boolean" id="isrow" mode="w" > True </property>
75 </object>
76 </tomanycont>
77 </object>
78
79 <!-- snip -->
80
81 </objects>
82 """
83 rootpath = ''
84 skipobj = 0
85
86 - def __init__(self, noopts=0, app=None, keeproot=False):
87 """
88 Initializer
89
90 @param noopts: don't use sys.argv for command-line options
91 @type noopts: boolean
92 @param app: app
93 @type app: object
94 @param keeproot: keeproot
95 @type keeproot: boolean
96 """
97 ZCmdBase.__init__(self, noopts, app, keeproot)
98 ContentHandler.__init__(self)
99
101 """
102 Return the bottom object in the stack
103
104 @return:
105 @rtype: object
106 """
107 return self.objstack[-1]
108
110 """
111 Convert all attributes to string values
112
113 @param attrs: (key,val) pairs
114 @type attrs: list
115 @return: same list, but with all values as strings
116 @rtype: list
117 """
118 myattrs = {}
119 for (key, val) in attrs.items():
120 myattrs[key] = str(val)
121 return myattrs
122
124 """
125 Function called when the parser finds the starting element
126 in the XML file.
127
128 @param name: name of the element
129 @type name: string
130 @param attrs: list of (key, value) tuples
131 @type attrs: list
132 """
133 ignoredElements = [ 'objects' ]
134 attrs = self.cleanattrs(attrs)
135 if self.skipobj > 0:
136 self.skipobj += 1
137 return
138
139 self.log.debug('tag %s, context %s, line %s' % (
140 name, self.context().id, self._locator.getLineNumber() ))
141
142 if name == 'object':
143 if attrs.get('class') == 'Device':
144 devId = attrs['id'].split('/')[-1]
145 dev = self.dmd.Devices.findDeviceByIdOrIp(devId)
146 if dev:
147 msg = 'The device %s already exists on this system! (Line %s)' % \
148 (devId, self._locator.getLineNumber())
149 raise Exception(msg)
150
151 if attrs.get('class') == 'IpAddress':
152 ipAddress = attrs['id']
153 dev = self.dmd.Devices.findDeviceByIdOrIp(ipAddress)
154 if dev:
155 msg = 'The IP address %s already exists on this system! (Line %s)' % \
156 (ipAddress, self._locator.getLineNumber())
157 raise Exception(msg)
158
159 obj = self.createObject(attrs)
160 if obj is None:
161 formattedAttrs = ''
162 for key, value in attrs.items():
163 formattedAttrs += ' * %s: %s\n' % (key, value)
164 raise Exception('Unable to create object using the following '
165 'attributes:\n%s' % formattedAttrs)
166
167 if not self.options.noindex and hasattr(aq_base(obj),
168 'reIndex') and not self.rootpath:
169 self.rootpath = obj.getPrimaryId()
170
171 self.objstack.append(obj)
172
173 elif name == 'tomanycont' or name == 'tomany':
174 nextobj = self.context()._getOb(attrs['id'], None)
175 if nextobj is None:
176 self.skipobj = 1
177 return
178 else:
179 self.objstack.append(nextobj)
180 elif name == 'toone':
181 relname = attrs.get('id')
182 self.log.debug('toone %s, on object %s', relname,
183 self.context().id)
184 rel = getattr(aq_base(self.context()), relname, None)
185 if rel is None:
186 return
187 objid = attrs.get('objid')
188 self.addLink(rel, objid)
189 elif name == 'link':
190 self.addLink(self.context(), attrs['objid'])
191 elif name == 'property':
192 self.curattrs = attrs
193 elif name in ignoredElements:
194 pass
195 else:
196 self.log.warning( "Ignoring an unknown XML element type: %s" % name )
197
199 """
200 Function called when the parser finds the starting element
201 in the XML file.
202
203 @param name: name of the ending element
204 @type name: string
205 """
206 ignoredElements = [ 'toone', 'link' ]
207 if self.skipobj > 0:
208 self.skipobj -= 1
209 return
210
211 noIncrementalCommit = self.options.noCommit or self.options.chunk_size==0
212
213 if name in ('object', 'tomany', 'tomanycont'):
214 obj = self.objstack.pop()
215 notify(IndexingEvent(obj))
216 if hasattr(aq_base(obj), 'index_object'):
217 obj.index_object()
218 if self.rootpath == obj.getPrimaryId():
219 self.log.info('Calling reIndex %s', obj.getPrimaryId())
220 obj.reIndex()
221 self.rootpath = ''
222 if (not noIncrementalCommit and
223 not self.objectnumber % self.options.chunk_size):
224 self.log.debug("Committing a batch of %s objects" %
225 self.options.chunk_size)
226 self.commit()
227
228 elif name == 'objects':
229 self.log.info('End loading objects')
230 self.log.info('Processing links')
231 self.processLinks()
232 if not self.options.noCommit:
233 self.commit()
234 self.log.info('Loaded %d objects into the ZODB database'
235 % self.objectnumber)
236 else:
237 self.log.info('Would have created %d objects in the ZODB database'
238 % self.objectnumber)
239
240 elif name == 'property':
241 self.setProperty(self.context(), self.curattrs,
242 self.charvalue)
243
244
245 self.charvalue = ''
246
247 elif name in ignoredElements:
248 pass
249 else:
250 self.log.warning( "Ignoring an unknown XML element type: %s" % name )
251
253 """
254 Called by xml.sax.parse() with data found in an element
255 eg <object>my characters stuff</object>
256
257 Note that this can be called character by character.
258
259 @param chars: chars
260 @type chars: string
261 """
262 self.charvalue += saxutils.unescape(chars)
263
265 """
266 Create an object and set it into its container
267
268 @param attrs: attrs
269 @type attrs: string
270 @return: newly created object
271 @rtype: object
272 """
273
274 id = attrs.get('id')
275 obj = None
276 try:
277 if id.startswith('/'):
278 obj = getObjByPath(self.app, id)
279 else:
280 obj = self.context()._getOb(id)
281 except (KeyError, AttributeError, NotFound):
282 pass
283
284 if obj is None:
285 klass = importClass(attrs.get('module'), attrs.get('class'))
286 if id.find('/') > -1:
287 (contextpath, id) = os.path.split(id)
288 try:
289 pathobj = getObjByPath(self.context(), contextpath)
290 except (KeyError, AttributeError, NotFound):
291 self.log.warn( "Unable to find context path %s (line %s ?) for %s" % (
292 contextpath, self._locator.getLineNumber(), id ))
293 if not self.options.noCommit:
294 self.log.warn( "Not committing any changes" )
295 self.options.noCommit = True
296 return None
297 self.objstack.append(pathobj)
298 self.log.debug('Building instance %s of class %s',id,klass.__name__)
299 try:
300 if klass.__name__ == 'AdministrativeRole':
301 user = [x for x in self.dmd.ZenUsers.objectValues() if x.id == id]
302 if user:
303 obj = klass(user[0], self.context().device())
304 else:
305 msg = "No AdminRole user %s exists (line %s)" % (
306 id, self._locator.getLineNumber())
307 self.log.error(msg)
308 raise Exception(msg)
309 else:
310 obj = klass(id)
311 except TypeError, ex:
312
313 self.log.exception("Unable to build %s instance of class %s (line %s)",
314 id, klass.__name__, self._locator.getLineNumber())
315 raise
316 self.context()._setObject(obj.id, obj)
317 obj = self.context()._getOb(obj.id)
318 self.objectnumber += 1
319 self.log.debug('Added object %s to database'
320 % obj.getPrimaryId())
321 else:
322 self.log.debug('Object %s already exists -- skipping' % id)
323 return obj
324
332
334 """
335 Set the value of a property on an object.
336
337 @param obj: obj
338 @type obj: string
339 @param attrs: attrs
340 @type attrs: string
341 @param value: value
342 @type value: string
343 @return:
344 @rtype:
345 """
346 name = attrs.get('id')
347 proptype = attrs.get('type')
348 setter = attrs.get('setter', None)
349 self.log.debug('Setting object %s attr %s type %s value %s'
350 % (obj.id, name, proptype, value))
351 linenum = self._locator.getLineNumber()
352
353
354 value = value.strip()
355 try:
356 value = str(value)
357 except UnicodeEncodeError:
358 self.log.warn('UnicodeEncodeError at line %s while attempting' % linenum + \
359 ' str(%s) for object %s attribute %s -- ignoring' % (
360 obj.id, name, proptype, value))
361
362 if name == 'zendoc':
363 return self.setZendoc( obj, value )
364
365
366 if proptype == 'selection':
367 try:
368 firstElement = getattr(obj, name)[0]
369 if isinstance(firstElement, basestring):
370 proptype = 'string'
371 except (TypeError, IndexError):
372 self.log.warn("Error at line %s when trying to " % linenum + \
373 " use (%s) as the value for object %s attribute %s -- assuming a string"
374 % (obj.id, name, proptype, value))
375 proptype = 'string'
376
377 if proptype == 'date':
378 try:
379 value = float(value)
380 except ValueError:
381 pass
382 value = DateTime(value)
383
384 elif proptype not in _STRING_PROPERTY_TYPES:
385 try:
386 value = eval(value)
387 except NameError:
388 self.log.critical('Error trying to evaluate %s at line %s',
389 value, linenum)
390 raise
391 except SyntaxError:
392 self.log.debug("Non-fatal SyntaxError at line %s while eval'ing '%s'" % (
393 linenum, value) )
394
395
396 if not obj.hasProperty(name):
397 obj._setProperty(name, value, type=proptype, setter=setter)
398 else:
399 obj._updateProperty(name, value)
400
402 """
403 Build list of links to form after all objects have been created
404 make sure that we don't add other side of a bidirectional relation
405
406 @param rel: relationship object
407 @type rel: relation object
408 @param objid: objid
409 @type objid: string
410 """
411 self.links.append((rel.getPrimaryId(), objid))
412
414 """
415 Walk through all the links that we saved and link them up
416 """
417 for (relid, objid) in self.links:
418 try:
419 self.log.debug('Linking relation %s to object %s',
420 relid, objid)
421 rel = getObjByPath(self.app, relid)
422 obj = getObjByPath(self.app, objid)
423 if not rel.hasobject(obj):
424 rel.addRelation(obj)
425 except:
426 self.log.critical('Failed linking relation %s to object %s' % (
427 relid, objid))
428
430 """
431 Command-line options specific to this command
432 """
433 ZCmdBase.buildOptions(self)
434 self.parser.add_option('-i', '--infile', dest='infile',
435 help='Input file for import. Default is stdin'
436 )
437 self.parser.add_option('--noindex', dest='noindex',
438 action='store_true', default=False,
439 help='Do not try to index the freshly loaded objects.'
440 )
441 self.parser.add_option('--chunksize', dest='chunk_size',
442 help='Number of objects to commit at a time.',
443 type='int',
444 default=100
445 )
446 self.parser.add_option(
447 '-n',
448 '--noCommit',
449 dest='noCommit',
450 action='store_true',
451 default=0,
452 help='Do not store changes to the DMD (ie for debugging purposes)',
453 )
454
456 """
457 This method can be used to load data for the root of Zenoss (default
458 behavior) or it can be used to operate on a specific point in the
459 Zenoss hierarchy (ZODB).
460
461 Upon loading the XML file to be processed, the content of the XML file
462 is handled by the methods in this class when called by xml.sax.parse().
463
464 Reads from a file if xmlfile is specified, otherwise reads
465 from the command-line option --infile. If no files are found from
466 any of these places, read from standard input.
467
468 @param xmlfile: Name of XML file to load, or file-like object
469 @type xmlfile: string or file-like object
470 """
471 self.objstack = [self.app]
472 self.links = []
473 self.objectnumber = 0
474 self.charvalue = ''
475 if xmlfile and isinstance(xmlfile, basestring):
476 self.infile = open(xmlfile)
477 elif hasattr(xmlfile, 'read'):
478 self.infile = xmlfile
479 elif self.options.infile:
480 self.infile = open(self.options.infile)
481 else:
482 self.infile = sys.stdin
483 parser = make_parser()
484 parser.setContentHandler(self)
485 try:
486 parser.parse(self.infile)
487 except SAXParseException, ex:
488 self.log.error("XML parse error at line %d column %d: %s",
489 ex.getLineNumber(), ex.getColumnNumber(),
490 ex.getMessage())
491 finally:
492 self.infile.close()
493
495 """
496 The default behavior of loadObjectFromXML() will be to use the Zope
497 app object, and thus operatate on the whole of Zenoss.
498 """
499 self.loadObjectFromXML()
500
502 """
503 Wrapper around the Zope database commit()
504 """
505 trans = transaction.get()
506 trans.note('Import from file %s using %s'
507 % (self.options.infile, self.__class__.__name__))
508 trans.commit()
509 if hasattr(self, 'connection'):
510
511 self.syncdb()
512
514 """
515 SpoofedOptions
516 """
517
519 self.infile = ''
520 self.noCommit = True
521 self.noindex = True
522 self.dataroot = '/zport/dmd'
523
524
526 """
527 An ImportRM that does not call the __init__ method on ZCmdBase
528 """
529
531 """
532 Initializer
533
534 @param app: app
535 @type app: string
536 """
537 import Products.ZenossStartup
538 from Products.Five import zcml
539 zcml.load_site()
540 import logging
541 self.log = logging.getLogger('zen.ImportRM')
542 self.app = app
543 ContentHandler.__init__(self)
544 self.options = SpoofedOptions()
545 self.dataroot = None
546 self.getDataRoot()
547
548
549 if __name__ == '__main__':
550 im = ImportRM()
551 im.loadDatabase()
552