1
2
3
4
5
6
7
8
9
10
11
12
13
14 __doc__="""RenderServer
15
16 Frontend that passes RRD graph options to rrdtool to render,
17 and then returns an URL to access the rendered graphic file.
18 """
19
20 import os
21 import time
22 import logging
23 import json
24 import urllib
25 import zlib
26 import mimetypes
27 import glob
28 import tarfile
29 import md5
30
31 from AccessControl import ClassSecurityInfo
32 from Globals import InitializeClass
33 from Globals import DTMLFile
34
35 try:
36 import rrdtool
37 except ImportError:
38 pass
39
40 from base64 import b64encode, urlsafe_b64decode
41
42 from Products.ZenRRD.RRDUtil import fixMissingRRDs
43 from Products.ZenUtils.PObjectCache import PObjectCache
44 from Products.ZenUtils.Utils import zenPath, rrd_daemon_running
45
46 from RRDToolItem import RRDToolItem
47
48 from Products.ZenModel.PerformanceConf import performancePath
49
50 log = logging.getLogger("RenderServer")
51
52
61
62
63 addRenderServer = DTMLFile('dtml/addRenderServer',globals())
64
65
67 """
68 Base class for turning graph requests into graphics.
69 NB: Any log messages get logged into the event.log file.
70 """
71
72 meta_type = "RenderServer"
73
74 cacheName = 'RRDRenderCache'
75
76 security = ClassSecurityInfo()
77
78 - def __init__(self, id, tmpdir = '/tmp/renderserver', cachetimeout=300):
79 self.id = id
80 self.tmpdir = tmpdir
81 self.cachetimeout = cachetimeout
82
83
84 security.declareProtected('View', 'render')
85 - def render(self, gopts=None, start=None, end=None, drange=None,
86 remoteUrl=None, width=None, ftype='PNG', getImage=True,
87 graphid='', comment=None, ms=None, REQUEST=None, zenrenderRequest=None):
88 """
89 Render a graph and return it
90
91 @param gopts: RRD graph creation options
92 @param start: requested start of data to graph
93 @param end: requested start of data to graph
94 @param drange: min/max values of the graph
95 @param remoteUrl: if the RRD is not here, where it lives
96 @param width: size of graphic to create
97 @param ftype: file type of graphic (eg PNG)
98 @param getImage: return the graph or a script location
99 @param graphid: (hopefully) unique identifier of a graph
100 @param comment: RRD graph comment
101 @param ms: a timestamp used to force IE to reload images
102 @param REQUEST: URL-marshalled object containg URL options
103 @return: graph or script location
104 """
105
106
107
108 for tries in range(3):
109 try:
110 gopts = zlib.decompress(urlsafe_b64decode(gopts))
111 except Exception:
112 gopts = urllib.unquote(gopts)
113 else:
114 break
115
116 comment = urllib.unquote(comment) if comment is not None else ''
117
118 gopts = gopts.split('|')
119 gopts = fixMissingRRDs(gopts)
120 gopts.append('HRULE:INF#00000000')
121 gopts.append('--width=%s' % width)
122 if start:
123 gopts.append('--start=%s' % start)
124 if end:
125 gopts.append('--end=%s' % end)
126 drange = int(drange)
127 id = self.graphId(gopts, drange, ftype)
128 graph = self.getGraph(id, ftype, REQUEST)
129 if not graph:
130 if not os.path.exists(self.tmpdir):
131 os.makedirs(self.tmpdir, 0750)
132 filename = "%s/graph-%s" % (self.tmpdir,id)
133 if remoteUrl:
134 f = open(filename, "w")
135 f.write(urllib.urlopen(remoteUrl).read())
136 f.close()
137 else:
138 if ftype.lower()=='html':
139 imgtype = 'PNG'
140 else:
141 imgtype = ftype
142 daemon = rrd_daemon_running()
143 if daemon:
144 gopts.insert(0, '--daemon=%s' % daemon)
145 gopts.insert(0, "--imgformat=%s" % imgtype)
146
147 end = int(time.time())-300
148 start = end - drange
149 if comment is not None:
150 gopts.insert(0, 'COMMENT:%s\\c' % comment)
151 gopts.insert(0, '--end=%d' % end)
152 gopts.insert(0, '--start=%d' % start)
153 gopts.insert(0, filename)
154 log.debug("RRD graphing options: %r", (gopts,))
155 try:
156 rrdtool.graph(*gopts)
157 except Exception, ex:
158 if ex.args[0].find('No such file or directory') > -1:
159 return None
160 log.exception("Failed to generate a graph")
161 log.warn(" ".join(gopts))
162 return None
163
164 self.addGraph(id, filename)
165 graph = self.getGraph(id, ftype, REQUEST)
166
167 if getImage:
168 return graph
169 else:
170 success = bool(graph)
171 ret = {'success':success}
172 if success:
173 ret['data'] = b64encode(graph)
174 if REQUEST:
175 REQUEST.RESPONSE.setHeader('Content-Type', 'text/javascript')
176 elif zenrenderRequest:
177 zenrenderRequest.setHeader('Content-Type', 'text/javascript')
178 return """Zenoss.SWOOP_CALLBACKS["%s"]('%s')""" % (graphid, json.dumps(ret))
179
180
181 - def deleteRRDFiles(self, device,
182 datasource=None, datapoint=None,
183 remoteUrl=None, REQUEST=None):
184 """
185 Delete RRD files associated with the given device id.
186 If datapoint is not None then delete the file corresponding to that dp.
187 Else if datasource is not None then delete the files corresponding to
188 all datapoints in the datasource.
189 Else delete all RRD files associated with the given device.
190
191 @param device: device name
192 @param datasource: RRD datasource (DS) name
193 @param datapoint: RRD datapoint name (lives in a DS)
194 @param remoteUrl: if the RRD is not here, where it lives
195 @param REQUEST: URL-marshalled object containg URL options
196 """
197 devDir = performancePath('/Devices/%s' % device)
198 if not os.path.isdir(devDir):
199 return
200 fileNames = []
201 dirNames = []
202 if datapoint:
203 fileNames = [
204 performancePath('/Devices/%s/%s.rrd' % (device, datapoint))]
205 elif datasource:
206 rrdPath = '/Devices/%s/%s_*.rrd' % (device, datasource)
207 fileNames = glob.glob(performancePath(rrdPath))
208 else:
209 for dPath, dNames, dFiles in os.walk(devDir, topdown=False):
210 fileNames += [os.path.join(dPath, f) for f in dFiles]
211 dirNames += [os.path.join(dPath, d) for d in dNames]
212 dirNames.append(devDir)
213 for fileName in fileNames:
214 try:
215 os.remove(fileName)
216 except OSError:
217 log.warn("File %s does not exist" % fileName)
218 for dirName in dirNames:
219 try:
220 os.rmdir(dirName)
221 except OSError:
222 log.warn('Directory %s could not be removed' % dirName)
223 if remoteUrl:
224 urllib.urlopen(remoteUrl)
225
226
228 """
229 Tar up RRD files into a nice, neat package
230
231 @param device: device name
232 @param REQUEST: URL-marshalled object containg URL options
233 """
234 srcdir = performancePath('/Devices/%s' % device)
235 tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
236 log.debug( "tarring up %s into %s" % ( srcdir, tarfilename ))
237 tar = tarfile.open(tarfilename, "w:gz")
238 for file in os.listdir(srcdir):
239 tar.add('%s/%s' % (srcdir, file), '/%s' % os.path.basename(file))
240 tar.close()
241
243 """
244 Untar a package of RRDFiles
245
246 @param device: device name
247 @param REQUEST: URL-marshalled object containg URL options
248 """
249 destdir = performancePath('/Devices/%s' % device)
250 tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
251 log.debug( "Untarring %s into %s" % ( tarfilename, destdir ))
252 tar = tarfile.open(tarfilename, "r:gz")
253 for file in tar.getmembers():
254 tar.extract(file, destdir)
255 tar.close()
256
258 """
259 Receive a device's RRD Files from another server
260 This function is called by sendRRDFiles()
261
262 @param REQUEST: 'tarfile', 'tarfilename'
263 @type REQUEST: URL-marshalled parameters
264 """
265 tarfile = REQUEST.get('tarfile')
266 tarfilename = REQUEST.get('tarfilename')
267 log.debug( "Receiving %s ..." % ( tarfilename ))
268 f=open('%s/%s' % (self.tmpdir, tarfilename), 'wb')
269 f.write(urllib.unquote(tarfile))
270 f.close()
271
273 """
274 Move a package of RRDFiles
275
276 @param device: device name
277 @param server: another RenderServer instance
278 @param REQUEST: URL-marshalled object containg URL options
279 """
280 tarfilename = '%s.tgz' % device
281 f=open('%s/%s' % (self.tmpdir, tarfilename), 'rb')
282 tarfilebody=f.read()
283 f.close()
284
285 params = urllib.urlencode({'tarfilename': tarfilename,
286 'tarfile':tarfilebody})
287
288
289 perfMon = self.dmd.getDmdRoot("Monitors").getPerformanceMonitor(server)
290 if perfMon.renderurl.startswith('http'):
291 remoteUrl = '%s/receiveRRDFiles' % (perfMon.renderurl)
292 log.debug( "Sending %s to %s ..." % ( tarfilename, remoteUrl ))
293 urllib.urlopen(remoteUrl, params)
294
295
296 - def moveRRDFiles(self, device, destServer, srcServer=None, REQUEST=None):
297 """
298 Send a device's RRD files to another server
299
300 @param device: device name
301 @param destServer: another RenderServer instance
302 @param srcServer: another RenderServer instance
303 @param REQUEST: URL-marshalled object containg URL options
304 """
305 monitors = self.dmd.getDmdRoot("Monitors")
306 destPerfMon = monitors.getPerformanceMonitor(destServer)
307 if srcServer:
308 srcPerfMon = monitors.getPerformanceMonitor(srcServer)
309 remoteUrl = '%s/moveRRDFiles?device=%s&destServer=%s' % (srcPerfMon.renderurl, device, destServer)
310 urllib.urlopen(remoteUrl)
311
312 else:
313 self.packageRRDFiles(device, REQUEST)
314 self.sendRRDFiles(device, destServer, REQUEST)
315 if destPerfMon.renderurl.startswith('http'):
316 remoteUrl = '%s/unpackageRRDFiles?device=%s' % (destPerfMon.renderurl, device)
317 urllib.urlopen(remoteUrl)
318 else:
319 self.unpackageRRDFiles(device, REQUEST)
320
321 security.declareProtected('View', 'plugin')
322 - def plugin(self, name, REQUEST=None):
323 """
324 Render a custom graph and return it
325
326 @param name: plugin name from Products/ZenRRD/plugins
327 @return: graph or None
328 """
329 try:
330 m = zenPath('Products/ZenRRD/plugins/%s.py' % name)
331 log.debug( "Trying plugin %s to generate a graph..." % m )
332 graph = None
333 exec open(m)
334 return graph
335 except Exception:
336 log.exception("Failed generating graph from plugin %s" % name)
337 raise
338
339
340 security.declareProtected('GenSummary', 'summary')
342 """
343 Return summary information as a list but no graph
344
345 @param gopts: RRD graph options
346 @return: values from the graph
347 """
348 gopts = fixMissingRRDs(gopts)
349 gopts.insert(0, '/dev/null')
350 daemon = rrd_daemon_running()
351 if daemon:
352 gopts.insert(0, '--daemon=%s' % daemon)
353 try:
354 values = rrdtool.graph(*gopts)[2]
355 except Exception, ex:
356 if ex.args[0].find('No such file or directory') > -1:
357 return None
358 log.exception("Failed while generating summary")
359 log.warn(" ".join(gopts))
360 raise
361 return values
362
363
364 security.declareProtected('GenSummary', 'fetchValues')
365 - def fetchValues(self, paths, cf, resolution, start, end=""):
366 """
367 Return the values recorded in the RRD file between the start and end period
368
369 @param paths: path names to files
370 @param cf: RRD consolidation function to use
371 @param resolution: requested resolution of RRD data
372 @param start: requested start of data to graph
373 @param end: requested start of data to graph
374 @return: values from the RRD files in the paths
375 """
376 if not end:
377 end = "now"
378 values = []
379 daemon = rrd_daemon_running()
380 args = ('--daemon', daemon) if daemon else ()
381 try:
382 for path in paths:
383 values.append(rrdtool.fetch(path, cf, "-r %d" % resolution,
384 "-s %s" % start,"-e %s" % end, *args))
385 return values
386 except NameError:
387 log.exception("It appears that the rrdtool bindings are not installed properly.")
388 except Exception, ex:
389 if ex.args[0].find('No such file or directory') > -1:
390 return None
391 log.exception("Failed while generating current values")
392 raise
393
394
395 security.declareProtected('GenSummary', 'currentValues')
397 """
398 Return the latest values recorded in the RRD file
399
400 @param paths: path names to files
401 @return: values from the RRD files in the path
402 """
403 try:
404 def value(p):
405 v = None
406 info = None
407 try:
408 info = rrdtool.info(p)
409 except:
410 log.debug('%s not found' % p)
411 if info:
412 last = info['last_update']
413 step = info['step']
414 v = rrdtool.graph('/dev/null',
415 'DEF:x=%s:ds0:AVERAGE' % p,
416 'VDEF:v=x,LAST',
417 'PRINT:v:%.2lf',
418 '--start=%d'%(last-step),
419 '--end=%d'%last)
420 v = float(v[2][0])
421 if str(v) == 'nan': v = None
422 return v
423 return map(value, paths)
424
425 except NameError:
426 log.exception("It appears that the rrdtool bindings are not installed properly.")
427
428 except Exception, ex:
429 if ex.args[0].find('No such file or directory') > -1:
430 return None
431 log.exception("Failed while generating current values")
432 raise
433
434 - def rrdcmd(self, gopts, ftype='PNG'):
435 """
436 Generate the RRD command using the graphing options specified.
437
438 @param gopts: RRD graphing options
439 @param ftype: graphic file type (eg PNG)
440 @return: RRD command usable on the command-line
441 @rtype: string
442 """
443 filename, gopts = self._setfile(gopts, ftype)
444 return "rrdtool graph " + " ".join(gopts)
445
446
447 - def graphId(self, gopts, drange, ftype):
448 """
449 Generate a graph id based on a hash of values
450
451 @param gopts: RRD graphing options
452 @param drange: min/max values of the graph
453 @param ftype: graphic file's type (eg PNG)
454 @return: An id for this graph usable in URLs
455 @rtype: string
456 """
457 id = md5.new(''.join(gopts)).hexdigest()
458 id += str(drange) + '.' + ftype.lower()
459 return id
460
462 try:
463 f = open(filename)
464 graph = f.read()
465 f.close()
466 return graph
467 except IOError:
468 log.info("File: %s not created yet." % filename);
469 return None
470
471
473 """
474 Make a new cache if we need one
475 """
476 if not hasattr(self, '_v_cache') or not self._v_cache:
477 tmpfolder = self.getPhysicalRoot().temp_folder
478 if not hasattr(tmpfolder, self.cacheName):
479 cache = PObjectCache(self.cacheName, self.cachetimeout)
480 tmpfolder._setObject(self.cacheName, cache)
481 self._v_cache = tmpfolder._getOb(self.cacheName)
482 return self._v_cache
483
484
486 """
487 Add a graph to temporary folder
488
489 @param id: graph id
490 @param filename: cacheable graphic file
491 """
492 cache = self.setupCache()
493 graph = self._loadfile(filename)
494 if graph:
495 cache.addToCache(id, graph)
496 try:
497 os.remove(filename)
498 except OSError, e:
499 if e.errno == 2:
500 log.debug("Unable to remove cached graph %s: %s" \
501 % (e.strerror, e.filename))
502 else:
503 raise e
504 cache.cleanCache()
505
506
507 - def getGraph(self, id, ftype, REQUEST):
508 """
509 Get a previously generated graph
510
511 @param id: graph id
512 @param ftype: file type of graphic (eg PNG)
513 @param REQUEST: graph id
514 """
515 cache = self.setupCache()
516 ftype = ftype.lower()
517
518 if REQUEST:
519 mimetype = mimetypes.guess_type('%s.%s' % (id, ftype))[0]
520 if mimetype is None:
521 mimetype = 'image/%s' % ftype
522 response = REQUEST.RESPONSE
523 response.setHeader('Content-Type', mimetype)
524
525 return cache.checkCache(id)
526
527
528 InitializeClass(RenderServer)
529