Package Products :: Package ZenRRD :: Module RenderServer
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenRRD.RenderServer

  1  ########################################################################### 
  2  # 
  3  # This program is part of Zenoss Core, an open source monitoring platform. 
  4  # Copyright (C) 2007, Zenoss Inc. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify it 
  7  # under the terms of the GNU General Public License version 2 or (at your 
  8  # option) any later version as published by the Free Software Foundation. 
  9  # 
 10  # For complete information please visit: http://www.zenoss.com/oss/ 
 11  # 
 12  ########################################################################### 
 13   
 14  __doc__="""RenderServer 
 15   
 16  Frontend that passes RRD graph options to rrdtool to render, 
 17  and then returns an URL to access the rendered graphic file. 
 18  """ 
 19   
 20  import os 
 21  import time 
 22  import logging 
 23  import json 
 24  import urllib 
 25  import zlib 
 26  import mimetypes 
 27  import glob 
 28  import tarfile 
 29  import md5 
 30   
 31  from AccessControl import ClassSecurityInfo 
 32  from Globals import InitializeClass 
 33  from Globals import DTMLFile 
 34   
 35  try: 
 36      import rrdtool 
 37  except ImportError: 
 38      pass 
 39   
 40  from base64 import b64encode, urlsafe_b64decode 
 41   
 42  from Products.ZenRRD.RRDUtil import fixMissingRRDs 
 43  from Products.ZenUtils.PObjectCache import PObjectCache 
 44  from Products.ZenUtils.Utils import zenPath, rrd_daemon_running 
 45   
 46  from RRDToolItem import RRDToolItem 
 47   
 48  from Products.ZenModel.PerformanceConf import performancePath 
 49   
 50  log = logging.getLogger("RenderServer") 
 51   
 52   
53 -def manage_addRenderServer(context, id, REQUEST = None):
54 """ 55 Make a RenderServer 56 """ 57 rs = RenderServer(id) 58 context._setObject(id, rs) 59 if REQUEST is not None: 60 REQUEST['RESPONSE'].redirect(context.absolute_url()+'/manage_main')
61 62 63 addRenderServer = DTMLFile('dtml/addRenderServer',globals()) 64 65
66 -class RenderServer(RRDToolItem):
67 """ 68 Base class for turning graph requests into graphics. 69 NB: Any log messages get logged into the event.log file. 70 """ 71 72 meta_type = "RenderServer" 73 74 cacheName = 'RRDRenderCache' 75 76 security = ClassSecurityInfo() 77
78 - def __init__(self, id, tmpdir = '/tmp/renderserver', cachetimeout=300):
79 self.id = id 80 self.tmpdir = tmpdir 81 self.cachetimeout = cachetimeout
82 83 84 security.declareProtected('View', 'render')
85 - def render(self, gopts=None, start=None, end=None, drange=None, 86 remoteUrl=None, width=None, ftype='PNG', getImage=True, 87 graphid='', comment=None, ms=None, REQUEST=None, zenrenderRequest=None):
88 """ 89 Render a graph and return it 90 91 @param gopts: RRD graph creation options 92 @param start: requested start of data to graph 93 @param end: requested start of data to graph 94 @param drange: min/max values of the graph 95 @param remoteUrl: if the RRD is not here, where it lives 96 @param width: size of graphic to create 97 @param ftype: file type of graphic (eg PNG) 98 @param getImage: return the graph or a script location 99 @param graphid: (hopefully) unique identifier of a graph 100 @param comment: RRD graph comment 101 @param ms: a timestamp used to force IE to reload images 102 @param REQUEST: URL-marshalled object containg URL options 103 @return: graph or script location 104 """ 105 106 # gopts may have repeated url quoting, possibly from multiple hops thru remote zenhubs 107 # extra quoting will create invalid zlib padding characters ('%3D' instead of '=') 108 for tries in range(3): 109 try: 110 gopts = zlib.decompress(urlsafe_b64decode(gopts)) 111 except Exception: 112 gopts = urllib.unquote(gopts) 113 else: 114 break 115 116 comment = urllib.unquote(comment) if comment is not None else '' 117 118 gopts = gopts.split('|') 119 gopts = fixMissingRRDs(gopts) 120 gopts.append('HRULE:INF#00000000') 121 gopts.append('--width=%s' % width) 122 if start: 123 gopts.append('--start=%s' % start) 124 if end: 125 gopts.append('--end=%s' % end) 126 drange = int(drange) 127 id = self.graphId(gopts, drange, ftype) 128 graph = self.getGraph(id, ftype, REQUEST) 129 if not graph: 130 if not os.path.exists(self.tmpdir): 131 os.makedirs(self.tmpdir, 0750) 132 filename = "%s/graph-%s" % (self.tmpdir,id) 133 if remoteUrl: 134 f = open(filename, "w") 135 f.write(urllib.urlopen(remoteUrl).read()) 136 f.close() 137 else: 138 if ftype.lower()=='html': 139 imgtype = 'PNG' 140 else: 141 imgtype = ftype 142 daemon = rrd_daemon_running() 143 if daemon: 144 gopts.insert(0, '--daemon=%s' % daemon) 145 gopts.insert(0, "--imgformat=%s" % imgtype) 146 #gopts.insert(0, "--lazy") 147 end = int(time.time())-300 148 start = end - drange 149 if comment is not None: 150 gopts.insert(0, 'COMMENT:%s\\c' % comment) 151 gopts.insert(0, '--end=%d' % end) 152 gopts.insert(0, '--start=%d' % start) 153 gopts.insert(0, filename) 154 log.debug("RRD graphing options: %r", (gopts,)) 155 try: 156 rrdtool.graph(*gopts) 157 except Exception, ex: 158 if ex.args[0].find('No such file or directory') > -1: 159 return None 160 log.exception("Failed to generate a graph") 161 log.warn(" ".join(gopts)) 162 return None 163 164 self.addGraph(id, filename) 165 graph = self.getGraph(id, ftype, REQUEST) 166 167 if getImage: 168 return graph 169 else: 170 success = bool(graph) 171 ret = {'success':success} 172 if success: 173 ret['data'] = b64encode(graph) 174 if REQUEST: 175 REQUEST.RESPONSE.setHeader('Content-Type', 'text/javascript') 176 elif zenrenderRequest: 177 zenrenderRequest.setHeader('Content-Type', 'text/javascript') 178 return """Zenoss.SWOOP_CALLBACKS["%s"]('%s')""" % (graphid, json.dumps(ret))
179 180
181 - def deleteRRDFiles(self, device, 182 datasource=None, datapoint=None, 183 remoteUrl=None, REQUEST=None):
184 """ 185 Delete RRD files associated with the given device id. 186 If datapoint is not None then delete the file corresponding to that dp. 187 Else if datasource is not None then delete the files corresponding to 188 all datapoints in the datasource. 189 Else delete all RRD files associated with the given device. 190 191 @param device: device name 192 @param datasource: RRD datasource (DS) name 193 @param datapoint: RRD datapoint name (lives in a DS) 194 @param remoteUrl: if the RRD is not here, where it lives 195 @param REQUEST: URL-marshalled object containg URL options 196 """ 197 devDir = performancePath('/Devices/%s' % device) 198 if not os.path.isdir(devDir): 199 return 200 fileNames = [] 201 dirNames = [] 202 if datapoint: 203 fileNames = [ 204 performancePath('/Devices/%s/%s.rrd' % (device, datapoint))] 205 elif datasource: 206 rrdPath = '/Devices/%s/%s_*.rrd' % (device, datasource) 207 fileNames = glob.glob(performancePath(rrdPath)) 208 else: 209 for dPath, dNames, dFiles in os.walk(devDir, topdown=False): 210 fileNames += [os.path.join(dPath, f) for f in dFiles] 211 dirNames += [os.path.join(dPath, d) for d in dNames] 212 dirNames.append(devDir) 213 for fileName in fileNames: 214 try: 215 os.remove(fileName) 216 except OSError: 217 log.warn("File %s does not exist" % fileName) 218 for dirName in dirNames: 219 try: 220 os.rmdir(dirName) 221 except OSError: 222 log.warn('Directory %s could not be removed' % dirName) 223 if remoteUrl: 224 urllib.urlopen(remoteUrl)
225 226
227 - def packageRRDFiles(self, device, REQUEST=None):
228 """ 229 Tar up RRD files into a nice, neat package 230 231 @param device: device name 232 @param REQUEST: URL-marshalled object containg URL options 233 """ 234 srcdir = performancePath('/Devices/%s' % device) 235 tarfilename = '%s/%s.tgz' % (self.tmpdir, device) 236 log.debug( "tarring up %s into %s" % ( srcdir, tarfilename )) 237 tar = tarfile.open(tarfilename, "w:gz") 238 for file in os.listdir(srcdir): 239 tar.add('%s/%s' % (srcdir, file), '/%s' % os.path.basename(file)) 240 tar.close()
241
242 - def unpackageRRDFiles(self, device, REQUEST=None):
243 """ 244 Untar a package of RRDFiles 245 246 @param device: device name 247 @param REQUEST: URL-marshalled object containg URL options 248 """ 249 destdir = performancePath('/Devices/%s' % device) 250 tarfilename = '%s/%s.tgz' % (self.tmpdir, device) 251 log.debug( "Untarring %s into %s" % ( tarfilename, destdir )) 252 tar = tarfile.open(tarfilename, "r:gz") 253 for file in tar.getmembers(): 254 tar.extract(file, destdir) 255 tar.close()
256
257 - def receiveRRDFiles(self, REQUEST=None):
258 """ 259 Receive a device's RRD Files from another server 260 This function is called by sendRRDFiles() 261 262 @param REQUEST: 'tarfile', 'tarfilename' 263 @type REQUEST: URL-marshalled parameters 264 """ 265 tarfile = REQUEST.get('tarfile') 266 tarfilename = REQUEST.get('tarfilename') 267 log.debug( "Receiving %s ..." % ( tarfilename )) 268 f=open('%s/%s' % (self.tmpdir, tarfilename), 'wb') 269 f.write(urllib.unquote(tarfile)) 270 f.close()
271
272 - def sendRRDFiles(self, device, server, REQUEST=None):
273 """ 274 Move a package of RRDFiles 275 276 @param device: device name 277 @param server: another RenderServer instance 278 @param REQUEST: URL-marshalled object containg URL options 279 """ 280 tarfilename = '%s.tgz' % device 281 f=open('%s/%s' % (self.tmpdir, tarfilename), 'rb') 282 tarfilebody=f.read() 283 f.close() 284 # urlencode the id, title and file 285 params = urllib.urlencode({'tarfilename': tarfilename, 286 'tarfile':tarfilebody}) 287 288 # send the file to Zope 289 perfMon = self.dmd.getDmdRoot("Monitors").getPerformanceMonitor(server) 290 if perfMon.renderurl.startswith('http'): 291 remoteUrl = '%s/receiveRRDFiles' % (perfMon.renderurl) 292 log.debug( "Sending %s to %s ..." % ( tarfilename, remoteUrl )) 293 urllib.urlopen(remoteUrl, params)
294 295
296 - def moveRRDFiles(self, device, destServer, srcServer=None, REQUEST=None):
297 """ 298 Send a device's RRD files to another server 299 300 @param device: device name 301 @param destServer: another RenderServer instance 302 @param srcServer: another RenderServer instance 303 @param REQUEST: URL-marshalled object containg URL options 304 """ 305 monitors = self.dmd.getDmdRoot("Monitors") 306 destPerfMon = monitors.getPerformanceMonitor(destServer) 307 if srcServer: 308 srcPerfMon = monitors.getPerformanceMonitor(srcServer) 309 remoteUrl = '%s/moveRRDFiles?device=%s&destServer=%s' % (srcPerfMon.renderurl, device, destServer) 310 urllib.urlopen(remoteUrl) 311 312 else: 313 self.packageRRDFiles(device, REQUEST) 314 self.sendRRDFiles(device, destServer, REQUEST) 315 if destPerfMon.renderurl.startswith('http'): 316 remoteUrl = '%s/unpackageRRDFiles?device=%s' % (destPerfMon.renderurl, device) 317 urllib.urlopen(remoteUrl) 318 else: 319 self.unpackageRRDFiles(device, REQUEST)
320 321 security.declareProtected('View', 'plugin')
322 - def plugin(self, name, REQUEST=None):
323 """ 324 Render a custom graph and return it 325 326 @param name: plugin name from Products/ZenRRD/plugins 327 @return: graph or None 328 """ 329 try: 330 m = zenPath('Products/ZenRRD/plugins/%s.py' % name) 331 log.debug( "Trying plugin %s to generate a graph..." % m ) 332 graph = None 333 exec open(m) 334 return graph 335 except Exception: 336 log.exception("Failed generating graph from plugin %s" % name) 337 raise
338 339 340 security.declareProtected('GenSummary', 'summary')
341 - def summary(self, gopts):
342 """ 343 Return summary information as a list but no graph 344 345 @param gopts: RRD graph options 346 @return: values from the graph 347 """ 348 gopts = fixMissingRRDs(gopts) 349 gopts.insert(0, '/dev/null') #no graph generated 350 daemon = rrd_daemon_running() 351 if daemon: 352 gopts.insert(0, '--daemon=%s' % daemon) 353 try: 354 values = rrdtool.graph(*gopts)[2] 355 except Exception, ex: 356 if ex.args[0].find('No such file or directory') > -1: 357 return None 358 log.exception("Failed while generating summary") 359 log.warn(" ".join(gopts)) 360 raise 361 return values
362 363 364 security.declareProtected('GenSummary', 'fetchValues')
365 - def fetchValues(self, paths, cf, resolution, start, end=""):
366 """ 367 Return the values recorded in the RRD file between the start and end period 368 369 @param paths: path names to files 370 @param cf: RRD consolidation function to use 371 @param resolution: requested resolution of RRD data 372 @param start: requested start of data to graph 373 @param end: requested start of data to graph 374 @return: values from the RRD files in the paths 375 """ 376 if not end: 377 end = "now" 378 values = [] 379 daemon = rrd_daemon_running() 380 args = ('--daemon', daemon) if daemon else () 381 try: 382 for path in paths: 383 values.append(rrdtool.fetch(path, cf, "-r %d" % resolution, 384 "-s %s" % start,"-e %s" % end, *args)) 385 return values 386 except NameError: 387 log.exception("It appears that the rrdtool bindings are not installed properly.") 388 except Exception, ex: 389 if ex.args[0].find('No such file or directory') > -1: 390 return None 391 log.exception("Failed while generating current values") 392 raise
393 394 395 security.declareProtected('GenSummary', 'currentValues')
396 - def currentValues(self, paths):
397 """ 398 Return the latest values recorded in the RRD file 399 400 @param paths: path names to files 401 @return: values from the RRD files in the path 402 """ 403 try: 404 def value(p): 405 v = None 406 info = None 407 try: 408 info = rrdtool.info(p) 409 except: 410 log.debug('%s not found' % p) 411 if info: 412 last = info['last_update'] 413 step = info['step'] 414 v = rrdtool.graph('/dev/null', 415 'DEF:x=%s:ds0:AVERAGE' % p, 416 'VDEF:v=x,LAST', 417 'PRINT:v:%.2lf', 418 '--start=%d'%(last-step), 419 '--end=%d'%last) 420 v = float(v[2][0]) 421 if str(v) == 'nan': v = None 422 return v
423 return map(value, paths) 424 425 except NameError: 426 log.exception("It appears that the rrdtool bindings are not installed properly.") 427 428 except Exception, ex: 429 if ex.args[0].find('No such file or directory') > -1: 430 return None 431 log.exception("Failed while generating current values") 432 raise
433
434 - def rrdcmd(self, gopts, ftype='PNG'):
435 """ 436 Generate the RRD command using the graphing options specified. 437 438 @param gopts: RRD graphing options 439 @param ftype: graphic file type (eg PNG) 440 @return: RRD command usable on the command-line 441 @rtype: string 442 """ 443 filename, gopts = self._setfile(gopts, ftype) 444 return "rrdtool graph " + " ".join(gopts)
445 446
447 - def graphId(self, gopts, drange, ftype):
448 """ 449 Generate a graph id based on a hash of values 450 451 @param gopts: RRD graphing options 452 @param drange: min/max values of the graph 453 @param ftype: graphic file's type (eg PNG) 454 @return: An id for this graph usable in URLs 455 @rtype: string 456 """ 457 id = md5.new(''.join(gopts)).hexdigest() 458 id += str(drange) + '.' + ftype.lower() 459 return id
460
461 - def _loadfile(self, filename):
462 try: 463 f = open(filename) 464 graph = f.read() 465 f.close() 466 return graph 467 except IOError: 468 log.info("File: %s not created yet." % filename); 469 return None
470 471
472 - def setupCache(self):
473 """ 474 Make a new cache if we need one 475 """ 476 if not hasattr(self, '_v_cache') or not self._v_cache: 477 tmpfolder = self.getPhysicalRoot().temp_folder 478 if not hasattr(tmpfolder, self.cacheName): 479 cache = PObjectCache(self.cacheName, self.cachetimeout) 480 tmpfolder._setObject(self.cacheName, cache) 481 self._v_cache = tmpfolder._getOb(self.cacheName) 482 return self._v_cache
483 484
485 - def addGraph(self, id, filename):
486 """ 487 Add a graph to temporary folder 488 489 @param id: graph id 490 @param filename: cacheable graphic file 491 """ 492 cache = self.setupCache() 493 graph = self._loadfile(filename) 494 if graph: 495 cache.addToCache(id, graph) 496 try: 497 os.remove(filename) 498 except OSError, e: 499 if e.errno == 2: 500 log.debug("Unable to remove cached graph %s: %s" \ 501 % (e.strerror, e.filename)) 502 else: 503 raise e 504 cache.cleanCache()
505 506
507 - def getGraph(self, id, ftype, REQUEST):
508 """ 509 Get a previously generated graph 510 511 @param id: graph id 512 @param ftype: file type of graphic (eg PNG) 513 @param REQUEST: graph id 514 """ 515 cache = self.setupCache() 516 ftype = ftype.lower() 517 518 if REQUEST: 519 mimetype = mimetypes.guess_type('%s.%s' % (id, ftype))[0] 520 if mimetype is None: 521 mimetype = 'image/%s' % ftype 522 response = REQUEST.RESPONSE 523 response.setHeader('Content-Type', mimetype) 524 525 return cache.checkCache(id)
526 527 528 InitializeClass(RenderServer) 529