Package Products :: Package ZenModel :: Module zenmib
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenModel.zenmib

   1  ########################################################################### 
   2  # 
   3  # This program is part of Zenoss Core, an open source monitoring platform. 
   4  # Copyright (C) 2007, 2009 Zenoss Inc. 
   5  # 
   6  # This program is free software; you can redistribute it and/or modify it 
   7  # under the terms of the GNU General Public License version 2 or (at your 
   8  # option) any later version as published by the Free Software Foundation. 
   9  # 
  10  # For complete information please visit: http://www.zenoss.com/oss/ 
  11  # 
  12  ########################################################################### 
  13  __doc__ = """zenmib 
  14     The zenmib program converts MIBs into python data structures and then 
  15  (by default) adds the data to the Zenoss DMD.  Essentially, zenmib is a 
  16  wrapper program around the smidump program, whose output (python code) is 
  17  then executed "inside" the Zope database. 
  18   
  19   Overview of terms: 
  20     SNMP Simple Network Management Protocol 
  21        A network protocol originally based on UDP which allows a management 
  22        application (ie SNMP manager) to contact and get information from a 
  23        device (ie router, computer, network-capable toaster).  The software 
  24        on the device that is capable of understanding SNMP and responding 
  25        appropriately is called an SNMP agent. 
  26   
  27     MIB Management of Information Base 
  28        A description of what a particular SNMP agent provides and what 
  29        traps it sends.  A MIB is a part of a tree structure based on a root 
  30        MIB.  Since a MIB is a rooted tree, it allows for delegation of areas 
  31        under the tree to different organizations. 
  32   
  33     ASN Abstract Syntax Notation 
  34        The notation used to construct a MIB. 
  35   
  36     OID Object IDentifier 
  37        A MIB is constructed of unique identifiers 
  38   
  39   
  40   Background information: 
  41     http://en.wikipedia.org/wiki/Simple_Network_Management_Protocol 
  42         Overview of SNMP. 
  43   
  44     http://www.ibr.cs.tu-bs.de/projects/libsmi/index.html?lang=en 
  45         The libsmi project is the creator of smidump.  There are several 
  46         interesting sub-projects available. 
  47   
  48     http://net-snmp.sourceforge.net/ 
  49         Homepage for Net-SNMP which is used by Zenoss for SNMP management. 
  50   
  51     http://www.et.put.poznan.pl/snmp/asn1/asn1.html 
  52         An overview of Abstract Syntax Notation (ASN), the language in 
  53         which MIBs are written. 
  54  """ 
  55   
  56  import os 
  57  import os.path 
  58  import sys 
  59  import re 
  60  from subprocess import Popen, PIPE 
  61  import tempfile 
  62  import urllib 
  63  import tarfile 
  64  import zipfile 
  65  from urlparse import urlsplit 
  66   
  67  import Globals 
  68  import transaction 
  69   
  70  from Products.ZenUtils.ZCmdBase import ZCmdBase 
  71  from Products.ZenUtils.Utils import zenPath 
  72  from zExceptions import BadRequest 
  73   
  74   
  75  CHUNK_SIZE = 50 
  76   
  77   
78 -class MibFile:
79 """ 80 A MIB file has the meta-data for a MIB inside of it. 81 """
82 - def __init__(self, fileName, fileContents=""):
83 self.fileName = fileName 84 self.mibs = [] # Order of MIBs defined in the file 85 self.mibToDeps = {} # Dependency defintions for each MIB 86 self.fileContents = self.removeMibComments(fileContents) 87 self.mibDefinitions = self.splitFileToMIBs(self.fileContents) 88 self.mapMibToDependents(self.mibDefinitions) 89 90 # Reclaim some memory 91 self.fileContents = "" 92 self.mibDefinitions = ""
93
94 - def removeMibComments(self, fileContents):
95 """ 96 Parses the string provided as an argument and extracts 97 all of the ASN.1 comments from the string. 98 99 Assumes that fileContents contains the contents of a well-formed 100 (no errors) MIB file. 101 102 @param fileContents: entire contents of a MIB file 103 @type fileContents: string 104 @return: text without any comments 105 @rtype: string 106 """ 107 def findSingleLineCommentEndPos(startPos): 108 """ 109 Beginning at startPos + 2, searches fileContents for the end of 110 a single line comment. If comment ends with a newline 111 character, the newline is not included in the comment. 112 113 MIB single line comment rules: 114 1. Begins with '--' 115 2. Ends with '--' or newline 116 3. Any characters between the beginning and end of the comment 117 are ignored as part of the comment, including quotes and 118 start/end delimiters for block comments ( '/*' and '*/') 119 120 @param startPos: character position of the beginning of the single 121 line comment within fileContents 122 @type fileContents: string 123 @return: startPos, endPos (character position of the last character 124 in the comment + 1) 125 @rtype: tuple (integer, integer) 126 """ 127 commentEndPosDash = fileContents.find('--', startPos + 2) 128 commentEndPosNewline = fileContents.find('\n', startPos + 2) 129 if commentEndPosDash != -1: 130 if commentEndPosNewline != -1: 131 if commentEndPosDash < commentEndPosNewline: 132 endPos = commentEndPosDash + 2 133 else: 134 endPos = commentEndPosNewline 135 else: 136 endPos = commentEndPosDash + 2 137 else: 138 if commentEndPosNewline != -1: 139 endPos = commentEndPosNewline 140 else: 141 endPos = len(fileContents) 142 143 return startPos, endPos
144 145 def findBlockCommentEndPos(searchStartPos): 146 """ 147 Beginning at startPos + 2, searches fileContents for the end of 148 a block comment. If block comments are nested, the 149 function interates into each block comment by calling itself. 150 151 MIB block comment rules: 152 1. Begins with '/*' 153 2. Ends with '*/' 154 3. Block comments can be nested 155 3. Any characters between the beginning and end of the comment 156 are ignored as part of the comment, including quotes and 157 start/end delimiters for single line comments ('--'). Newlines 158 are included as part of the block comment. 159 160 @param startPos: character position of the beginning of the block 161 comment within fileContents 162 @type fileContents: string 163 @return: startPos, endPos (character position of the last character 164 in the comment + 1) 165 @rtype: tuple (integer, integer) 166 """ 167 # Locate the next start and end markers 168 nextBlockStartPos = fileContents.find('/*', searchStartPos + 2) 169 nextBlockEndPos = fileContents.find('*/', searchStartPos + 2) 170 171 # If a nested comment exists, find the end 172 if nextBlockStartPos != -1 and \ 173 nextBlockStartPos < nextBlockEndPos: 174 nestedComment = findBlockCommentEndPos(nextBlockStartPos) 175 nextBlockEndPos = fileContents.find('*/', nestedComment[1]) 176 177 return searchStartPos, nextBlockEndPos + 2
178 179 # START removeMibComments 180 if not fileContents: 181 return fileContents 182 183 # Get rid of any lines that are completely made up of hyphens 184 fileContents = re.sub(r'[ \t]*-{2}[ \t]*$', '', fileContents) 185 186 # commentRanges holds a list of tuples in the form (startPos, endPos) 187 # that define the beginning and end of comments within fileContents 188 commentRanges = [] 189 searchStartPos = 0 # character position within fileContents 190 functions = {'SINGLE': findSingleLineCommentEndPos, 191 'BLOCK': findBlockCommentEndPos} 192 193 # Parse fileContents, looking for single line comments, block comments 194 # and string literals 195 while searchStartPos < len(fileContents): 196 # Find the beginning of the next occurrance of each item 197 singleLineStartPos = fileContents.find('--', searchStartPos) 198 blockStartPos = fileContents.find('/*', searchStartPos) 199 stringStartPos = fileContents.find('\"', searchStartPos) 200 201 nextItemPos = sys.maxint 202 nextItemType = '' 203 204 # Compare the next starting point for each item type. 205 if singleLineStartPos != -1 and \ 206 singleLineStartPos < nextItemPos: 207 nextItemPos = singleLineStartPos 208 nextItemType = 'SINGLE' 209 210 if blockStartPos != -1 and \ 211 blockStartPos < nextItemPos: 212 nextItemPos = blockStartPos 213 nextItemType = 'BLOCK' 214 215 # If the next item type is a string literal, just search for the 216 # next double quote and continue from there. This works because 217 # all double quotes (that are not part of a comment) appear in 218 # pairs. Even double-double quotes (escaped quotes) will work 219 # with this method since the first double quote will look like a 220 # string literal close quote and the second double quote will look 221 # like the beginning of a string literal. 222 if stringStartPos != -1 and \ 223 stringStartPos < nextItemPos: 224 newSearchStartPos = \ 225 fileContents.find('\"', stringStartPos + 1) + 1 226 if newSearchStartPos > searchStartPos: 227 searchStartPos = newSearchStartPos 228 else: # Weird error case 229 break 230 231 # If the next item is a comment, use the functions dictionary 232 # to call the appropriate function 233 elif nextItemPos != sys.maxint: 234 commentRange = functions[nextItemType](nextItemPos) 235 commentRanges.append(commentRange) 236 #searchStartPos = commentRange[1] 237 if commentRange[1] > 0: 238 searchStartPos = commentRange[1] 239 240 else: # No other items are found! 241 break 242 243 startPos = 0 244 mibParts = [] 245 246 # Iterate through each comment, adding the non-comment parts 247 # to mibParts. Finally, return the text without comments. 248 for commentRange in commentRanges: 249 mibParts.append(fileContents[startPos:(commentRange[0])]) 250 startPos = commentRange[1] 251 if startPos != len(fileContents): 252 mibParts.append(fileContents[startPos:(len(fileContents))]) 253 return ''.join(mibParts) 254
255 - def splitFileToMIBs(self, fileContents):
256 """ 257 Isolates each MIB definition in fileContents into a separate string 258 259 @param fileContents: the complete contents of a MIB file 260 @type fileContents: string 261 @return: MIB definition strings 262 @rtype: list of strings 263 """ 264 if fileContents is None: 265 return [] 266 267 DEFINITIONS = re.compile(r'([A-Za-z-0-9]+\s+DEFINITIONS' 268 '(\s+EXPLICIT TAGS|\s+IMPLICIT TAGS|\s+AUTOMATIC TAGS|\s*)' 269 '(\s+EXTENSIBILITY IMPLIED|\s*)\s*::=\s*BEGIN)') 270 271 definitionSpans = [] 272 for definitionMatch in DEFINITIONS.finditer(fileContents): 273 definitionSpans.append(list(definitionMatch.span())) 274 # If more than one definiton in the file, set the end of the 275 # last span to the beginning of the current span 276 if len(definitionSpans) > 1: 277 definitionSpans[-2][1] = definitionSpans[-1][0] 278 279 # Use the start and end positions to create a string for each 280 # MIB definition 281 mibDefinitions = [] 282 if definitionSpans: 283 # Set the end of the last span to the end of the file 284 definitionSpans[-1][1] = len(fileContents) 285 for definitionSpan in definitionSpans: 286 mibDefinitions.append( 287 fileContents[definitionSpan[0]:definitionSpan[1]]) 288 289 return mibDefinitions
290
291 - def mapMibToDependents(self, mibDefinitions):
292 # ASN.1 syntax regular expressions for declaring MIBs 293 # 294 # An example from http://www.faqs.org/rfcs/rfc1212.html 295 # 296 # RFC1213-MIB DEFINITIONS ::= BEGIN 297 # 298 # IMPORTS 299 # experimental, OBJECT-TYPE, Counter 300 # FROM RFC1155-SMI; 301 # 302 # -- a MIB may or may not have an IMPORTS section 303 # 304 # root OBJECT IDENTIFIER ::= { experimental xx } 305 # 306 # END 307 IMPORTS = re.compile(r'\sIMPORTS\s.+;', re.DOTALL) 308 DEPENDENCIES = re.compile( 309 r'\sFROM\s+(?P<dependency>[A-Za-z-0-9]+)') 310 311 for definition in mibDefinitions: 312 mibName = re.split(r'([A-Za-z0-9-]+)', definition)[1] 313 dependencies = set() 314 importsMatch = IMPORTS.search(definition) 315 if importsMatch: 316 imports = importsMatch.group() 317 for dependencyMatch in DEPENDENCIES.finditer(imports): 318 dependencies.add(dependencyMatch.group('dependency')) 319 self.mibs.append(mibName) 320 self.mibToDeps[mibName] = dependencies
321
322 -class PackageManager:
323 """ 324 Given an URL, filename or archive (eg zip, tar), extract the files from 325 the package and return a list of filenames. 326 """
327 - def __init__(self, log, downloaddir, extractdir):
328 """ 329 Initialize the packagae manager. 330 331 @parameter log: logging object 332 @type log: logging class object 333 @parameter downloaddir: directory name to store downloads 334 @type downloaddir: string 335 @parameter extractdir: directory name to store downloads 336 @type extractdir: string 337 """ 338 self.log = log 339 self.downloaddir = downloaddir 340 if self.downloaddir[-1] != '/': 341 self.downloaddir += '/' 342 self.extractdir = extractdir 343 if self.extractdir[-1] != '/': 344 self.extractdir += '/'
345
346 - def downloadExtract(self, url):
347 """ 348 Download and extract the list of filenames. 349 """ 350 try: 351 localFile = self.download(url) 352 except Exception: 353 self.log.error("Problems downloading the file from %s: %s" % ( 354 url, sys.exc_info()[1] ) ) 355 return [] 356 self.log.debug("Will attempt to load %s", localFile) 357 try: 358 return self.processPackage(localFile) 359 except IOError, ex: 360 self.log.error("Unable to process '%s' because: %s", 361 localFile, str(ex)) 362 sys.exit(1)
363
364 - def download(self, url):
365 """ 366 Download the package from the given URL, or if it's a filename, 367 return the filename. 368 """ 369 urlParts = urlsplit(url) 370 schema = urlParts[0] 371 path = urlParts[2] 372 if not schema: 373 return os.path.abspath(url) 374 file = path.split(os.sep)[-1] 375 os.makedirs(self.downloaddir) 376 downloadFile = os.path.join(self.downloaddir, file) 377 self.log.debug("Downloading to file '%s'", downloadFile) 378 filename, _ = urllib.urlretrieve(url, downloadFile) 379 return filename
380
381 - def processPackage(self, pkgFileName):
382 """ 383 Figure out what type of file we have and extract out any 384 files and then enumerate the file names. 385 """ 386 self.log.debug("Determining file type of %s" % pkgFileName) 387 if zipfile.is_zipfile(pkgFileName): 388 return self.unbundlePackage(pkgFileName, self.unzip) 389 390 elif tarfile.is_tarfile(pkgFileName): 391 return self.unbundlePackage(pkgFileName, self.untar) 392 393 elif os.path.isdir(pkgFileName): 394 return self.processDir(pkgFileName) 395 396 else: 397 return [ os.path.abspath(pkgFileName) ]
398
399 - def unzip(self, file):
400 """ 401 Unzip the given file into the current directory and return 402 the directory in which files can be loaded. 403 """ 404 pkgZip= zipfile.ZipFile(file, 'r') 405 if pkgZip.testzip() != None: 406 self.log.error("File %s is corrupted -- please download again", file) 407 return 408 409 for file in pkgZip.namelist(): 410 self.log.debug("Unzipping file/dir %s..." % file) 411 try: 412 if re.search(r'/$', file) != None: 413 os.makedirs(file) 414 else: 415 contents = pkgZip.read(file) 416 try: 417 unzipped = open(file, "w") 418 except IOError: # Directory missing? 419 os.makedirs(os.path.dirname(file)) 420 unzipped = open(file, "w") 421 unzipped.write(contents) 422 unzipped.close() 423 except (SystemExit, KeyboardInterrupt): raise 424 except: 425 self.log.error("Error in extracting %s because %s" % ( 426 file, sys.exc_info()[1] ) ) 427 return 428 429 return os.getcwd()
430
431 - def untar(self, file):
432 """ 433 Given a tar file, extract from the tar into the current directory. 434 """ 435 try: 436 self.log.debug("Extracting files from tar...") 437 pkgTar = tarfile.open(file, 'r') 438 for tarInfo in pkgTar: 439 pkgTar.extract(tarInfo) 440 pkgTar.close() 441 except (SystemExit, KeyboardInterrupt): raise 442 except: 443 self.log.error("Error in un-tarring %s because %s" % ( file, 444 sys.exc_info()[1] ) ) 445 return 446 return os.getcwd()
447
448 - def processDir(self, dir):
449 """ 450 Note all of the files in a directory. 451 """ 452 fileList = [] 453 self.log.debug("Enumerating files in %s", dir) 454 if not os.path.isdir(dir): 455 self.log.debug("%s is not a directory", dir) 456 return [] 457 for directoryName, _, fileNames in os.walk(dir): 458 for fileName in fileNames: 459 fileList.append(os.path.join(directoryName, fileName)) 460 return fileList
461
462 - def unbundlePackage(self, package, unpackageMethod):
463 """ 464 Extract the files and then add to the list of files. 465 """ 466 self.makeExtractionDir() 467 baseDir = unpackageMethod(package) 468 if baseDir is not None: 469 return self.processDir(baseDir) 470 return []
471
472 - def makeExtractionDir(self):
473 """ 474 Create an uniquely named extraction directory starting from a base 475 extraction directory. 476 """ 477 try: 478 if not os.path.isdir(self.extractdir): 479 os.makedirs(self.extractdir) 480 extractDir = tempfile.mkdtemp(prefix=self.extractdir) 481 except (SystemExit, KeyboardInterrupt): raise 482 except: 483 self.log.error("Error in creating temp dir because %s", 484 sys.exc_info()[1] ) 485 sys.exit(1) 486 os.chdir(extractDir)
487
488 - def cleanup(self):
489 """ 490 Remove any clutter left over from the installation. 491 """ 492 self.cleanupDir(self.downloaddir) 493 self.cleanupDir(self.extractdir)
494
495 - def cleanupDir(self, dirName):
496 for root, dirs, files in os.walk(dirName, topdown=False): 497 if root == os.sep: # Should *never* get here 498 break 499 for name in files: 500 os.remove(os.path.join(root, name)) 501 for name in dirs: 502 try: 503 os.removedirs(os.path.join(root, name)) 504 except OSError: 505 pass 506 try: 507 os.removedirs(dirName) 508 except OSError: 509 pass
510 511
512 -class ZenMib(ZCmdBase):
513 """ 514 Wrapper around the smidump utilities used to convert MIB definitions into 515 python code which is in turn loaded into the DMD tree. 516 """
517 - def makeMibFileObj(self, fileName):
518 """ 519 Scan the MIB file to determine what MIBs are defined in the file and 520 what their dependencies are. 521 522 @param fileName: MIB fileName 523 @type fileName: string 524 @return: dependencyDict, indexDict 525 dependencyDict - a dictionary that associates MIB definitions 526 found in fileName with their dependencies 527 indexDict - a dictionary that associates MIB definitions with their 528 ordinal position within fileName} 529 @rtype: 530 dependencyDict = {mibName: [string list of MIB definition names 531 that mibName is dependant on]} 532 indexDict = {mibname: number} 533 """ 534 # Retrieve the entire contents of the MIB file 535 self.log.debug("Processing %s", fileName) 536 file = open(fileName) 537 fileContents = file.read() 538 file.close() 539 return MibFile(fileName, fileContents)
540
541 - def populateDependencyMap(self, importFileNames, depFileNames):
542 """ 543 Populates the self.mibToMibFile instance object with data. 544 Exit the program if we're missing any files. 545 546 @param importFileNames: fully qualified file names of MIB files to import 547 @type importFileNames: list of strings 548 @param depFileNames: fully qualified file names of all MIB files 549 @type depFileNames: list of strings 550 @return: mibFileObjects of files to import 551 @rtype: MibFile 552 """ 553 self.log.debug("Collecting MIB meta-data and creating depedency map.") 554 toImportMibFileObjs = [] 555 for fileName in depFileNames.union(importFileNames): 556 try: 557 mibFileObj = self.makeMibFileObj(fileName) 558 except IOError: 559 self.log.error("Couldn't open file %s", fileName) 560 continue 561 562 mibDependencies = mibFileObj.mibToDeps 563 if not mibDependencies: 564 self.log.warn("Unable to parse information from " 565 "%s -- skipping", fileName) 566 continue 567 568 if fileName in importFileNames: 569 toImportMibFileObjs.append(mibFileObj) 570 571 for mibName, dependencies in mibDependencies.items(): 572 self.mibToMibFile[mibName] = mibFileObj 573 return toImportMibFileObjs
574
575 - def getDependencyFileNames(self, mibFileObj):
576 """ 577 smidump needs to know the list of dependent files for a MIB file in 578 order to properly resolve external references. 579 580 @param mibFileObj: MibFile object 581 @type mibFileObj: MibFile 582 @return: list of dependency fileNames 583 @rtype: list of strings 584 """ 585 dependencies = [] 586 dependencyFileNames = set() 587 588 def dependencySearch(mibName): 589 """ 590 Create a list of files required by a MIB definition. 591 592 @param mibName: name of MIB definition 593 @type mibName: string 594 """ 595 dependencies.append(mibName) 596 mibFileObj = self.mibToMibFile.get(mibName) 597 if not mibFileObj: 598 self.log.warn("Unable to find a file that defines %s", mibName) 599 return 600 601 dependencyFileNames.add(mibFileObj.fileName) 602 for dependency in mibFileObj.mibToDeps[mibName]: 603 if dependency not in dependencies: 604 dependencySearch(dependency)
605 606 for mibName in mibFileObj.mibs: 607 dependencySearch(mibName) 608 609 dependencyFileNames.discard(mibFileObj.fileName) 610 return dependencyFileNames
611
612 - def savePythonCode(self, pythonCode, fileName):
613 """ 614 Stores the smidump-generated Python code to a file. 615 """ 616 if not os.path.exists(self.options.pythoncodedir): 617 self.options.keeppythoncode = False 618 self.log.warn('The directory %s to store converted MIB file code ' 619 'does not exist.' % self.options.pythoncodedir) 620 return 621 try: 622 pythonFileName = os.path.join(self.options.pythoncodedir, 623 os.path.basename(fileName) ) + '.py' 624 pythonFile = open(pythonFileName, 'w') 625 pythonFile.write(pythonCode) 626 pythonFile.close() 627 except (SystemExit, KeyboardInterrupt): raise 628 except: 629 self.log.warn('Could not output converted MIB to %s' % 630 pythonFileName)
631
632 - def generatePythonFromMib(self, fileName, dependencyFileNames, 633 mibNamesInFile):
634 """ 635 Use the smidump program to convert a MIB into Python code. 636 637 One major caveat: smidump by default only outputs the last MIB 638 definition in a file. For that matter, it always outputs the last MIB 639 definition in a file whether it is requested or not. Therefore, if 640 there are multiple MIB definitions in a file, all but the last must be 641 explicitly named on the command line. If you name the last, it will 642 come out twice. We don't want that. 643 644 OK, so we need to determine if there are multiple MIB definitions 645 in fileName and then add all but the last to the command line. That 646 works except the resulting python code will create a dictionary 647 for each MIB definition, all of them named MIB. Executing the code is 648 equivalent to running a=1; a=2; a=3. You only wind up with a=3. 649 Therefore, we separate each dictionary definition into its own string 650 and return a list of strings so each one can be executed individually. 651 652 @param fileName: name of the file containing MIB definitions 653 @type fileName: string 654 @param dependencyFileNames: list of fileNames that fileName is 655 dependent on 656 @type dependencyFileNames: list of strings 657 @param mibNamesInFile: names of MIB definitions in file 658 @type mibNamesInFile: list of strings 659 @return: list of dictionaries. Each dictionary containing the contents 660 of a MIB definition. [ {'mibName': MIB data} ] 661 @rtype: list 662 """ 663 dumpCommand = ['smidump', '--keep-going', '--format', 'python'] 664 for dependencyFileName in dependencyFileNames: 665 # Add command-line flag for our dependent files 666 dumpCommand.append('--preload') 667 dumpCommand.append(dependencyFileName) 668 dumpCommand.append(fileName) 669 670 # If more than one MIB definition exists in the file, name all but the 671 # last on the command line. (See method description for reasons.) 672 if len(mibNamesInFile) > 1: 673 dumpCommand += mibNamesInFile[:-1] 674 675 self.log.debug('Running %s', ' '.join(dumpCommand)) 676 sys.stdout.flush() 677 proc = Popen(dumpCommand, stdout=PIPE, stderr=PIPE, close_fds=True) 678 679 # If the child process generates enough output to the PIPE, 680 # then proc.wait() blocks waiting for the OS pipe buffer to accept more data 681 # (ie forever). Poll from time to time to avoid the problem with large MIBs. 682 pythonCode = '' 683 warnings = '' 684 while proc.poll() is None: 685 output, err = proc.communicate() 686 pythonCode += output 687 warnings += err 688 689 if proc.poll() != 0 or proc.returncode: 690 if warnings.strip(): 691 self.log.error(warnings) 692 return None 693 694 if warnings: 695 self.log.debug("Found warnings while trying to import MIB:\n%s" \ 696 % warnings) 697 698 if self.options.keeppythoncode: 699 self.savePythonCode(pythonCode, fileName) 700 701 return self.evalPythonToMibs(pythonCode, fileName)
702
703 - def evalPythonToMibs(self, pythonCode, name):
704 """ 705 Evaluate the code and return an array of MIB dictionaries. 706 """ 707 def executePythonCode(pythonCode, name): 708 """ 709 Executes the python code generated smidump 710 711 @param pythonCode: Code generated by smidump 712 @type pythonCode: string 713 @return: a dictionary which contains one key: MIB 714 @rtype: dictionary 715 """ 716 result = {} 717 try: 718 exec pythonCode in result 719 except (SystemExit, KeyboardInterrupt): raise 720 except: 721 self.log.exception("Unable to import Pythonized-MIB: %s", 722 name) 723 return result.get('MIB', None)
724 725 # If more than one MIB definition exists in a file, pythonCode will 726 # contain a 'MIB = {...}' section for each MIB definition. We must 727 # split each section into its own string and return a string list. 728 smiMibDelim = 'MIB = {' 729 mibCodeParts = pythonCode.split(smiMibDelim) 730 mibDicts = [] 731 if len(mibCodeParts) > 1: 732 for mibCodePart in mibCodeParts[1:]: 733 mibDict = executePythonCode(smiMibDelim + mibCodePart, name) 734 if mibDict is not None: 735 mibDicts.append(mibDict) 736 else: 737 mibDict = executePythonCode(pythonCode, name) 738 if mibDict is not None: 739 mibDicts = [mibDict] 740 741 return mibDicts 742
743 - def evalAddSavedPythonCode(self):
744 """ 745 Read the file named in the command-line option, evaluate it, and add 746 it to our list of MIBs. 747 """ 748 pythonMibs = [] 749 for fileName in set(self.options.evalSavedPython): 750 try: 751 pythonCode = open(fileName).read() 752 except IOError: 753 self.log.warn("Unable to open '%s' -- skipping", 754 fileName) 755 continue 756 pythonMibs += self.evalPythonToMibs(pythonCode, fileName) 757 758 self.loadPythonMibs(pythonMibs)
759 760
761 - def getDmdMibDict(self, dmdMibDict, mibOrganizer):
762 """ 763 Populate a dictionary containing the MIB definitions that have been 764 loaded into the DMD Mibs directory 765 766 @param dmdMibDict: maps a MIB definition to the path where 767 it is located with in the DMD. 768 Format: 769 {'mibName': 'DMD path where mibName is stored'} 770 Example: MIB-Dell-10892 is located in the DMD tree at 771 Mibs/SITE/Dell, Directory entry is 772 {'MIB-Dell-10892': '/SITE/Dell'] } 773 @param mibOrganizer: the DMD directory to be searched 774 @type mibOrganizer: MibOrganizer 775 """ 776 organizerPath = mibOrganizer.getOrganizerName() 777 778 # Record each module from this organizer as a dictionary entry. 779 # mibOrganizer.mibs.objectItems() returns tuple: 780 # ('mibName', <MibModule at /zport/dmd/Mibs/...>) 781 for mibModule in mibOrganizer.mibs.objectItems(): 782 mibName = mibModule[0] 783 if mibName not in dmdMibDict: 784 dmdMibDict[mibName] = organizerPath 785 else: 786 self.log.warn('\nFound two copies of %s:' 787 ' %s and %s' % 788 (mibName, dmdMibDict[mibName], 789 mibOrganizer.getOrganizerName())) 790 791 # If there are suborganizers, recurse into them 792 for childOrganizer in mibOrganizer.children(): 793 self.getDmdMibDict(dmdMibDict, childOrganizer)
794
795 - def addMibEntries(self, leafType, pythonMib, mibModule):
796 """ 797 Add the different MIB leaves (ie nodes, notifications) into the DMD. 798 799 @paramater leafType: 'nodes', 'notifications' 800 @type leafType: string 801 @paramater pythonMib: dictionary of nodes and notifications 802 @type pythonMib: dictionary 803 @paramater mibModule: class containing functions to load leaves 804 @type mibModule: class 805 @return: number of leaves added 806 @rtype: int 807 """ 808 entriesAdded = 0 809 functor = { 'nodes':mibModule.createMibNode, 810 'notifications':mibModule.createMibNotification, 811 }.get(leafType, None) 812 if not functor or leafType not in pythonMib: 813 return entriesAdded 814 815 mibName = pythonMib['moduleName'] 816 817 for name, values in pythonMib[leafType].items(): 818 try: 819 functor(name, **values) 820 entriesAdded += 1 821 except BadRequest: 822 try: 823 self.log.warn("Unable to add %s id '%s' as this" 824 " name is reserved for use by Zope", 825 leafType, name) 826 newName = '_'.join([name, mibName]) 827 self.log.warn("Trying to add %s '%s' as '%s'", 828 leafType, name, newName) 829 functor(newName, **values) 830 entriesAdded += 1 831 self.log.warn("Renamed '%s' to '%s' and added to" 832 " MIB %s", name, newName, leafType) 833 except (SystemExit, KeyboardInterrupt): raise 834 except: 835 self.log.warn("Unable to add %s id '%s' -- skipping", 836 leafType, name) 837 else: 838 if not entriesAdded % CHUNK_SIZE: 839 self.commit("Loaded MIB %s into the DMD" % mibName) 840 self.commit("Loaded MIB %s into the DMD" % mibName) 841 return entriesAdded
842
843 - def loadMibFile(self, mibFileObj, dmdMibDict):
844 """ 845 Attempt to load the MIB definitions in fileName into DMD 846 847 @param fileName: name of the MIB file to be loaded 848 @type fileName: string 849 @return: whether the MIB load was successful or not 850 @rtype: boolean 851 """ 852 fileName = mibFileObj.fileName 853 self.log.debug('Attempting to load %s' % fileName) 854 855 # Check to see if any MIB definitions in fileName have already 856 # been loaded into Zenoss. If so, warn but don't fail 857 mibNamesInFile = mibFileObj.mibs 858 for mibName in mibNamesInFile: 859 if mibName in dmdMibDict: 860 dmdMibPath = dmdMibDict[mibName] 861 self.log.warn('MIB definition %s found in %s is already ' 862 'loaded at %s.' % (mibName, fileName, dmdMibPath)) 863 864 # Retrieve a list of all the files containing MIB definitions that are 865 # required by the MIB definitions in fileName 866 dependencyFileNames = self.getDependencyFileNames(mibFileObj) 867 868 # Convert the MIB file data into python dictionaries. pythonMibs 869 # contains a list of dictionaries, one for each MIB definition in 870 # fileName. 871 pythonMibs = self.generatePythonFromMib(fileName, dependencyFileNames, 872 mibNamesInFile) 873 if not pythonMibs: 874 return False 875 876 self.loadPythonMibs(pythonMibs) 877 return True
878
879 - def commit(self, message):
880 if not self.options.nocommit: 881 self.log.debug('Committing a batch of objects') 882 trans = transaction.get() 883 trans.setUser('zenmib') 884 trans.note(message) 885 trans.commit() 886 self.syncdb()
887
888 - def loadPythonMibs(self, pythonMibs):
889 """ 890 Walk through the MIB dictionaries and add the MIBs to the DMD. 891 """ 892 # Standard MIB attributes that we expect in all MIBs 893 MIB_MOD_ATTS = ('language', 'contact', 'description') 894 895 self.syncdb() 896 897 # Add the MIB data for each MIB into Zenoss 898 for pythonMib in pythonMibs: 899 mibName = pythonMib['moduleName'] 900 901 # Create the container for the MIBs and define meta-data. 902 # In the DMD this creates another container class which 903 # contains mibnodes. These data types are found in 904 # Products.ZenModel.MibModule and Products.ZenModel.MibNode 905 mibModule = self.dmd.Mibs.createMibModule( 906 mibName, self.options.path) 907 908 def gen(): 909 for key, val in pythonMib[mibName].iteritems(): 910 if key in MIB_MOD_ATTS: 911 yield key, val
912 913 for i, attr in enumerate(gen()): 914 setattr(mibModule, *attr) 915 if not i % CHUNK_SIZE: 916 self.commit("Loaded MIB %s into the DMD" % mibName) 917 self.commit("Loaded MIB %s into the DMD" % mibName) 918 919 nodesAdded = self.addMibEntries('nodes', pythonMib, mibModule) 920 trapsAdded = self.addMibEntries('notifications', pythonMib, mibModule) 921 self.log.info("Parsed %d nodes and %d notifications from %s", 922 nodesAdded, trapsAdded, mibName) 923 924 # Add the MIB tree permanently to the DMD unless --nocommit flag. 925 msg = "Loaded MIB %s into the DMD" % mibName 926 self.commit(msg) 927 if not self.options.nocommit: 928 self.log.info(msg) 929 930
931 - def getAllMibDepFileNames(self):
932 """ 933 Use command line parameters to create a list of files containing MIB 934 definitions that will be used as a reference list for the files being 935 loaded into the DMD 936 937 @return: set of file names 938 @rtype: set 939 """ 940 defaultMibDepDirs = [ 'ietf', 'iana', 'irtf', 'tubs', 'site' ] 941 mibDepFileNames = set() 942 for subdir in defaultMibDepDirs: 943 depDir = os.path.join(self.options.mibdepsdir, subdir) 944 mibDepFileNames.update(self.pkgMgr.processDir(depDir)) 945 return mibDepFileNames
946
947 - def getMibsToImport(self):
948 """ 949 Uses command-line parameters to create a list of files containing MIB 950 definitions that are to be loaded into the DMD 951 952 @return: list of file names that are to be loaded into the DMD 953 @rtype: list 954 """ 955 loadFileNames = [] 956 if self.args: 957 for fileName in self.args: 958 loadFileNames.extend(self.pkgMgr.downloadExtract(fileName)) 959 else: 960 loadFileNames = self.pkgMgr.processDir(self.options.mibsdir) 961 962 if loadFileNames: 963 self.log.debug("Will attempt to load the following files: %s", 964 loadFileNames) 965 else: 966 self.log.error("No MIB files to load!") 967 sys.exit(1) 968 969 return set(loadFileNames)
970
971 - def main(self):
972 """ 973 Main loop of the program 974 """ 975 if self.options.evalSavedPython: 976 self.evalAddSavedPythonCode() 977 return 978 979 # Verify MIBs search directory is valid. Fail if not 980 if not os.path.exists(self.options.mibsdir): 981 self.log.error("The directory %s doesn't exist!" % 982 self.options.mibsdir ) 983 sys.exit(1) 984 985 self.pkgMgr = PackageManager(self.log, self.options.downloaddir, 986 self.options.extractdir) 987 self.mibToMibFile = {} 988 989 requestedFiles = self.getMibsToImport() 990 mibDepFileNames = self.getAllMibDepFileNames() 991 mibFileObjs = self.populateDependencyMap(requestedFiles, mibDepFileNames) 992 993 # dmdMibDict = {'mibName': 'DMD path to MIB'} 994 dmdMibDict = {} 995 self.getDmdMibDict(dmdMibDict, self.dmd.Mibs) 996 997 # Load the MIB files 998 self.log.info("Found %d MIBs to import.", len(mibFileObjs)) 999 loadedMibFiles = 0 1000 for mibFileObj in mibFileObjs: 1001 try: 1002 if self.loadMibFile(mibFileObj, dmdMibDict): 1003 loadedMibFiles += 1 1004 except Exception: 1005 self.log.exception("Failed to load MIB: %s", mibFileObj.fileName) 1006 1007 action = "Loaded" 1008 if self.options.nocommit: 1009 action = "Processed" 1010 1011 self.log.info("%s %d MIB file(s)" % (action, loadedMibFiles)) 1012 self.pkgMgr.cleanup() 1013 1014 sys.exit(0)
1015
1016 - def buildOptions(self):
1017 """ 1018 Command-line options 1019 """ 1020 ZCmdBase.buildOptions(self) 1021 self.parser.add_option('--mibsdir', 1022 dest='mibsdir', default=zenPath('share/mibs/site'), 1023 help="Directory of input MIB files [ default: %default ]") 1024 self.parser.add_option('--mibdepsdir', 1025 dest='mibdepsdir', default=zenPath('share/mibs'), 1026 help="Directory of input MIB files [ default: %default ]") 1027 self.parser.add_option('--path', 1028 dest='path', default="/", 1029 help="Path to load MIB into the DMD") 1030 self.parser.add_option('--nocommit', action='store_true', 1031 dest='nocommit', default=False, 1032 help="Don't commit the MIB to the DMD after loading") 1033 self.parser.add_option('--keeppythoncode', action='store_true', 1034 dest='keeppythoncode', default=False, 1035 help="Don't commit the MIB to the DMD after loading") 1036 self.parser.add_option('--pythoncodedir', dest='pythoncodedir', 1037 default=tempfile.gettempdir() + "/mib_pythoncode/", 1038 help="This is the directory where the converted MIB will be output. " \ 1039 "[ default: %default ]") 1040 self.parser.add_option('--downloaddir', dest='downloaddir', 1041 default=tempfile.gettempdir() + "/mib_downloads/", 1042 help="This is the directory where the MIB will be downloaded. " \ 1043 "[ default: %default ]") 1044 self.parser.add_option('--extractdir', dest='extractdir', 1045 default=tempfile.gettempdir() + "/mib_extract/", 1046 help="This is the directory where unzipped MIB files will be stored. " \ 1047 "[ default: %default ]") 1048 self.parser.add_option('--evalSavedPython', dest='evalSavedPython', 1049 default=[], action='append', 1050 help="Execute the Python code previously generated" \ 1051 " and saved.")
1052 1053 1054 if __name__ == '__main__': 1055 zm = ZenMib() 1056 zm.main() 1057