Package Products :: Package ZenUtils :: Module CmdBase
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenUtils.CmdBase

  1  ########################################################################### 
  2  # 
  3  # This program is part of Zenoss Core, an open source monitoring platform. 
  4  # Copyright (C) 2007, Zenoss Inc. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify it 
  7  # under the terms of the GNU General Public License version 2 or (at your 
  8  # option) any later version as published by the Free Software Foundation. 
  9  # 
 10  # For complete information please visit: http://www.zenoss.com/oss/ 
 11  # 
 12  ########################################################################### 
 13   
 14  __doc__="""CmdBase 
 15   
 16  Provide utility functions for logging and config file parsing 
 17  to command-line programs 
 18  """ 
 19   
 20  import os 
 21  import sys 
 22  import datetime 
 23  import logging 
 24  import re 
 25  from copy import copy 
 26  import zope.component 
 27  from zope.traversing.adapters import DefaultTraversable 
 28  from Products.Five import zcml 
 29   
 30  from optparse import OptionParser, SUPPRESS_HELP, NO_DEFAULT, OptionValueError, BadOptionError, Values, Option 
 31  from urllib import quote 
 32   
 33  # There is a nasty incompatibility between pkg_resources and twisted. 
 34  # This pkg_resources import works around the problem. 
 35  # See http://dev.zenoss.org/trac/ticket/3146 for details 
 36  from Products.ZenUtils.PkgResources import pkg_resources 
 37   
 38  from Products.ZenUtils.Utils import unused, load_config_override, zenPath 
 39  from Products.ZenUtils.GlobalConfig import _convertConfigLinesToArguments, applyGlobalConfToParser 
 40  unused(pkg_resources) 
 41   
42 -class DMDError: pass
43 44
45 -def checkLogLevel(option, opt, value):
46 if re.match(r'^\d+$', value): 47 value = int(value) 48 else: 49 intval = getattr(logging, value.upper(), None) 50 if intval: 51 value = intval 52 else: 53 raise OptionValueError('"%s" is not a valid log level.' % value) 54 55 return value
56
57 -def remove_args(argv, remove_args_novals, remove_args_vals):
58 """ 59 Removes arguments from the argument list. Arguments in 60 remove_args_novals have no arguments. Arguments in 61 remove_args_vals have arguments, either in the format 62 --arg=<val> or --arg <val>. 63 """ 64 new_args = [] 65 it = iter(argv) 66 for arg in it: 67 if arg in remove_args_novals: 68 continue 69 add_arg = True 70 for remove_arg in remove_args_vals: 71 if remove_arg == arg: 72 add_arg = False 73 it.next() # Skip the argument value 74 break 75 elif arg.startswith(remove_arg + '='): 76 add_arg = False 77 break 78 if add_arg: 79 new_args.append(arg) 80 return new_args
81
82 -class LogSeverityOption(Option):
83 TYPES = Option.TYPES + ("loglevel",) 84 TYPE_CHECKER = copy(Option.TYPE_CHECKER) 85 TYPE_CHECKER["loglevel"] = checkLogLevel
86 87
88 -class CmdBase(object):
89 """ 90 Class used for all Zenoss commands 91 """ 92 93 doesLogging = True 94
95 - def __init__(self, noopts=0, args=None):
96 97 zope.component.provideAdapter(DefaultTraversable, (None,)) 98 # We must import ZenossStartup at this point so that all Zenoss daemons 99 # and tools will have any ZenPack monkey-patched methods available. 100 import Products.ZenossStartup 101 unused(Products.ZenossStartup) 102 if not zcml._initialized: 103 import Products.Five, Products.ZenModel, Products.ZenRelations, Products.Zuul 104 try: 105 zcml.load_config('meta.zcml', Products.Five) 106 zcml.load_config('indexing.zcml', Products.ZenModel) 107 zcml.load_config('zendoc.zcml', Products.ZenModel) 108 zcml.load_config('configure.zcml', Products.ZenRelations) 109 zcml.load_config('configure.zcml', Products.Zuul) 110 zcml.load_config('configure.zcml', Products.ZenUtils) 111 except AttributeError: 112 # Could be that we're in a pre-Product-installation Zope, e.g. in 113 # zenwipe. No problem, we won't need this stuff now anyway. 114 pass 115 import Products.ZenMessaging 116 zcml.load_config('configure.zcml', Products.ZenMessaging) 117 import Products.ZenWidgets 118 load_config_override('scriptmessaging.zcml', Products.ZenWidgets) 119 120 self.usage = "%prog [options]" 121 self.noopts = noopts 122 self.inputArgs = args 123 124 # inputArgs was created to allow unit tests to pass in command line 125 # arguments and get around whatever Zope was doing to sys.argv. 126 if self.inputArgs is None: 127 self.inputArgs = sys.argv[1:] 128 129 self.parser = None 130 self.args = [] 131 132 self.buildParser() 133 self.buildOptions() 134 135 # Get defaults from global.conf. They will be overridden by 136 # daemon-specific config file or command line arguments. 137 applyGlobalConfToParser(self.parser) 138 self.parseOptions() 139 if self.options.configfile: 140 self.parser.defaults = self.getConfigFileDefaults(self.options.configfile) 141 # We've updated the parser with defaults from configs, now we need 142 # to reparse our command-line to get the correct overrides from 143 # the command-line 144 self.parseOptions() 145 146 if self.doesLogging: 147 self.setupLogging()
148 149
150 - def buildParser(self):
151 """ 152 Create the options parser 153 """ 154 if not self.parser: 155 from Products.ZenModel.ZenossInfo import ZenossInfo 156 try: 157 zinfo= ZenossInfo('') 158 version= str(zinfo.getZenossVersion()) 159 except: 160 from Products.ZenModel.ZVersion import VERSION 161 version= VERSION 162 self.parser = OptionParser(usage=self.usage, 163 version="%prog " + version, 164 option_class=LogSeverityOption)
165 166
167 - def buildOptions(self):
168 """ 169 Basic options setup. Other classes should call this before adding 170 more options 171 """ 172 self.buildParser() 173 if self.doesLogging: 174 self.parser.add_option('-v', '--logseverity', 175 dest='logseverity', 176 default='INFO', 177 type='loglevel', 178 help='Logging severity threshold', 179 ) 180 181 self.parser.add_option('--logpath',dest='logpath', 182 help='Override the default logging path') 183 184 self.parser.add_option('--maxlogsize', 185 dest='maxLogKiloBytes', 186 help='Max size of log file in KB; default 10240', 187 default=10240, 188 type='int') 189 190 self.parser.add_option('--maxbackuplogs', 191 dest='maxBackupLogs', 192 help='Max number of back up log files; default 3', 193 default=3, 194 type='int') 195 196 self.parser.add_option("-C", "--configfile", 197 dest="configfile", 198 help="Use an alternate configuration file" ) 199 200 self.parser.add_option("--genconf", 201 action="store_true", 202 default=False, 203 help="Generate a template configuration file" ) 204 205 self.parser.add_option("--genxmltable", 206 action="store_true", 207 default=False, 208 help="Generate a Docbook table showing command-line switches." ) 209 210 self.parser.add_option("--genxmlconfigs", 211 action="store_true", 212 default=False, 213 help="Generate an XML file containing command-line switches." )
214 215
216 - def parseOptions(self):
217 """ 218 Uses the optparse parse previously populated and performs common options. 219 """ 220 221 if self.noopts: 222 args = [] 223 else: 224 args = self.inputArgs 225 226 (self.options, self.args) = self.parser.parse_args(args=args) 227 228 if self.options.genconf: 229 self.generate_configs( self.parser, self.options ) 230 231 if self.options.genxmltable: 232 self.generate_xml_table( self.parser, self.options ) 233 234 if self.options.genxmlconfigs: 235 self.generate_xml_configs( self.parser, self.options )
236 237
238 - def getConfigFileDefaults(self, filename):
239 # TODO: This should be refactored - duplicated code with CmdBase. 240 """ 241 Parse a config file which has key-value pairs delimited by white space, 242 and update the parser's option defaults with these values. 243 244 @parameter filename: name of configuration file 245 @type filename: string 246 """ 247 248 options = self.parser.get_default_values() 249 lines = self.loadConfigFile(filename) 250 if lines: 251 lines, errors = self.validateConfigFile(filename, lines) 252 253 args = self.getParamatersFromConfig(lines) 254 try: 255 self.parser._process_args([], args, options) 256 except (BadOptionError, OptionValueError) as err: 257 print >>sys.stderr, 'WARN: %s in config file %s' % (err, filename) 258 259 return options.__dict__
260 261
263 # Deprecated: This method is going away - it is duplicated in GlobalConfig.py 264 """ 265 Parse a config file which has key-value pairs delimited by white space, 266 and update the parser's option defaults with these values. 267 """ 268 269 filename = zenPath('etc', 'global.conf') 270 options = self.parser.get_default_values() 271 lines = self.loadConfigFile(filename) 272 if lines: 273 args = self.getParamatersFromConfig(lines) 274 275 try: 276 self.parser._process_args([], args, options) 277 except (BadOptionError, OptionValueError) as err: 278 # Ignore it, we only care about our own options as defined in the parser 279 pass 280 281 return options.__dict__
282 283
284 - def loadConfigFile(self, filename):
285 # TODO: This should be refactored - duplicated code with CmdBase. 286 """ 287 Parse a config file which has key-value pairs delimited by white space. 288 289 @parameter filename: path to the configuration file 290 @type filename: string 291 """ 292 lines = [] 293 try: 294 with open(filename) as file: 295 for line in file: 296 if line.lstrip().startswith('#') or line.strip() == '': 297 lines.append(dict(type='comment', line=line)) 298 else: 299 try: 300 # add default blank string for keys with no default value 301 # valid delimiters are space, ':' and/or '=' (see ZenUtils/config.py) 302 key, value = (re.split(r'[\s:=]+', line.strip(), 1) + ['',])[:2] 303 except ValueError: 304 lines.append(dict(type='option', line=line, key=line.strip(), value=None, option=None)) 305 else: 306 option = self.parser.get_option('--%s' % key) 307 lines.append(dict(type='option', line=line, key=key, value=value, option=option)) 308 except IOError as e: 309 errorMessage = 'WARN: unable to read config file {filename} \ 310 -- skipping. ({exceptionName}: {exception})'.format( 311 filename=filename, 312 exceptionName=e.__class__.__name__, 313 exception=e 314 ) 315 print >>sys.stderr, errorMessage 316 return [] 317 318 return lines
319 320
321 - def validateConfigFile(self, filename, lines, correctErrors=True, warnErrors=True):
322 """ 323 Validate config file lines which has key-value pairs delimited by white space, 324 and validate that the keys exist for this command's option parser. If 325 the option does not exist or has an empty value it will comment it out 326 in the config file. 327 328 @parameter filename: path to the configuration file 329 @type filename: string 330 @parameter lines: lines from config parser 331 @type lines: list 332 @parameter correctErrors: Whether or not invalid conf values should be 333 commented out. 334 @type correctErrors: boolean 335 """ 336 337 output = [] 338 errors = [] 339 validLines = [] 340 date = datetime.datetime.now().isoformat() 341 errorTemplate = '## Commenting out by config parser on %s: %%s\n' % date 342 343 for lineno, line in enumerate(lines): 344 if line['type'] == 'comment': 345 output.append(line['line']) 346 elif line['type'] == 'option': 347 if line['value'] is None: 348 errors.append((lineno + 1, 'missing value for "%s"' % line['key'])) 349 output.append(errorTemplate % 'missing value') 350 output.append('## %s' % line['line']) 351 elif line['option'] is None: 352 errors.append((lineno + 1, 'unknown option "%s"' % line['key'])) 353 output.append(errorTemplate % 'unknown option') 354 output.append('## %s' % line['line']) 355 else: 356 validLines.append(line) 357 output.append(line['line']) 358 else: 359 errors.append((lineno + 1, 'unknown line "%s"' % line['line'])) 360 output.append(errorTemplate % 'unknown line') 361 output.append('## %s' % line['line']) 362 363 if errors: 364 if correctErrors: 365 for lineno, message in errors: 366 print >>sys.stderr, 'INFO: Commenting out %s on line %d in %s' % (message, lineno, filename) 367 368 with open(filename, 'w') as file: 369 file.writelines(output) 370 371 if warnErrors: 372 for lineno, message in errors: 373 print >>sys.stderr, 'WARN: %s on line %d in %s' % (message, lineno, filename) 374 375 return validLines, errors
376 377
378 - def getParamatersFromConfig(self, lines):
379 # Deprecated: This method is going away 380 return _convertConfigLinesToArguments(self.parser, lines)
381 382
383 - def setupLogging(self):
384 """ 385 Set common logging options 386 """ 387 rlog = logging.getLogger() 388 rlog.setLevel(logging.WARN) 389 mname = self.__class__.__name__ 390 self.log = logging.getLogger("zen."+ mname) 391 zlog = logging.getLogger("zen") 392 try: 393 loglevel = int(self.options.logseverity) 394 except ValueError: 395 loglevel = getattr(logging, self.options.logseverity.upper(), logging.INFO) 396 zlog.setLevel(loglevel) 397 398 logdir = self.checkLogpath() 399 if logdir: 400 logfile = os.path.join(logdir, mname.lower()+".log") 401 maxBytes = self.options.maxLogKiloBytes * 1024 402 backupCount = self.options.maxBackupLogs 403 h = logging.handlers.RotatingFileHandler(logfile, maxBytes, backupCount) 404 h.setFormatter(logging.Formatter( 405 "%(asctime)s %(levelname)s %(name)s: %(message)s", 406 "%Y-%m-%d %H:%M:%S")) 407 rlog.addHandler(h) 408 else: 409 logging.basicConfig()
410 411
412 - def checkLogpath(self):
413 """ 414 Validate the logpath is valid 415 """ 416 if not self.options.logpath: 417 return None 418 else: 419 logdir = self.options.logpath 420 if not os.path.exists(logdir): 421 # try creating the directory hierarchy if it doesn't exist... 422 try: 423 os.makedirs(logdir) 424 except OSError, ex: 425 raise SystemExit("logpath:%s doesn't exist and cannot be created" % logdir) 426 elif not os.path.isdir(logdir): 427 raise SystemExit("logpath:%s exists but is not a directory" % logdir) 428 return logdir
429 430
431 - def pretty_print_config_comment( self, comment ):
432 """ 433 Quick and dirty pretty printer for comments that happen to be longer than can comfortably 434 be seen on the display. 435 """ 436 437 max_size= 40 438 # 439 # As a heuristic we'll accept strings that are +- text_window 440 # size in length. 441 # 442 text_window= 5 443 444 if len( comment ) <= max_size + text_window: 445 return comment 446 447 # 448 # First, take care of embedded newlines and expand them out to array entries 449 # 450 new_comment= [] 451 all_lines= comment.split( '\n' ) 452 for line in all_lines: 453 if len(line) <= max_size + text_window: 454 new_comment.append( line ) 455 continue 456 457 start_position= max_size - text_window 458 while len(line) > max_size + text_window: 459 index= line.find( ' ', start_position ) 460 if index > 0: 461 new_comment.append( line[ 0:index ] ) 462 line= line[ index: ] 463 464 else: 465 if start_position == 0: 466 # 467 # If we get here it means that the line is just one big string with no spaces 468 # in it. There's nothing that we can do except print it out. Doh! 469 # 470 new_comment.append( line ) 471 break 472 473 # 474 # Okay, haven't found anything to split on -- go back and try again 475 # 476 start_position= start_position - text_window 477 if start_position < 0: 478 start_position= 0 479 480 else: 481 new_comment.append( line ) 482 483 return "\n# ".join( new_comment )
484 485 486
487 - def generate_configs( self, parser, options ):
488 """ 489 Create a configuration file based on the long-form of the option names 490 491 @parameter parser: an optparse parser object which contains defaults, help 492 @parameter options: parsed options list containing actual values 493 """ 494 495 # 496 # Header for the configuration file 497 # 498 unused(options) 499 daemon_name= os.path.basename( sys.argv[0] ) 500 daemon_name= daemon_name.replace( '.py', '' ) 501 502 print """# 503 # Configuration file for %s 504 # 505 # To enable a particular option, uncomment the desired entry. 506 # 507 # Parameter Setting 508 # --------- -------""" % ( daemon_name ) 509 510 511 options_to_ignore= ( 'help', 'version', '', 'genconf', 'genxmltable' ) 512 513 # 514 # Create an entry for each of the command line flags 515 # 516 # NB: Ideally, this should print out only the option parser dest 517 # entries, rather than the command line options. 518 # 519 import re 520 for opt in parser.option_list: 521 if opt.help is SUPPRESS_HELP: 522 continue 523 524 # 525 # Get rid of the short version of the command 526 # 527 option_name= re.sub( r'.*/--', '', "%s" % opt ) 528 529 # 530 # And what if there's no short version? 531 # 532 option_name= re.sub( r'^--', '', "%s" % option_name ) 533 534 # 535 # Don't display anything we shouldn't be displaying 536 # 537 if option_name in options_to_ignore: 538 continue 539 540 # 541 # Find the actual value specified on the command line, if any, 542 # and display it 543 # 544 545 value= getattr( parser.values, opt.dest ) 546 547 default_value= parser.defaults.get( opt.dest ) 548 if default_value is NO_DEFAULT or default_value is None: 549 default_value= "" 550 default_string= "" 551 if default_value != "": 552 default_string= ", default: " + str( default_value ) 553 554 comment= self.pretty_print_config_comment( opt.help + default_string ) 555 556 # 557 # NB: I would prefer to use tabs to separate the parameter name 558 # and value, but I don't know that this would work. 559 # 560 print """# 561 # %s 562 #%s %s""" % ( comment, option_name, value ) 563 564 # 565 # Pretty print and exit 566 # 567 print "#" 568 sys.exit( 0 )
569 570 571
572 - def generate_xml_table( self, parser, options ):
573 """ 574 Create a Docbook table based on the long-form of the option names 575 576 @parameter parser: an optparse parser object which contains defaults, help 577 @parameter options: parsed options list containing actual values 578 """ 579 580 # 581 # Header for the configuration file 582 # 583 unused(options) 584 daemon_name= os.path.basename( sys.argv[0] ) 585 daemon_name= daemon_name.replace( '.py', '' ) 586 587 print """<?xml version="1.0" encoding="UTF-8"?> 588 589 <section version="4.0" xmlns="http://docbook.org/ns/docbook" 590 xmlns:xlink="http://www.w3.org/1999/xlink" 591 xmlns:xi="http://www.w3.org/2001/XInclude" 592 xmlns:svg="http://www.w3.org/2000/svg" 593 xmlns:mml="http://www.w3.org/1998/Math/MathML" 594 xmlns:html="http://www.w3.org/1999/xhtml" 595 xmlns:db="http://docbook.org/ns/docbook" 596 597 xml:id="%s.options" 598 > 599 600 <title>%s Options</title> 601 <para /> 602 <table frame="all"> 603 <caption>%s <indexterm><primary>Daemons</primary><secondary>%s</secondary></indexterm> options</caption> 604 <tgroup cols="2"> 605 <colspec colname="option" colwidth="1*" /> 606 <colspec colname="description" colwidth="2*" /> 607 <thead> 608 <row> 609 <entry> <para>Option</para> </entry> 610 <entry> <para>Description</para> </entry> 611 </row> 612 </thead> 613 <tbody> 614 """ % ( daemon_name, daemon_name, daemon_name, daemon_name ) 615 616 617 options_to_ignore= ( 'help', 'version', '', 'genconf', 'genxmltable' ) 618 619 # 620 # Create an entry for each of the command line flags 621 # 622 # NB: Ideally, this should print out only the option parser dest 623 # entries, rather than the command line options. 624 # 625 import re 626 for opt in parser.option_list: 627 if opt.help is SUPPRESS_HELP: 628 continue 629 630 # 631 # Create a Docbook-happy version of the option strings 632 # Yes, <arg></arg> would be better semantically, but the output 633 # just looks goofy in a table. Use literal instead. 634 # 635 all_options= '<literal>' + re.sub( r'/', '</literal>,</para> <para><literal>', "%s" % opt ) + '</literal>' 636 637 # 638 # Don't display anything we shouldn't be displaying 639 # 640 option_name= re.sub( r'.*/--', '', "%s" % opt ) 641 option_name= re.sub( r'^--', '', "%s" % option_name ) 642 if option_name in options_to_ignore: 643 continue 644 645 default_value= parser.defaults.get( opt.dest ) 646 if default_value is NO_DEFAULT or default_value is None: 647 default_value= "" 648 default_string= "" 649 if default_value != "": 650 default_string= "<para> Default: <literal>" + str( default_value ) + "</literal></para>\n" 651 652 comment= self.pretty_print_config_comment( opt.help ) 653 654 # 655 # TODO: Determine the variable name used and display the --option_name=variable_name 656 # 657 if opt.action in [ 'store_true', 'store_false' ]: 658 print """<row> 659 <entry> <para>%s</para> </entry> 660 <entry> 661 <para>%s</para> 662 %s</entry> 663 </row> 664 """ % ( all_options, comment, default_string ) 665 666 else: 667 target= '=<replaceable>' + opt.dest.lower() + '</replaceable>' 668 all_options= all_options + target 669 all_options= re.sub( r',', target + ',', all_options ) 670 print """<row> 671 <entry> <para>%s</para> </entry> 672 <entry> 673 <para>%s</para> 674 %s</entry> 675 </row> 676 """ % ( all_options, comment, default_string ) 677 678 679 680 # 681 # Close the table elements 682 # 683 print """</tbody></tgroup> 684 </table> 685 <para /> 686 </section> 687 """ 688 sys.exit( 0 )
689 690 691
692 - def generate_xml_configs( self, parser, options ):
693 """ 694 Create an XML file that can be used to create Docbook files 695 as well as used as the basis for GUI-based daemon option 696 configuration. 697 """ 698 699 # 700 # Header for the configuration file 701 # 702 unused(options) 703 daemon_name= os.path.basename( sys.argv[0] ) 704 daemon_name= daemon_name.replace( '.py', '' ) 705 706 export_date = datetime.datetime.now() 707 708 print """<?xml version="1.0" encoding="UTF-8"?> 709 710 <!-- Default daemon configuration generated on %s --> 711 <configuration id="%s" > 712 713 """ % ( export_date, daemon_name ) 714 715 options_to_ignore= ( 716 'help', 'version', '', 'genconf', 'genxmltable', 717 'genxmlconfigs', 718 ) 719 720 # 721 # Create an entry for each of the command line flags 722 # 723 # NB: Ideally, this should print out only the option parser dest 724 # entries, rather than the command line options. 725 # 726 import re 727 for opt in parser.option_list: 728 if opt.help is SUPPRESS_HELP: 729 continue 730 731 # 732 # Don't display anything we shouldn't be displaying 733 # 734 option_name= re.sub( r'.*/--', '', "%s" % opt ) 735 option_name= re.sub( r'^--', '', "%s" % option_name ) 736 if option_name in options_to_ignore: 737 continue 738 739 default_value= parser.defaults.get( opt.dest ) 740 if default_value is NO_DEFAULT or default_value is None: 741 default_string= "" 742 else: 743 default_string= str( default_value ) 744 745 # 746 # TODO: Determine the variable name used and display the --option_name=variable_name 747 # 748 if opt.action in [ 'store_true', 'store_false' ]: 749 print """ <option id="%s" type="%s" default="%s" help="%s" /> 750 """ % ( option_name, "boolean", default_string, quote(opt.help), ) 751 752 else: 753 target= opt.dest.lower() 754 print """ <option id="%s" type="%s" default="%s" target="%s" help="%s" /> 755 """ % ( option_name, opt.type, quote(default_string), target, quote(opt.help), ) 756 757 758 # 759 # Close the table elements 760 # 761 print """ 762 </configuration> 763 """ 764 sys.exit( 0 )
765