logger.py
上传用户:quxuerui
上传日期:2018-01-08
资源大小:41811k
文件大小:28k
- #Licensed to the Apache Software Foundation (ASF) under one
- #or more contributor license agreements. See the NOTICE file
- #distributed with this work for additional information
- #regarding copyright ownership. The ASF licenses this file
- #to you under the Apache License, Version 2.0 (the
- #"License"); you may not use this file except in compliance
- #with the License. You may obtain a copy of the License at
- # http://www.apache.org/licenses/LICENSE-2.0
- #Unless required by applicable law or agreed to in writing, software
- #distributed under the License is distributed on an "AS IS" BASIS,
- #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- #See the License for the specific language governing permissions and
- #limitations under the License.
- """hodLogger provides a customized interface to Python's core logging package.
- """
- import sys, os, re, logging, logging.handlers, inspect, pprint, types
- from tcp import get_address_tuple
- fileFormatString = '[%(asctime)s] %(levelname)s/%(levelno)s
- %(module)s:%(lineno)s - %(message)s'
- streamFormatString = '%(levelname)s - %(message)s'
- debugStreamFormatString = '[%(asctime)s] %(levelname)s/%(levelno)s
- %(module)s:%(lineno)s - %(message)s'
- syslogFormatString = '(%(process)d) %(levelname)s/%(levelno)s
- %(module)s:%(lineno)s - %(message)s'
- smtpFormatString = '[%(asctime)s] %(levelname)s/%(levelno)s
- %(module)s:%(lineno)snn%(message)s'
- fileFormater = logging.Formatter(fileFormatString)
- streamFormater = logging.Formatter(streamFormatString)
- debugStreamFormater = logging.Formatter(debugStreamFormatString)
- syslogFormater = logging.Formatter(syslogFormatString)
- smtpFormater = logging.Formatter(smtpFormatString)
- defaultFileLevel = 3
- defaultStreamLevel = 4
- defaultSyslogLevel = 3
- defaultSmtpLevel = 0
- hodLogLevelMap = { 0 : logging.CRITICAL,
- 1 : logging.ERROR,
- 2 : logging.WARNING,
- 3 : logging.INFO,
- 4 : logging.DEBUG }
- hodStreamFormatMap = { 0 : streamFormater,
- 1 : streamFormater,
- 2 : streamFormater,
- 3 : streamFormater,
- 4 : debugStreamFormater }
- rehodLogLevelMap = {}
- for key in hodLogLevelMap.keys():
- rehodLogLevelMap[hodLogLevelMap[key]] = key
- reModule = re.compile("^(.*)..*$")
- hodLogs = {}
- class hodRotatingFileHandler(logging.handlers.RotatingFileHandler):
- """ This class needs to be used in place of RotatingFileHandler when
- the 2.4.0 Python interpreter is used."""
- def emit(self, record):
- """
- Emit a record.
- If a formatter is specified, it is used to format the record.
- The record is then written to the stream with a trailing newline
- [N.B. this may be removed depending on feedback]. If exception
- information is present, it is formatted using
- traceback.print_exception and appended to the stream.
- *****
- THIS IS A HACK, when instances of hodLogger get passed to the child of
- a child thread for some reason self.stream gets closed. This version
- of emit re-opens self.stream if it is closed. After testing it appears
- that self.stream is only closed once after the second thread is
- initialized so there is not performance penalty to this hack. This
- problem only exists in python 2.4.
- *****
- """
- try:
- if self.shouldRollover(record):
- self.doRollover()
- try:
- msg = self.format(record)
- fs = "%sn"
- if not hasattr(types, "UnicodeType"): #if no unicode support...
- self.stream.write(fs % msg)
- else:
- try:
- self.stream.write(fs % msg)
- except UnicodeError:
- self.stream.write(fs % msg.encode("UTF-8"))
- except ValueError:
- self.stream = open(self.baseFilename, self.mode)
- self.stream.write(fs % msg)
- self.flush()
- except:
- self.handleError(record)
- except:
- self.handleError(record)
- def shouldRollover(self, record):
- """
- Determine if rollover should occur.
- Basically, see if the supplied record would cause the file to exceed
- the size limit we have.
- *****
- THIS IS A HACK, when instances of hodLogger get passed to the child of
- a child thread for some reason self.stream gets closed. This version
- of emit re-opens self.stream if it is closed. After testing it appears
- that self.stream is only closed once after the second thread is
- initialized so there is not performance penalty to this hack. This
- problem only exists in python 2.4.
- *****
- """
- if self.maxBytes > 0: # are we rolling over?
- msg = "%sn" % self.format(record)
- try:
- #due to non-posix-compliant Windows feature
- self.stream.seek(0, 2)
- except ValueError:
- self.stream = open(self.baseFilename, self.mode)
- self.stream.seek(0, 2)
- if self.stream.tell() + len(msg) >= self.maxBytes:
- return 1
- return 0
- class hodCustomLoggingLogger(logging.Logger):
- """ Slight extension of the logging.Logger class used by the hodLog class.
- """
- def findCaller(self):
- """ findCaller() is supposed to return the callers file name and line
- number of the caller. This was broken when the logging package was
- wrapped by hodLog. We should return much more relevant info now.
- """
- callerModule = ''
- callerLine = 0
- currentModule = os.path.basename(__file__)
- currentModule = reModule.sub("g<1>", currentModule)
- frames = inspect.stack()
- for i in range(len(frames)):
- frameModule = os.path.basename(frames[i][1])
- frameModule = reModule.sub("g<1>", frameModule)
- if frameModule == currentModule:
- previousFrameModule = os.path.basename(frames[i+1][1])
- previousFrameModule = reModule.sub("g<1>",
- previousFrameModule)
- callerFile = frames[i+1][1]
- callerLine = frames[i+1][2]
- continue
- returnValues = (callerFile, callerLine)
- if sys.version.startswith('2.4.4') or sys.version.startswith('2.5'):
- returnValues = (callerFile, callerLine, None)
-
- return returnValues
- class hodLog:
- """ Cluster management logging class.
- logging levels: 0 - log only critical messages
- 1 - log critical and error messages
- 2 - log critical, error, and warning messages
- 3 - log critical, error, warning, and info messages
- 4 - log critical, error, warning, info, and debug
- messages"""
- def __init__(self, appName):
- """Constructs a hodLogger object.
- appName - name of logging application, log filenames will be
- prepended with this name"""
- self.__appName = appName
- # initialize a dictionary to hold loggerNames
- self.__loggerNames = {}
- # initialize a dictionary to track log handlers and handler classes
- self.__logObjs = { 'file' : {}, 'smtp' : {},
- 'syslog' : {}, 'strm' : {} }
- # use a custom logging.Logger class
- logging.setLoggerClass(hodCustomLoggingLogger)
- # get the root app logger
- self.__logger = logging.getLogger(appName)
- self.__logger.setLevel(logging.DEBUG)
-
- hodLogs[self.__appName] = self
- def __attr__(self, attrname):
- """loggerNames - list of defined logger names"""
- if attrname == "loggerNames": return self.__loggerNames.keys()
- else: raise AttributeError, attrname
- def __repr__(self):
- """Returns a string representation of a hodLog object of the form:
- LOG_NAME
- file: FILENAME (level LEVEL)
- smtp: SMTP_SERVER from FROM_ADDRESS (level LEVEL)
- strm: STRM_OBJECT (level LEVEL)
- ... """
- hodLogString = "hodLog: %snn" % self.__appName
- for loggerName in self.__loggerNames.keys():
- hodLogString = "%s logger: %sn" % (hodLogString, loggerName)
- handlerClasses = self.__logObjs.keys()
- handlerClasses.sort()
- for handlerClass in handlerClasses:
- try:
- loggerLevelName = logging.getLevelName(
- self.__logObjs[handlerClass][loggerName]['level'])
- hodLogString = "%s %s: %s (level %s)n" % (
- hodLogString, handlerClass,
- self.__logObjs[handlerClass][loggerName]['data'],
- loggerLevelName)
- except:
- hodLogString = "%s %s: nonen" % (
- hodLogString, handlerClass)
- hodLogString = "%sn" % hodLogString
- return hodLogString
- # 'private' method which adds handlers to self.__logObjs
- def __add_to_handlers(self, handlerClass, loggerName, handler, data,
- level):
- self.__logObjs[handlerClass][loggerName] = {}
- self.__logObjs[handlerClass][loggerName]['handler'] = handler
- self.__logObjs[handlerClass][loggerName]['data'] = data
- self.__logObjs[handlerClass][loggerName]['level'] = level
- # 'private' method which determines whether a hod log level is valid and
- # returns a valid logging.Logger level
- def __get_logging_level(self, level, defaultLevel):
- loggingLevel = ''
- try:
- loggingLevel = hodLogLevelMap[int(level)]
- except:
- loggingLevel = hodLogLevelMap[defaultLevel]
- return loggingLevel
- # make a logging.logger name rootLogger.childLogger in our case the
- # appName.componentName
- def __get_logging_logger_name(self, loggerName):
- return "%s.%s" % (self.__appName, loggerName)
- def add_logger(self, loggerName):
- """Adds a logger of name loggerName.
- loggerName - name of component of a given application doing the
- logging
- Returns a hodLogger object for the just added logger."""
- try:
- self.__loggerNames[loggerName]
- except:
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- logging.getLogger(loggingLoggerName)
- self.__loggerNames[loggerName] = 1
- return hodLogger(self.__appName, loggingLoggerName)
- def add_file(self, logDirectory, maxBytes=0, backupCount=0,
- level=defaultFileLevel, addToLoggerNames=None):
- """Adds a file handler to all defined loggers or a specified set of
- loggers. Each log file will be located in logDirectory and have a
- name of the form appName-loggerName.log.
- logDirectory - logging directory
- maxBytes - maximum log size to write in bytes before rotate
- backupCount - number of rotated logs to keep
- level - cluster management log level
- addToLoggerNames - list of logger names to which stream handling
- will be added"""
- def add_file_handler(loggerName):
- if not self.__logObjs['file'].has_key(loggerName):
- loggingLevel = self.__get_logging_level(level,
- defaultFileLevel)
- logFile = os.path.join(logDirectory, "%s-%s.log" % (
- self.__appName, loggerName))
- logFilePresent = False
- if(os.path.exists(logFile)):
- logFilePresent = True
- if sys.version.startswith('2.4'):
- fileHandler = hodRotatingFileHandler(logFile,
- maxBytes=maxBytes, backupCount=backupCount)
- else:
- fileHandler = logging.handlers.RotatingFileHandler(logFile,
- maxBytes=maxBytes, backupCount=backupCount)
- if logFilePresent and backupCount:
- fileHandler.doRollover()
- fileHandler.setLevel(loggingLevel)
- fileHandler.setFormatter(fileFormater)
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- aLogger = logging.getLogger(loggingLoggerName)
- aLogger.addHandler(fileHandler)
- fileData = "%s" % logFile
- self.__add_to_handlers('file', loggerName, fileHandler,
- fileData, loggingLevel)
- if addToLoggerNames:
- for loggerName in addToLoggerNames:
- add_file_handler(loggerName)
- else:
- for loggerName in self.__loggerNames:
- add_file_handler(loggerName)
- def add_stream(self, stream=sys.stderr, level=defaultStreamLevel,
- addToLoggerNames=None):
- """Adds a stream handler to all defined loggers or a specified set of
- loggers.
- stream - a stream such as sys.stderr or sys.stdout
- level - cluster management log level
- addToLoggerNames - tupple of logger names to which stream handling
- will be added"""
- def add_stream_handler(loggerName):
- if not self.__logObjs['strm'].has_key(loggerName):
- loggingLevel = self.__get_logging_level(level,
- defaultStreamLevel)
- streamHandler = logging.StreamHandler(stream)
-
- streamHandler.setLevel(loggingLevel)
-
- streamHandler.setFormatter(hodStreamFormatMap[int(level)])
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- aLogger = logging.getLogger(loggingLoggerName)
- aLogger.addHandler(streamHandler)
- streamData = "%s" % stream
- self.__add_to_handlers('strm', loggerName, streamHandler,
- streamData, loggingLevel)
- if addToLoggerNames:
- for loggerName in addToLoggerNames:
- add_stream_handler(loggerName)
- else:
- for loggerName in self.__loggerNames:
- add_stream_handler(loggerName)
- def add_syslog(self, address, level=defaultSyslogLevel,
- addToLoggerNames=None):
- def add_syslog_handler(loggerName):
- if not self.__logObjs['syslog'].has_key(loggerName):
- loggingLevel = self.__get_logging_level(level,
- defaultSyslogLevel)
- address[1] = int(address[1])
- syslogHandler = logging.handlers.SysLogHandler(tuple(address),
- 9)
-
- syslogHandler.setLevel(loggingLevel)
-
- syslogHandler.setFormatter(syslogFormater)
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- aLogger = logging.getLogger(loggingLoggerName)
- aLogger.addHandler(syslogHandler)
- syslogData = "%s:%s" % (address[0], address[1])
- self.__add_to_handlers('syslog', loggerName, syslogHandler,
- syslogData, loggingLevel)
- if addToLoggerNames:
- for loggerName in addToLoggerNames:
- add_syslog_handler(loggerName)
- else:
- for loggerName in self.__loggerNames:
- add_syslog_handler(loggerName)
-
- def add_smtp(self, mailHost, fromAddress, toAddresses,
- level=defaultSmtpLevel, addToLoggerNames=None):
- """Adds an SMTP handler to all defined loggers or a specified set of
- loggers.
- mailHost - SMTP server to used when sending mail
- fromAddress - email address to use as the from address when
- sending mail
- toAdresses - comma seperated list of email address to which
- mail will be sent
- level - cluster management log level
- addToLoggerNames - tupple of logger names to which smtp handling
- will be added"""
- def add_email_handler(loggerName):
- if not self.__logObjs['smtp'].has_key(loggerName):
- loggingLevel = self.__get_logging_level(level,
- defaultSmtpLevel)
- subject = loggerName
- if loggingLevel == 50:
- subject = "%s - a critical error has occured." % subject
- elif loggingLevel == 40:
- subject = "%s - an error has occured." % subject
- elif loggingLevel == 30:
- subject = "%s - warning message." % subject
- elif loggingLevel == 20:
- subject = "%s - information message." % subject
- elif loggingLevel == 10:
- subject = "%s - debugging message." % subject
- mailHostTuple = get_address_tuple(mailHost)
- emailHandler = logging.handlers.SMTPHandler(mailHostTuple,
- fromAddress, toAddresses, subject)
- emailHandler.setFormatter(smtpFormater)
- emailHandler.setLevel(loggingLevel)
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- aLogger = logging.getLogger(loggingLoggerName)
- aLogger.addHandler(emailHandler)
- emailData = "%s from %s" % (mailHost, fromAddress)
- self.__add_to_handlers('smtp', loggerName, emailHandler,
- emailData, loggingLevel)
- if addToLoggerNames:
- for loggerName in addToLoggerNames:
- add_email_handler(loggerName)
- else:
- for loggerName in self.__loggerNames:
- add_email_handler(loggerName)
- def status(self):
- statusStruct = {}
- for loggerName in self.__loggerNames.keys():
- statusStruct[loggerName] = []
- for handlerClass in self.__logObjs.keys():
- loggerDict = {}
- try:
- level = self.__logObjs[handlerClass][loggerName]['level']
- level = rehodLogLevelMap[level]
- loggerDict['handler'] = handlerClass
- loggerDict['level'] = level
- loggerDict['data'] =
- self.__logObjs[handlerClass][loggerName]['data']
- except:
- pass
- else:
- statusStruct[loggerName].append(loggerDict)
- return statusStruct
- def lock_handlers(self):
- for handlerClass in self.__logObjs.keys():
- for loggerName in self.__logObjs[handlerClass].keys():
- self.__logObjs[handlerClass][loggerName]['handler'].acquire()
- def release_handlers(self):
- for handlerClass in self.__logObjs.keys():
- for loggerName in self.__logObjs[handlerClass].keys():
- self.__logObjs[handlerClass][loggerName]['handler'].release()
- def get_level(self, handler, loggerName):
- return rehodLogLevelMap[self.__logObjs[handler][loggerName]['level']]
- def set_level(self, handler, loggerName, level):
- """Sets the logging level of a particular logger and logger handler.
- handler - handler (smtp, file, or stream)
- loggerName - logger to set level on
- level - level to set logger
- """
- level = self.__get_logging_level(level, defaultFileLevel)
- self.__logObjs[handler][loggerName]['handler'].setLevel(level)
- self.__logObjs[handler][loggerName]['level'] = level
-
- if handler == 'stream':
- self.__logObjs[handler][loggerName]['handler'].setFormatter(
- hodStreamFormatMap[int(level)])
- def set_logger_level(self, loggerName, level):
- status = 0
- for handlerClass in self.__logObjs.keys():
- if self.__logObjs[handlerClass].has_key(loggerName):
- self.set_level(handlerClass, loggerName, level)
- else:
- status = 1
-
- return status
- def rollover(self, loggerName):
- status = 0
- if self.__logObjs['file'].has_key(loggerName):
- if self.__logObjs['file'][loggerName]['handler'].shouldRollover():
- self.__logObjs['file'][loggerName]['handler'].doRollover()
- else:
- status = 1
-
- return status
-
- def set_max_bytes(self, maxBytes):
- status = 0
- if self.__logObjs.has_key('file'):
- for loggerName in self.__logObjs['file'].keys():
- self.__logObjs['file'][loggerName]['handler'].maxBytes = 0
- else:
- status = 1
-
- return status
- def get_logger(self, loggerName):
- """ Returns a hodLogger object for a logger by name. """
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- return hodLogger(self.__appName, loggingLoggerName)
- def critical(self, loggerName, msg):
- """Logs a critical message and flushes log buffers. This method really
- should only be called upon a catastrophic failure.
- loggerName - logger to use
- msg - message to be logged"""
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- logger = logging.getLogger(loggingLoggerName)
- logger.critical(msg)
- self.flush()
- def error(self, loggerName, msg):
- """Logs an error message and flushes log buffers.
- loggerName - logger to use
- msg - message to be logged"""
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- logger = logging.getLogger(loggingLoggerName)
- logger.error(msg)
- self.flush()
- def warn(self, loggerName, msg):
- """Logs a warning message.
- loggerName - logger to use
- msg - message to be logged"""
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- logger = logging.getLogger(loggingLoggerName)
- logger.warn(msg)
- def info(self, loggerName, msg):
- """Logs an information message.
- loggerName - logger to use
- msg - message to be logged"""
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- logger = logging.getLogger(loggingLoggerName)
- logger.info(msg)
- def debug(self, loggerName, msg):
- """Logs a debugging message.
- loggerName - logger to use
- msg - message to be logged"""
- loggingLoggerName = self.__get_logging_logger_name(loggerName)
- logger = logging.getLogger(loggingLoggerName)
- logger.debug(msg)
- def flush(self):
- """Flush all log handlers."""
- for handlerClass in self.__logObjs.keys():
- for loggerName in self.__logObjs[handlerClass].keys():
- self.__logObjs[handlerClass][loggerName]['handler'].flush()
- def shutdown(self):
- """Shutdown all logging, flushing all buffers."""
- for handlerClass in self.__logObjs.keys():
- for loggerName in self.__logObjs[handlerClass].keys():
- self.__logObjs[handlerClass][loggerName]['handler'].flush()
- # Causes famous 'ValueError: I/O operation on closed file'
- # self.__logObjs[handlerClass][loggerName]['handler'].close()
- class hodLogger:
- """ Encapsulates a particular logger from a hodLog object. """
- def __init__(self, appName, loggingLoggerName):
- """Constructs a hodLogger object (a particular logger in a hodLog
- object).
- loggingLoggerName - name of a logger in hodLog object"""
- self.__appName = appName
- self.__loggerName = loggingLoggerName
- self.__logger = logging.getLogger(self.__loggerName)
- def __repr__(self):
- """Returns a string representation of a hodComponentLog object."""
- return "%s hodLog" % self.__loggerName
- def __call__(self):
- pass
- def set_logger_level(self, loggerName, level):
-
- return hodLogs[self.__appName].set_logger_level(loggerName, level)
-
- def set_max_bytes(self, maxBytes):
-
- return hodLogs[self.__appName].set_max_bytes(maxBytes)
- def rollover(self):
- return hodLogs[self.__appName].rollover(self.__loggerName)
- def get_level(self, handler, loggerName):
-
- return hodLogs[self.__appName].get_level(handler, loggerName)
-
- def critical(self, msg):
- """Logs a critical message and calls sys.exit(1).
- msg - message to be logged"""
- self.__logger.critical(msg)
- def error(self, msg):
- """Logs an error message.
- msg - message to be logged"""
- self.__logger.error(msg)
- def warn(self, msg):
- """Logs a warning message.
- msg - message to be logged"""
- self.__logger.warn(msg)
- def info(self, msg):
- """Logs an information message.
- msg - message to be logged"""
- self.__logger.info(msg)
- def debug(self, msg):
- """Logs a debugging message.
- msg - message to be logged"""
- self.__logger.debug(msg)
- class hodDummyLogger:
- """ Dummy hodLogger class. Other hod classes requiring a hodLogger default
- to this hodLogger if no logger is passed."""
- def __init__(self):
- """pass"""
- pass
- def __repr__(self):
- return "dummy hodLogger"
- def __call__(self):
- """pass"""
- pass
- def set_logger_level(self, loggerName, level):
-
- return 0
-
- def set_max_bytes(self, loggerName, maxBytes):
-
- return 0
- def get_level(self, handler, loggerName):
-
- return 4
- def rollover(self):
-
- return 0
- def critical(self, msg):
- """pass"""
- pass
- def error(self, msg):
- """pass"""
- pass
- def warn(self, msg):
- """pass"""
- pass
- def info(self, msg):
- """pass"""
- pass
- def debug(self, msg):
- """pass"""
- pass
- def ensureLogDir(logDir):
- """Verify that the passed in log directory exists, and if it doesn't
- create it."""
- if not os.path.exists(logDir):
- try:
- old_mask = os.umask(0)
- os.makedirs(logDir, 01777)
- os.umask(old_mask)
- except Exception, e:
- print >>sys.stderr, "Could not create log directories %s. Exception: %s. Stack Trace: %s" % (logDir, get_exception_error_string(), get_exception_string())
- raise e
- def getLogger(cfg, logName):
- if cfg['debug'] > 0:
- user = cfg['userid']
- baseLogger = hodLog(logName)
- log = baseLogger.add_logger('main')
- if cfg.has_key('log-dir'):
- serviceId = os.getenv('PBS_JOBID')
- if serviceId:
- logDir = os.path.join(cfg['log-dir'], "%s.%s" % (user, serviceId))
- else:
- logDir = os.path.join(cfg['log-dir'], user)
- if not os.path.exists(logDir):
- os.mkdir(logDir)
- baseLogger.add_file(logDirectory=logDir, level=cfg['debug'],
- addToLoggerNames=('main',))
- try:
- if cfg.has_key('stream') and cfg['stream']:
- baseLogger.add_stream(level=cfg['debug'], addToLoggerNames=('main',))
- if cfg.has_key('syslog-address'):
- baseLogger.add_syslog(cfg['syslog-address'],
- level=cfg['debug'], addToLoggerNames=('main',))
- except Exception,e:
- # Caught an exception while initialising logger
- log.critical("%s Logger failed to initialise. Reason : %s" % (logName, e))
- pass
- return log