Python入门之——logging日志模块 通过gppylib.gplog学习logging模块使用

gppylib.gplog在文件lib/python/gppylib/gplog.py下。该模块包含一些建立python内建logging模块的帮助函数。主要是方便tool、helper等模块建立各自的Logger（setup_helper_tool_logging和setup_tool_logging函数）。我们通过一个例子来对该文件中相关函数进行分析。

  from gppylib import gplog
  logger = gplog.setup_tool_logging(EXECNAME, hostname, username, logdir)
  if options.verbose:
    gplog.enable_verbose_logging()
  if options.quiet:
    gplog.quiet_stdout_logging()
  logger.info("Start myTool")
  ...

首先是调用setup_tool_logging函数，参数是EXECNAME, hostname, username, logdir。该函数返回单例模式的logger，可用于向stdout和文件中输出记录。主要是个性化定制logger的名字name，向SOUT_HANDLER和FILE_HANDLER设置Formatter，并使用个性化的格式化字符串。

def setup_tool_logging(appName, hostname, userName, logdir=None, nonuser=False):
    """
    Returns a singleton logger for standard Greenplum tools:
      - Logs output to stdout
      - Logs output to a file, typically in ~/gpAdminLogs
    """
    global _DEFAULT_FORMATTER
    global _APP_NAME_FOR_DEFAULT_FORMAT
    logger_name = "%s:%s" % (hostname, userName)
    if nonuser:
        appName = appName + "_" + logger_name
    _APP_NAME_FOR_DEFAULT_FORMAT = appName
    _enable_gpadmin_logging(appName, logdir)
    #
    # now reset the default formatter (someone may have called get_default_logger before calling setup_tool_logging)
    #
    logger = get_default_logger()
    logger.name = logger_name
    _DEFAULT_FORMATTER = None
    f = _get_default_formatter()
    _SOUT_HANDLER.setFormatter(f)
    _FILE_HANDLER.setFormatter(f)
    return logger

该函数用于为default logger设置文件输出handler，验证log的目录的正确性和存在性，最后通过调用_set_file_logging添加file handler。stream handler是在get_default_logger中加入logger的。

def _enable_gpadmin_logging(name, logdir=None):
    """
    Sets up the file output handler for the default logger.
      - if logdir is not specified it uses ~/gpAdminLogs
      - the file is constructed as appended with "<logdir>/<name>_<date>.log"

    NOTE: internal use only
    """
    global _FILE_HANDLER
    get_default_logger()
    now = datetime.date.today()
    if logdir is None:
        home_dir = os.path.expanduser("~")
        gpadmin_logs_dir = home_dir + "/gpAdminLogs"
    else:
        gpadmin_logs_dir = logdir
    if not os.path.exists(gpadmin_logs_dir):
        os.mkdir(gpadmin_logs_dir)
    filename = "%s/%s_%s.log" % (gpadmin_logs_dir, name, now.strftime("%Y%m%d"))
    _set_file_logging(filename)

get_default_logger单例模式返回默认logger，值创建stream handle也就是项stdout输出记录，并设置file handler。

def get_default_logger():
    """
    Return the singleton default logger.
    If a logger has not yet been established it creates one that:
      - Logs output to stdout
      - Does not setup file logging.
    Typical usage would be to call one of the setup_*_logging() functions
    at the beginning of a script in order to establish the exact type of
    logging desired, after which later calls to get_default_logger() can be
    used to return a reference to the logger.
    """
    global _LOGGER, _SOUT_HANDLER
    if _LOGGER is None:
        _LOGGER = logging.getLogger('default')
        f = _get_default_formatter()
        _SOUT_HANDLER = EncodingStreamHandler(sys.stdout)
        _SOUT_HANDLER.setFormatter(f)
        _LOGGER.addHandler(_SOUT_HANDLER)
        _LOGGER.setLevel(logging.INFO)
    return _LOGGER

自定义的Handler类，继承自logging.StreamHandler，主要用于确保message的编码为utf-8。在重写的emit函数中进行处理，最终还需调用父类的emit的函数对记录进行处理（写到Stream中）

class EncodingStreamHandler(logging.StreamHandler):
    """This handler makes sure that the encoding of the message is utf-8 before
    passing it along to the StreamHandler.  This will prevent encode/decode
    errors later on."""

    def __init__(self, strm=None):
        logging.StreamHandler.__init__(self, strm)

    def emit(self, record):
        if not isinstance(record.msg, str) and not isinstance(record.msg, unicode):
            record.msg = str(record.msg)
        if not isinstance(record.msg, unicode):
            record.msg = unicode(record.msg, 'utf-8')
        logging.StreamHandler.emit(self, record)

同样这里设置file handle，用于向文件中输入记录，其中的EncodingFileHandler继承ogging.FileHandler，也是确保message的编码为utf-8。

def _set_file_logging(filename):
    """
    Establishes a file output HANDLER for the default formatter.
    
    NOTE: internal use only
    """
    global _LOGGER, _SOUT_HANDLER, _FILENAME, _FILE_HANDLER
    _FILENAME = filename
    _FILE_HANDLER = EncodingFileHandler(filename, 'a')
    _FILE_HANDLER.setFormatter(_get_default_formatter())
    _LOGGER.addHandler(_FILE_HANDLER)

class EncodingFileHandler(logging.FileHandler):
    """This handler makes sure that the encoding of the message is utf-8 before
    passing it along to the FileHandler.  This will prevent encode/decode
    errors later on."""
    def __init__(self, filename, mode='a', encoding=None, delay=0):
        logging.FileHandler.__init__(self, filename, mode, encoding, delay)
    def emit(self, record):
        if not isinstance(record.msg, str) and not isinstance(record.msg, unicode):
            record.msg = str(record.msg)
        if not isinstance(record.msg, unicode):
            record.msg = unicode(record.msg, 'utf-8')
        logging.FileHandler.emit(self, record)

_get_default_formatter函数用于对formatter中的格式化进行定制后，返回Formatter对象。

def _get_default_formatter():
    """
    Returns the default formatter, constructing it if needed.
    The default formatter formats things using Greenplum standard logging:
      <date>:<pid> <programname>:<hostname>:<username>:[LEVEL]:-message
    NOTE: internal use only
    """
    global _DEFAULT_FORMATTER
    global _APP_NAME_FOR_DEFAULT_FORMAT
    if _DEFAULT_FORMATTER is None:
        format_str = "%(asctime)s:%(programname)s:%(name)s-[%(levelname)-s]:-%(message)s"
        app_name = _APP_NAME_FOR_DEFAULT_FORMAT.replace("%", "")  # to make sure we don't produce a format string
        format_str = format_str.replace("%(programname)s", "%06d %s" % (os.getpid(), app_name))
        _DEFAULT_FORMATTER = logging.Formatter(format_str, "%Y%m%d:%H:%M:%S")
    return _DEFAULT_FORMATTER

enable_verbose_logging和quiet_stdout_logging调用Logger对象的setLevel函数用于设置输出记录的级别阈值

def enable_verbose_logging():
    """
    Increases the log level to be verbose.
     - Applies to all logging handlers (stdout/file).
    """
    _LOGGER.setLevel(logging.DEBUG)

def quiet_stdout_logging():
    """ 
    Reduce log level for stdout logging 
    """
    global _SOUT_HANDLER
    _SOUT_HANDLER.setLevel(logging.WARN)