Hadoop动态调整日志级别

Hadoop 动态调整日志级别

hadoop 线上基础服务 ResourceManager，线上排查问题，不能直接重启调整日志级别，hadoop给我们提供了动态修改类日志级别的方式。可以指定某个类调整，调整这个类的日志级别。前提对出现问题的地方有个大概的定位，大概是哪个类的逻辑有问题，因此要对hadoop 代码相对较熟悉。

通过Yarn命令行调整日志级别

拿ResourceManager类举例子

#ResourceManager
yarn daemonlog -setlevel <host:httpport>  org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler INFO

yarn daemonlog -getlevel <host:httpport> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
#NameNode
 hadoop daemonlog -getlevel {namenode_host}:50070 BlockStateChange
 hadoop daemonlog -setlevel {namenode_host}:50070 BlockStateChange DEBUG 

说明：
host：ResourceManager host
httpport：ResourceManager web ui port

通过YarnWebUI调整日志级别

yarn 也提供了基于ui的方式设置,和命令行的方式一致

http://host:httpport/logLevel

说明：
host：ResourceManager host
httpport：ResourceManager web ui port

页面截图：

源码实现

实现了一个servelet，调用相关log实现的代码，关键代码

Log log = LogFactory.getLog(logName);
 log.setLevel(org.apache.log4j.Level.toLevel(level));```
       

完整代码：

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.log;

import java.io.*;
import java.net.*;
import java.util.regex.Pattern;

import javax.servlet.*;
import javax.servlet.http.*;

import com.google.common.base.Charsets;
import org.apache.commons.logging.*;
import org.apache.commons.logging.impl.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.util.ServletUtil;

/**
 * Change log level in runtime.
 */
@InterfaceStability.Evolving
public class LogLevel {
  public static final String USAGES = "\nUsage: General options are:\n"
      + "\t[-getlevel <host:httpPort> <name>]\n"
      + "\t[-setlevel <host:httpPort> <name> <level>]\n";

  /**
   * A command line implementation
   */
  public static void main(String[] args) {
    if (args.length == 3 && "-getlevel".equals(args[0])) {
      process("http://" + args[1] + "/logLevel?log=" + args[2]);
      return;
    }
    else if (args.length == 4 && "-setlevel".equals(args[0])) {
      process("http://" + args[1] + "/logLevel?log=" + args[2]
              + "&level=" + args[3]);
      return;
    }

    System.err.println(USAGES);
    System.exit(-1);
  }

  private static void process(String urlstring) {
    try {
      URL url = new URL(urlstring);
      System.out.println("Connecting to " + url);
      URLConnection connection = url.openConnection();
      connection.connect();

      BufferedReader in = new BufferedReader(new InputStreamReader(
          connection.getInputStream(), Charsets.UTF_8));
      for(String line; (line = in.readLine()) != null; )
        if (line.startsWith(MARKER)) {
          System.out.println(TAG.matcher(line).replaceAll(""));
        }
      in.close();
    } catch (IOException ioe) {
      System.err.println("" + ioe);
    }
  }

  static final String MARKER = "<!-- OUTPUT -->";
  static final Pattern TAG = Pattern.compile("<[^>]*>");

  /**
   * A servlet implementation
   */
  @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
  @InterfaceStability.Unstable
  public static class Servlet extends HttpServlet {
    private static final long serialVersionUID = 1L;

    @Override
    public void doGet(HttpServletRequest request, HttpServletResponse response
        ) throws ServletException, IOException {

      // If user is a static user and auth Type is null, that means
      // there is a non-security environment and no need authorization,
      // otherwise, do the authorization.
      final ServletContext servletContext = getServletContext();
      if (!HttpServer2.isStaticUserAndNoneAuthType(servletContext, request) &&
          !HttpServer2.hasAdministratorAccess(servletContext,
              request, response)) {
        return;
      }

      PrintWriter out = ServletUtil.initHTML(response, "Log Level");
      String logName = ServletUtil.getParameter(request, "log");
      String level = ServletUtil.getParameter(request, "level");

      if (logName != null) {
        out.println("<br /><hr /><h3>Results</h3>");
        out.println(MARKER
            + "Submitted Log Name: <b>" + logName + "</b><br />");

        Log log = LogFactory.getLog(logName);
        out.println(MARKER
            + "Log Class: <b>" + log.getClass().getName() +"</b><br />");
        if (level != null) {
          out.println(MARKER + "Submitted Level: <b>" + level + "</b><br />");
        }

        if (log instanceof Log4JLogger) {
          process(((Log4JLogger)log).getLogger(), level, out);
        }
        else if (log instanceof Jdk14Logger) {
          process(((Jdk14Logger)log).getLogger(), level, out);
        }
        else {
          out.println("Sorry, " + log.getClass() + " not supported.<br />");
        }
      }

      out.println(FORMS);
      out.println(ServletUtil.HTML_TAIL);
    }

    static final String FORMS = "\n<br /><hr /><h3>Get / Set</h3>"
        + "\n<form>Log: <input type='text' size='50' name='log' /> "
        + "<input type='submit' value='Get Log Level' />"
        + "</form>"
        + "\n<form>Log: <input type='text' size='50' name='log' /> "
        + "Level: <input type='text' name='level' /> "
        + "<input type='submit' value='Set Log Level' />"
        + "</form>";

    private static void process(org.apache.log4j.Logger log, String level,
        PrintWriter out) throws IOException {
      if (level != null) {
        if (!level.equalsIgnoreCase(org.apache.log4j.Level.toLevel(level)
            .toString())) {
          out.println(MARKER + "Bad Level : <b>" + level + "</b><br />");
        } else {
          log.setLevel(org.apache.log4j.Level.toLevel(level));
          out.println(MARKER + "Setting Level to " + level + " ...<br />");
        }
      }
      out.println(MARKER
          + "Effective Level: <b>" + log.getEffectiveLevel() + "</b><br />");
    }

    private static void process(java.util.logging.Logger log, String level,
        PrintWriter out) throws IOException {
      if (level != null) {
        String levelToUpperCase = level.toUpperCase();
        try {
          log.setLevel(java.util.logging.Level.parse(levelToUpperCase));
        } catch (IllegalArgumentException e) {
          out.println(MARKER + "Bad Level : <b>" + level + "</b><br />");
        }
        out.println(MARKER + "Setting Level to " + level + " ...<br />");
      }

      java.util.logging.Level lev;
      for(; (lev = log.getLevel()) == null; log = log.getParent());
      out.println(MARKER + "Effective Level: <b>" + lev + "</b><br />");
    }
  }
}

LOG框架设置日志级别的方式

#Log4j2
LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false);
LoggerConfig loggerConfig = loggerContext.getConfiguration().getLoggers().get("root");
loggerConfig.setLevel(level);
#Logback
LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
Logger logger = loggerContext.getLogger("root");
((ch.qos.logback.classic.Logger) logger).setLevel(level);
#Java Util Logging
Logger logger = Logger.getLogger("root");
logger.setLevel(level);