java applog_yarn logs -applicationId命令java版本简单实现

最新推荐文章于 2024-02-21 17:21:33 发布

Fccf

最新推荐文章于 2024-02-21 17:21:33 发布

阅读量287

点赞数

文章标签： java applog

本文链接：https://blog.csdn.net/weixin_42512096/article/details/114511161

版权

这个Java程序展示了如何使用Hadoop和YARN的相关API来读取和打印YARN应用的日志。通过配置Hadoop的配置文件，获取ApplicationId，然后遍历节点日志目录，读取并解析聚合日志，输出每个容器的日志内容。

摘要由CSDN通过智能技术生成

1 import java.io.DataInputStream;2 import java.io.EOFException;3 import java.io.FileNotFoundException;4 import java.io.PrintStream;5

6 import org.apache.commons.lang.StringUtils;7 import org.apache.hadoop.conf.Configuration;8 import org.apache.hadoop.fs.FileContext;9 import org.apache.hadoop.fs.FileStatus;10 import org.apache.hadoop.fs.Path;11 import org.apache.hadoop.fs.RemoteIterator;12 import org.apache.hadoop.security.UserGroupInformation;13 import org.apache.hadoop.yarn.api.records.ApplicationId;14 import org.apache.hadoop.yarn.conf.YarnConfiguration;15 import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;16 import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;17 import org.apache.hadoop.yarn.util.ConverterUtils;18

19 public classGetYarnLog {20 public static voidmain(String[] args) {21 run("application_1535700682133_0496");22 }23

24 public static intrun(String appIdStr) throws Throwable{25

27 Configuration conf = newYarnConfiguration();28 conf.addResource(new Path("/etc/hadoop/conf.cloudera.yarn/core-site.xml"));29 conf.addResource(new Path("/etc/hadoop/conf.cloudera.yarn/yarn-site.xml"));30 conf.addResource(new Path("/etc/hadoop/conf.cloudera.yarn/hdfs-site.xml"));31 if(appIdStr == null || appIdStr.equals(""))32 {33 System.out.println("appId is null!");34 return -1;35 }36 PrintStream out=newPrintStream(appIdStr);37 ApplicationId appId = null;38 appId =ConverterUtils.toApplicationId(appIdStr);39

40 Path remoteRootLogDir = new Path(conf.get("yarn.nodemanager.remote-app-log-dir", "/tmp/logs"));41

42 String user =UserGroupInformation.getCurrentUser().getShortUserName();;43 String logDirSuffix =LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);44

45 Path remoteAppLogDir =LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, user, logDirSuffix);46 RemoteIteratornodeFiles;47 try

48 {49 Path qualifiedLogDir =FileContext.getFileContext(conf).makeQualified(remoteAppLogDir);50 nodeFiles =FileContext.getFileContext(qualifiedLogDir.toUri(), conf).listStatus(remoteAppLogDir);51 }52 catch(FileNotFoundException fnf)53 {54 logDirNotExist(remoteAppLogDir.toString());55 return -1;56 }57

58 boolean foundAnyLogs = false;59 while(nodeFiles.hasNext())60 {61 FileStatus thisNodeFile =(FileStatus)nodeFiles.next();62 if (!thisNodeFile.getPath().getName().endsWith(".tmp"))63 {64 System.out.println("NodeFileName ="+thisNodeFile.getPath().getName());65 AggregatedLogFormat.LogReader reader = newAggregatedLogFormat.LogReader(conf, thisNodeFile.getPath());66 try

67 {68 AggregatedLogFormat.LogKey key = newAggregatedLogFormat.LogKey();69 DataInputStream valueStream =reader.next(key);70 for(;;)71 {72 if (valueStream != null)73 {74 String containerString = "\n\nContainer:" + key + "on" +thisNodeFile.getPath().getName();75

76 out.println(containerString);77 out.println(StringUtils.repeat("=", containerString.length()));78 try

79 {80 for(;;)81 {82 AggregatedLogFormat.LogReader.readAContainerLogsForALogType(valueStream, out, thisNodeFile.getModificationTime());83

84 foundAnyLogs = true;85 }86

87 }88 catch(EOFException eof)89 {90 key = newAggregatedLogFormat.LogKey();91 valueStream =reader.next(key);92

93 }94

95 }else{96 break;97 }98 }99 }100 finally

101 {102 reader.close();103 }104 }105 }106 if (!foundAnyLogs)107 {108 emptyLogDir(remoteAppLogDir.toString());109 return -1;110 }111 return 0;112 }113 }