数据治理之yarn日志解析

 

package log;


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;

import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.apache.http.util.EntityUtils;

import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;

import java.io.*;
import java.util.*;
import java.util.logging.Logger;



/**
 * @Auther: CHO 
 * @Date: 2019/2/25 17:56
 * @Description:
 */


public class MetaClusterJobServiceImpl {


     //Logger log = Logger.getLogger(Object.class);
    //Log log = Logger.getLogger(MetaClusterJobServiceImpl.class);

    //文件路径
    private String filePath="/export/home/CHO";

    public static void main(String[] args){
        MetaClusterJobServiceImpl yarn = new MetaClusterJobServiceImpl();

        yarn.readData(args);
    }

    public void readData(String[] args) {
        BufferedWriter out = null;
        try {
           // log.info("WorkerLogEsServiceImpl查询es开始");
            RestClient restClient = RestClient.builder(
                    new HttpHost("172.23.227.82", 9201, "http")).build();

            int startIndex = 0;
            Calendar calendar = Calendar.getInstance();
            calendar.set(Calendar.HOUR_OF_DAY, 0);
            calendar.set(Calendar.MINUTE, 0);
            calendar.set(Calendar.SECOND, 0);
            long maxTime = calendar.getTimeInMillis();
           // log.info("maxTime:{}",maxTime);

            calendar.add(Calendar.DAY_OF_YEAR, -45);
            long minTime = calendar.getTimeInMillis();
           // log.info("minTime:{}",minTime);
            Map<String, String> paramMap = new HashMap<String, String>();
            String source = genereateQueryString(minTime,maxTime);
            int insertTotalCount=0;
            int i =0;

            File fileDictory = new File(filePath);
            if(!fileDictory.exists()){
                fileDictory.mkdirs();
            }
            String fileName = args[0] +".txt";
            File txtFile = new File(filePath + File.separator + fileName);
            if (!txtFile.exists()) {
                txtFile.createNewFile();
            }
            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(txtFile), "utf-8"));
            while (true) {
               // log.info("WorkerLogEsServiceImpl第{}次查询，startIndex={}",i,startIndex);
                paramMap.put("from", String.valueOf(startIndex));
                paramMap.put("size", "1000");
                startIndex = startIndex+1000;
                HttpEntity httpEntity = new NStringEntity(source, ContentType.APPLICATION_JSON);
                Response response = restClient.performRequest("POST", "/"+args[0]+"/_search", paramMap, httpEntity);
                HttpEntity entity = response.getEntity();
                String result = EntityUtils.toString(entity);
                // 解析成结果Map对象
                Map map = JSON.parseObject(result, Map.class);
                JSONObject hitsMap = (JSONObject) map.get("hits");
                int total = hitsMap.getIntValue("total");
                //log.info("WorkerLogEsServiceImpl返回结果总条数total={}", total);
                if (total > 0) {
                    JSONArray rsArray = hitsMap.getJSONArray("hits");
                    if(rsArray.size()>0) {
                        int insertCount = 0;
                        for (Iterator<Object> it = rsArray.iterator(); it.hasNext(); ) {
                            List strList = new ArrayList();
                            JSONObject obj = (JSONObject) it.next();
                            JSONObject sourceObject = obj.getJSONObject("_source");
                            String ESid = sourceObject.getString("ESid");
                            String clusterId = sourceObject.getString("clusterId");
                            String appId = sourceObject.getString("appId");
                            String name = sourceObject.getString("name");
                            String username = sourceObject.getString("username");
                            String queueName = sourceObject.getString("queueName");
                            String startTime = sourceObject.getString("startTime");
                            String finishTime = sourceObject.getString("finishTime");
                            String trackingUrl = sourceObject.getString("trackingUrl");
                            String jobType = sourceObject.getString("jobType");
                            String severity = sourceObject.getString("severity");
                            String score = sourceObject.getString("score");
                            String jobName = sourceObject.getString("jobName");
                            String jobExecId = sourceObject.getString("jobExecId");
                            String resourceUsed = sourceObject.getString("resourceUsed");
                            String resourceWasted = sourceObject.getString("resourceWasted");
                            String totalDelay = sourceObject.getString("totalDelay");
                            String state = sourceObject.getString("state");
                            String finalStatus = sourceObject.getString("finalStatus");
                            String amContainerLogs = sourceObject.getString("amContainerLogs");
                            String jobDefId = sourceObject.getString("jobDefId");
                            String diagnostics = sourceObject.getString("diagnostics");
                            String prod = args[0];
                            strList.add(ESid);
                            strList.add(clusterId);
                            strList.add(appId);
                            strList.add(name);
                            strList.add(username);
                            strList.add(queueName);
                            strList.add(startTime);
                            strList.add(finishTime);
                            strList.add(trackingUrl);
                            strList.add(jobType);
                            strList.add(severity);
                            strList.add(score);
                            strList.add(jobName);
                            strList.add(jobExecId);
                            strList.add(resourceUsed);
                            strList.add(resourceWasted);
                            strList.add(totalDelay);
                            strList.add(state);
                            strList.add(finalStatus);
                            strList.add(amContainerLogs);
                            strList.add(jobDefId);
                            strList.add(diagnostics);
                            strList.add(prod);
                            String str2 = StringUtils.join(strList,"^^");
                            out.write(str2.replaceAll("\\n","@#@").replaceAll("&#13;", " ").replaceAll("\\t", " ")
                                    .replaceAll("\\r", " "));
                            out.newLine();
                            insertCount++;
                        }
                  //      log.info("WorkerLogEsServiceImpl第{}次插入，条数={}", i++, insertCount);
                        insertTotalCount += insertCount;
                    }
                }
                if(startIndex>total){
                    break;
                }
            }
            //log.info("WorkerLogEsServiceImpl执行完成，总条数={}",insertTotalCount);
            restClient.close();
        } catch (IOException e) {
            //log.info("WorkerLogEsServiceImpl查询出错：{}", e.getMessage());
            e.printStackTrace();
        }finally {
            if(out != null){
                try {
                    out.flush();
                    out.close();;
                } catch (IOException e) {
                    e.printStackTrace();
                }

            }
        }
    }

    public String genereateQueryString(long minTime,long maxTime) {
        IndexRequest indexRequest = new IndexRequest();
        XContentBuilder builder;
        try {
            builder = JsonXContent.contentBuilder()
                    .startObject()
                    .startObject("query")
                    .startObject("range")
                    .startObject("startTime")
                    .field("lt", maxTime)
                    .field("gte",minTime)
                    .endObject()
                    .endObject()
                    .endObject()
                    .startObject("sort")
                    .field("startTime","desc")
                    .endObject()
                    .array("_source","ESid","clusterId","appId","name","username","queueName","startTime","finishTime","trackingUrl","jobType","severity","score","jobName"
                            ,"jobExecId","resourceUsed","resourceWasted","totalDelay","state","finalStatus","amContainerLogs","jobDefId","diagnostics")
                    .endObject();
            indexRequest.source(builder);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        String source = indexRequest.source().utf8ToString();
        return source;
    }

}

pom.xml

 

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.jd</groupId>
    <artifactId>yarnlog</artifactId>
    <packaging>pom</packaging>
    <version>1.0-SNAPSHOT</version>
    <modules>
        <module>main</module>
    </modules>


    <dependencies>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>6.3.2</version>
        </dependency>

        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.7</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.56</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.16.4</version>
        </dependency>

        <dependency>
            <groupId>ch.qos.logback</groupId>
            <artifactId>logback</artifactId>
            <version>0.5</version>
            <type>pom</type>
        </dependency>

        <dependency>
            <groupId>commons-lang</groupId>
            <artifactId>commons-lang</artifactId>
            <version>2.6</version>
        </dependency>

    </dependencies>
</project>