Nginx+Flume+Hive日志采集

流程简介：

 

1.Nginx数据格式

Nginx定义日志格式:

$remote_addr 客户端IP

$time_local 通用日志格式下的本地时间

$status 状态码

$body_bytes_sent 发送给客户端的字节数，不包括响应头的大小

$http_user_agent 客户端浏览器信息

$http_referer 请求的referer地址。

$request 完整的原始请求

$request_method #HTTP请求方法，通常为"GET"或"POST"

$request_time 请求处理时长

$request_uri 完整的请求地址

$server_protocol #服务器的HTTP版本，通常为 "HTTP/1.0" 或 "HTTP/1.1"

$request_body POST请求参数,参数需放form中

 

token $http_token （自定义header字段前加http_，即可将指定的自定义header字段打印到log中。)

version $arg_version （自定义body字段前加arg_，即可将指定的自定义header字段打印到log中。）

 

Nginx配置文件中配置输出日志格式：

log_format user_log_format "$remote_addr,$time_local,$status,$body_bytes_sent,$http_user_agent,$http_referer,$request_method,$request_time,$request_uri,$server_protocol,$request_body,$http_token";

示例：

1.119.140.194,29/Dec/2018:02:08:50 +0000,200,556,okhttp/3.8.1,-,GET,0.028,/phone/resource?type=3,HTTP/1.1,-,-

 

2.Flume采集清洗

flume依赖的包：

flume-conf.properties：

# agent
a1.sources = s1
a1.channels = c1
a1.sinks = k1

# sources
a1.sources.s1.type = spooldir
a1.sources.s1.channels = c1
# FTP路径
a1.sources.s1.spoolDir = /var/ftp/
# 自定义拦截器
a1.sources.s1.interceptors = f1
a1.sources.s1.interceptors.f1.type = com.hx.common.flume.FlumeBuilder

# sink
a1.sinks.k1.type=hive
a1.sinks.k1.channel = c1
# hive地址
a1.sinks.k1.hive.metastore=thrift://11.11.11.11:9083
a1.sinks.k1.hive.database=hive_test
a1.sinks.k1.hive.table=nginx_acc_log
a1.sinks.k1.serializer=delimited
# 输入分隔符
a1.sinks.k1.serializer.delimiter=","
# 输出分隔符
a1.sinks.k1.serializer.serdeSeparator=','
a1.sinks.k1.serializer.fieldnames=remote_addr,time_local,status,body_bytes_sent,http_user_agent,http_referer,request_method,request_time,request_uri,server_protocol,request_body,http_token,id,appkey,sing,version

# channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 100

 

启动flume：

nohup ./flume-ng agent -c /opt/flume/apache-flume/conf -f /opt/flume/apache-flume/conf/flume-conf.properties -n a1 -Dflume.root.logger=INFO,console &

 

经过Flume拦截器处理后字段:

remote_addr,time_local,status,body_bytes_sent,http_user_agent,http_referer,request_method,request_time,request_uri,server_protocol,request_body,http_token,id,appkey,sing,version

 

自定义拦截器Common-1.1.0.jar包

  compile ('org.apache.flume:flume-ng-sdk:1.8.0')
            {
                exclude group: 'org.slf4j'
            }
    compile ('org.apache.flume:flume-ng-core:1.8.0')
            {
                exclude group: 'org.slf4j'
            }

package com.hx;

import org.apache.flume.Context;
import org.apache.flume.interceptor.Interceptor;


public class FlumeBuilder implements Interceptor.Builder {

    @Override
    public void configure(Context context) {

    }

    @Override
    public Interceptor build() {
        return new FlumeInterceptor();
    }

}

package com.hx;

import com.google.common.base.Charsets;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Locale;
import java.util.Map;


public class FlumeInterceptor implements Interceptor {

    RedisClientService redisClient = new RedisClientService();

    @Override
    public void close() {

    }

    @Override
    public void initialize() {

    }

    @Override
    public Event intercept(Event event) {
        if (event == null) {
            return null;
        }

        // 通过获取event数据，转化成字符串
        Map<String, String> headerMap = event.getHeaders();
        headerMap.put("timestamp", String.valueOf(System.currentTimeMillis()));
        System.out.println("采集的header信息:" + headerMap.toString());
        String body = new String(event.getBody(), Charsets.UTF_8);
        System.out.println("采集的body信息:" + body);

//0  remote_addr     11.11.11.11,
//1  time_local      29/Dec/2018:02:08:50 +0000,
//2  status      200,
//3  body_bytes_sent     556,
//4  http_user_agent     okhttp/3.8.1,
//5  http_referer    -,
//6  request_method    GET  POST,
//7  request_time    0.028,
//8  request_uri     /api/account/phone/resource?appkey=3b20168fdc76b&channel=10&sign=9b553673437f4af3&type=3,
//9  server_protocol     HTTP/1.1,
//10  request_body    -   aa=aaaaa,
//11  token   bb01-c13b-4f4d

        StringBuilder sb = new StringBuilder();
        if (StringUtils.isNotEmpty(body)) {
            try {
                String[] body_sp = body.split(",");
                //  remote_addr
                sb.append(body_sp[0] + ",");
                //  status,
                sb.append(body_sp[2] + ",");
                //  body_bytes_sent
                sb.append(body_sp[3] + ",");
                //  http_user_agent
                sb.append(body_sp[4] + ",");
                //  request_method
                sb.append(body_sp[6] + ",");
                //  request_time
                sb.append(body_sp[7] + ",");

                String request_method = body_sp[6];
                Map<String, String> params = null;
                if ("GET".equals(request_method)) {
                    String request_uri = body_sp[8];
                    params = AnalysisUri.UrlRequest(request_uri);

                    //  request_uri
                    sb.append(AnalysisUri.UrlPage(request_uri) + ",");
                    //  server_protocol
                    sb.append(body_sp[9] + ",");
                    //  request_body
                    sb.append(AnalysisUri.TruncateUrlPage(request_uri) + ",");
                } else if ("POST".equals(request_method)) {
                    String request_body = body_sp[10];
                    params = AnalysisUri.NoUrlRequest(request_body);

                    //  request_uri
                    sb.append(body_sp[8] + ",");
                    //  server_protocol
                    sb.append(body_sp[9] + ",");
                    //  request_body
                    sb.append(request_body + ",");
                } else {
                    // 无效请求
                    params = null;
                }
                if (params != null) {
                    String token = body_sp[11];
                    sb.append(token + ",");
                    sb.append(params.get("appkey") + ",");
                    sb.append(params.get("version") + ",");
                    sb.append(params.get("channel") + ",");
                    sb.append(params.get("checktype") + ",");

                    String uid = getIdByToken(token, 0);
                    String box_id = getIdByToken(token, 1);
                    String device_type;
                    if (uid != null) {
                        device_type = "phone";
                        sb.append(uid + ",");
                    } else if (box_id != null) {
                        device_type = "box";
                        sb.append(box_id + ",");
                    } else {
                        device_type = "other";
                        sb.append(null + ",");
                    }
                    sb.append(device_type + ",");

                    // yyyy-MM-dd HH:mm:ss
                    String c_time = timeFormat(body_sp[1]);
                    sb.append(c_time + ",");
                    String dates = c_time.substring(0, 10);
                    sb.append(dates + ",");
                    String hours = c_time.substring(11, 13);
                    sb.append(hours);

                    event.setBody(sb.toString().getBytes(Charsets.UTF_8));
                    System.out.println("清洗后的body信息:" + sb.toString());
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return event;
    }

    @Override
    public List<Event> intercept(List<Event> events) {
        for (final Event event : events) {
            intercept(event);
        }
        return events;
    }

    /**
     * 获取id
     *
     * @param token
     * @return
     */
    public String getIdByToken(String token, int type) {
        try {
            String id = redisClient.getStr("token:" + token);
            
            String k = "token:" + type + ":" + id;
            if (redisClient.exist(k) && token.equals(redisClient.getStr(k))) {
                return id;
            } else {
                return null;
            }
        } catch (Exception e) {
            return null;
        }
    }

    /**
     * nginx日期格式化
     *
     * @param time
     * @return
     */
    private String timeFormat(String time) {
        SimpleDateFormat sourceFormat = new SimpleDateFormat("dd/MMM/yyyy:hh:mm:ss Z", Locale.ENGLISH);
        SimpleDateFormat transferFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        String fTime = "";
        try {
            fTime = transferFormat.format(sourceFormat.parse(time));
        } catch (ParseException e) {
            e.printStackTrace();
        }
        return fTime;
    }

}

package com.hx;

import java.util.HashMap;
import java.util.Map;

/**
 * 解析出url请求的路径和参数键值
 * 
 */
public class AnalysisUri {

    /**
     * 解析出url请求的路径，包括页面
     *
     * @param strURL url地址
     * @return url路径
     */
    public static String UrlPage(String strURL) {
        String strPage = null;
        String[] arrSplit = null;

//        strURL = strURL.trim();

        arrSplit = strURL.split("[?]");
        if (strURL.length() > 0) {
            if (arrSplit.length > 1) {
                if (arrSplit[0] != null) {
                    strPage = arrSplit[0];
                }
            }
        }

        return strPage;
    }

    /**
     * 去掉url中的路径，留下请求参数部分
     *
     * @param strURL url地址
     * @return url请求参数部分
     */
    public static String TruncateUrlPage(String strURL) {
        String strAllParam = null;
        String[] arrSplit = null;

//        strURL = strURL.trim();

        arrSplit = strURL.split("[?]");
        if (strURL.length() > 1) {
            if (arrSplit.length > 1) {
                if (arrSplit[1] != null) {
                    strAllParam = arrSplit[1];
                }
            }
        }

        return strAllParam;
    }

    /**
     * 解析出url参数中的键值对
     * 如 "index.jsp?Action=del&id=123"，解析出Action:del,id:123存入map中
     *
     * @param URL url地址
     * @return url请求参数部分
     */
    public static Map<String, String> UrlRequest(String URL) {
        Map<String, String> mapRequest = new HashMap<String, String>();

        String[] arrSplit = null;

        String strUrlParam = TruncateUrlPage(URL);
        if (strUrlParam == null) {
            return mapRequest;
        }
        //每个键值为一组 www.2cto.com
        arrSplit = strUrlParam.split("[&]");
        for (String strSplit : arrSplit) {
            String[] arrSplitEqual = null;
            arrSplitEqual = strSplit.split("[=]");

            //解析出键值
            if (arrSplitEqual.length > 1) {
                //正确解析
                mapRequest.put(arrSplitEqual[0], arrSplitEqual[1]);

            } else {
                if (arrSplitEqual[0] != "") {
                    //只有参数没有值，不加入
                    mapRequest.put(arrSplitEqual[0], "");
                }
            }
        }
        return mapRequest;
    }

    /**
     * 解析出url参数中的键值对
     * 如 "index.jsp?Action=del&id=123"，解析出Action:del,id:123存入map中
     *
     * @param URL url地址
     * @return url请求参数部分
     */
    public static Map<String, String> NoUrlRequest(String URL) {
        Map<String, String> mapRequest = new HashMap<String, String>();

        String[] arrSplit = null;

        arrSplit = URL.split("[&]");
        for (String strSplit : arrSplit) {
            String[] arrSplitEqual = null;
            arrSplitEqual = strSplit.split("[=]");

            //解析出键值
            if (arrSplitEqual.length > 1) {
                //正确解析
                mapRequest.put(arrSplitEqual[0], arrSplitEqual[1]);

            } else {
                if (arrSplitEqual[0] != "") {
                    //只有参数没有值，不加入
                    mapRequest.put(arrSplitEqual[0], "");
                }
            }
        }
        return mapRequest;
    }


}

3.Hive操作

先切换账户：

su hadoop

启动hive：

nohup hive --service metastore >> ~/metastore.log 2>&1 &        

nohup  hive --service hiveserver2 >> ~/hiveserver2.log 2>&1 &   

 

修改权限：

hdfs dfs -ls /user/hive/warehouse

hadoop dfs -chmod 777 /user/hive/warehouse/hive_test.db

 

建表语句：（与flume输出一致）

DROP TABLE IF EXISTS nginx_acc_log;

 

create table nginx_acc_log(remote_addr string,time_local string,status string,body_bytes_sent string,http_user_agent string,http_referer string,request_method string,request_time string,request_uri string,server_protocol string,request_body string,http_token string,id string,appkey string,sing string,version string) clustered by (id) into 5 buckets stored as orc TBLPROPERTIES ('transactional'='true');

 

设置hive事务：

<!--支持事务-->
<property>
    <name>hive.support.concurrency</name>
    <value>true</value>
</property>
<property>
    <name>hive.exec.dynamic.partition.mode</name>
    <value>nonstrict</value>
</property>
<property>
    <name>hive.txn.manager</name>
    <value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
</property>
<property>
    <name>hive.compactor.initiator.on</name>
    <value>true</value>
</property>
<property>
    <name>hive.compactor.worker.threads</name>
    <value>5</value>
</property>
<property>
    <name>hive.enforce.bucketing</name>
    <value>true</value>
</property>

 

显示表：

hive> show databases;

hive> use hive_test;

hive> show tables;

hive> select * from nginx_acc_log;