nginx 日志入库

最新推荐文章于 2024-06-20 11:23:27 发布

Cincinnati_De

最新推荐文章于 2024-06-20 11:23:27 发布

阅读量1.1k

点赞数 1

分类专栏：个人日记

本文链接：https://blog.csdn.net/Cincinnati_De/article/details/84308535

版权

个人日记专栏收录该内容

142 篇文章 4 订阅

订阅专栏

CREATE TABLE  tmp7_nginx_log(
   host_ip STRING,
   port STRING,
   event_01 STRING,
   uri_time STRING,
   uri STRING,
   status_01 STRING,
   status_02 STRING,
   referer STRING,
   ua STRING,
   video_url STRING,
   event_02 STRING,
   unkonw_numbe STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe'
WITH SERDEPROPERTIES (
   "input.regex" = "([^ ]*) ([^ ]*) (- -) (\\[.*\\]) (\".*?\") (-|[0-9]*) (-|[0-9]*) (-\"[^ ]*\") (\".*?\") ([^ ]*) (-) ([^ ]*)",
   "output.format.string" = "%1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s %10$s %11$s %12$s"
)
STORED AS TEXTFILE;

日志:

111.27.54.11 21550 - - [06/Nov/2018:21:59:58 +0800] "GET /jhy/1.1.5/set/GetState?uid=7627064&tick=1541512797&ver=134&key=f5d16f39bd1b0ae723ce86add7bbbe01 HTTP/1.1" 200 95 -"https://servicewechat.com/wxaa46e77919aec8d9/50/page-frame.html" "Mozilla/5.0 (Linux; Android 6.0.1; OPPO A57t Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070338) NetType/WIFI Language/zh_CN Process/appbrand3" mas.wanzhushipin.cn - 0.014
106.8.225.70 26922 - - [06/Nov/2018:21:59:59 +0800] "GET /jhy/1.1.5/set/gameFromAppid?appid=wx7ff042bcb43bee59&tick=1541512793&ver=134&key=92240944ab93204d421f2120fe0fde55 HTTP/1.1" 200 233 -"https://servicewechat.com/wxaa46e77919aec8d9/50/page-frame.html" "Mozilla/5.0 (Linux; Android 6.0.1; vivo Y66 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/51.0.2704.81 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070338) NetType/WIFI Language/zh_CN Process/appbrand0" mas.wanzhushipin.cn - 0.004
125.84.179.70 30104 - - [06/Nov/2018:21:59:59 +0800] "GET /uo/1.1.6/uo/querycards?gift=0&tick=1541512798&uid=31847539&key=4c1da2bcc56d7533324ff0d7096e6b59 HTTP/1.1" 200 96 -"https://servicewechat.com/wx3c889b4f402e924e/102/page-frame.html" "Mozilla/5.0 (Linux; Android 6.0.1; Redmi Note 3 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070333) NetType/WIFI Language/zh_CN Process/appbrand1" mas.wanzhushipin.cn - 0.003
101.248.96.34 30448 - - [06/Nov/2018:21:59:59 +0800] "GET /uo/1.1.6/uo/querycards?gift=0&tick=1541512796&uid=32606679&key=8275ef8970a83167201d617aee463f28 HTTP/1.1" 200 93 -"https://servicewechat.com/wx3c889b4f402e924e/102/page-frame.html" "Mozilla/5.0 (Linux; Android 7.0; BND-AL10 Build/HONORBND-AL10; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070337) NetType/4G Language/zh_CN Process/appbrand2" mas.wanzhushipin.cn - 0.004
119.103.136.94 30516 - - [06/Nov/2018:21:59:59 +0800] "GET /uo/1.1.6/uo/querycards?gift=0&tick=1541512797&uid=32635487&key=7f488dd58be96a785269b10bff0e8888 HTTP/1.1" 200 94 -"https://servicewechat.com/wx3c889b4f402e924e/102/page-frame.html" "Mozilla/5.0 (Linux; Android 8.1.0; PADM00 Build/O11019; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070338) NetType/4G Language/zh_CN Process/appbrand1" mas.wanzhushipin.cn - 0.004
110.166.198.24 14343 - - [06/Nov/2018:21:59:59 +0800] "GET /jhy/2.0.0/set/config?ver=134&tick=1541512798&key=7747d47914dd096e0da363dc7a5dad8b HTTP/1.1" 200 75 -"https://servicewechat.com/wxaa46e77919aec8d9/50/page-frame.html" "Mozilla/5.0 (Linux; Android 7.0; SM-G9280 Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/64.0.3282.137 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070338) NetType/WIFI Language/zh_CN Process/appbrand0" mas.wanzhushipin.cn - 0.004
27.156.27.212 16903 - - [06/Nov/2018:21:59:59 +0800] "GET /uo/1.2.8/uo/data?id=102&tick=1541512799&uid=32634720&key=31ba24d423853043c93c568a4e52702b HTTP/1.1" 200 47 -"https://servicewechat.com/wx3c889b4f402e924e/102/page-frame.html" "Mozilla/5.0 (iPhone; CPU iPhone OS 11_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15G77 MicroMessenger/6.7.3(0x16070321) NetType/WIFI Language/zh_CN" mas.wanzhushipin.cn - 0.007
114.138.89.161 31530 - - [06/Nov/2018:21:59:59 +0800] "GET /uo/1.2.2/uo/ad?agentid=0&tick=1541512799&type=0&uid=30925261&key=2d5a04646c2807ebd923c3519d3a639c HTTP/1.1" 200 47 -"https://servicewechat.com/wx3c889b4f402e924e/102/page-frame.html" "Mozilla/5.0 (Linux; Android 7.1.2; vivo X9 Build/N2G47H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070338) NetType/WIFI Language/zh_CN Process/appbrand2" mas.wanzhushipin.cn - 0.006
27.106.134.4 23830 - - [06/Nov/2018:21:59:59 +0800] "GET /jhy/1.1.5/set/RecentlyGame?uid=6021749&tick=1541512799&ver=134&key=e2aaef04d318f6ac2700a6089a20a12e HTTP/1.1" 200 985 -"https://servicewechat.com/wxaa46e77919aec8d9/50/page-frame.html" "Mozilla/5.0 (Linux; Android 7.0; TRT-AL00 Build/HUAWEITRT-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/64.0.3282.137 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070338) NetType/WIFI Language/zh_CN Process/appbrand1" mas.wanzhushipin.cn - 0.019
182.104.70.46 23964 - - [06/Nov/2018:21:59:59 +0800] "GET /uo/1.1.5/uo/report?continue=0&help=0&id=0&level=0&mark=91.54330500000042&shares=0&tick=1541512799&uid=32465574&key=1cdeadc76fbc34414d833312f8ffef8a HTTP/1.1" 200 91 -"https://servicewechat.com/wx3c889b4f402e924e/102/page-frame.html" "Mozilla/5.0 (Linux; Android 8.1.0; PADM00 Build/O11019; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070338) NetType/4G Language/zh_CN Process/appbrand1" mas.wanzhushipin.cn - 0.007
117.136.24.132 47577 - - [06/Nov/2018:21:02:07 +0800] "GET /jhy/1.1.5/set/lists?agentid=8&uid=5616441&page=1&sex=2&standby=0&tick=1541509325&ver=134&key=cc7ddba60fd1d1247956c545487dc65e HTTP/1.1" 200 4803 -"https://servicewechat.com/wxaa46e77919aec8d9/50/page-frame.html" "Mozilla/5.0 (Linux; Android 7.1.1; OPPO R11s Plus Build/NMF26X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070338) NetType/4G Language/zh_CN Process/appbrand3" mas.wanzhushipin.cn - 0.115

"27.38.5.159" "-" "31/Aug/2015:00:04:37 +0800" "GET /course/view.php?id=27 HTTP/1.1" "303" "440" - "http://www.ibeifeng.com/user.php?act=mycourse" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36" "-" "learn.ibeifeng.com"
"27.38.5.159" "-" "31/Aug/2015:00:04:37 +0800" "GET /login/index.php HTTP/1.1" "303" "465" - "http://www.ibeifeng.com/user.php?act=mycourse" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36" "-" "learn.ibeifeng.com"

感谢一些博客的帮助，自己写出来了。

参照: https://www.jianshu.com/p/da1f9bc326ee