- 本配置主要是在针对视频点播网站做日志分析清洗的时候的开发配置，主要有三个方面可以参考配置，正则表达式以及输出时的模版
- 当数据量很大时，正则表达式的好坏决定了，清洗的效率也是导致效率低下最容易的节点。
- 发出此开发的配置，一是方便自己记录，二是方便有需求的同志们。
针对版本1.4.2
运行 logstash -f D:\logstash-1.4.2\config\shipper.config
以下是配置文件shipper.config,正则表达式,以及输出template
input {
#node part
# file{
# type=>"dbagent_access"
# path=>"/mnt/develop/elk/logs/张小寒/access.log"
# start_position=>"beginning"
# }
file{
type=>"dbagent_error"
path=>"/mnt/develop/elk/logs/张小寒/error.log"
start_position=>"beginning"
}
}
filter {
if[type]=="cdn"
{
#this is a flash player error
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
match=>["message","%{CDN_PATTERN_HTTP_UUID}"]
match=>["message","%{CDN_PATTERN_HTTP}"]
match=>["message","%{CDN_PATTERN_MOVIEINFO_HTTP}"]
match=>["message","%{CDN_PATTERN_MOVIEINFO}"]
match=>["message","%{CDN_PATTERN_PCVCR_HTTP}"]
match=>["message","%{CDN_PATTERN_PCVCR}"]
match=>["message","%{CDN_PATTERN_VCR_HTTP}"]
match=>["message","%{CDN_PATTERN_VCR}"]
match=>["message","%{CDN_PATTERN_prvcr_HTTP}"]
match=>["message","%{CDN_PATTERN_prvcr}"]
add_tag=>["%{[type]}","grokked","%{[etype]}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
if "unknown"==[chost] or ![chost]
{#because when client ip is 'unknown', we do not need this value
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["chost"]
}
}else{
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}
if [chost]
{
geoip {
source => "chost"
target=>"cgeo"
remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
#database=>["path"]
}
}
#ruby{
# in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
# code=>"
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# "
#}
mutate{
remove_field=>["path","@version","@timestamp","type","etype"]
convert => [ "status", "integer" ]
}
}else if [type]=="nginx_access"
{
grok{
break_on_match=>true
#condition 1
match=>["message","%{NGINX_ACCESS}"]
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
add_tag=>["%{[type]}","grokked","access"]
}
# if "_grokparsefailure" in [tags]
# {
# drop{}
# }
geoip {
source => "chost"
target=>"cgeo"
remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
#database=>["path"]
}
#ruby{
# in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
# code=>"
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# "
#}
mutate{
gsub => [
"a_month", "Jan(?:uary)?", "01",
"a_month", "Feb(?:ruary)?", "02",
"a_month", "Mar(?:ch)?", "03",
"a_month", "Apr(?:il)?", "04",
"a_month", "May", "05",
"a_month", "Jun(?:e)?", "06",
"a_month", "Jul(?:y)?", "07",
"a_month", "Aug(?:ust)?", "08",
"a_month", "Sep(?:tember)?", "09",
"a_month", "Oct(?:ober)?", "10",
"a_month", "Nov(?:ember)?", "11",
"a_month", "Dec(?:ember)?", "12"
]
add_field =>{
"cdn_date" => "%{a_year}-%{a_month}-%{a_day}"
"timestamp" =>"%{cdn_date} %{time}"
}
remove_field=>["path","@version","@timestamp","type","time","a_day","a_month","a_year","message"]
convert=>["status","integer"]
}
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}else if [type]=="nginx_error"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
match=>["message","%{NGINX_ERROR_1301}"]
match=>["message","%{NGINX_ERROR}"]
add_tag=>["%{[type]}","grokked","error"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
geoip {
source => "chost"
target=>"cgeo"
remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
#database=>["path"]
}
ruby {
#note 1 ruby need to be put before mutate
#note 2 if not run or correctly run ruby ,please install jruby and try it again
#in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
code => "
#IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
event['interval']=event['rtime'].to_i-event['qtime'].to_i
"
}
mutate{
strip => ["info"]
convert=>{
"status"=>"integer"
}
add_field => {
"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
"timestamp" =>"%{cdn_date} %{time}"
}
remove_field=>["path","@version","@timestamp","type","time","e_year","e_month","e_day","message","qtime","rtime"]
}
date {
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}else if [type]=="res-monitor"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
match=>["message","%{RESMONITOR_PATTERN_VRB}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
add_tag=>["%{[type]}","grokked","%{[etype]}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
ruby {
#note 1 ruby need to be put before mutate
#note 2 if not run or correctly run ruby ,please install jruby and try it again
#in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
code => "
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# get date from path
event['e_year']=event['path'][-12..-9]
event['e_month']=event['path'][-8..-7]
event['e_day']=event['path'][-6..-5]
"
}
mutate{
add_field => {
"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
}
remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
}
if [etype]=="VRB"
{
date {
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["message","etype"]
}
}else{
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["etype"]
}
}
}else if [type]=="cdndelivery_filecollect"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
match=>["message","%{FILECOLLECT_PATTERN_VRB}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
add_tag=>["%{[type]}","grokked","%{[etype]}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
ruby {
#note 1 ruby need to be put before mutate
#note 2 if not run or correctly run ruby ,please install jruby and try it again
#in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
code => "
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# get date from path
event['e_year']=event['path'][-12..-9]
event['e_month']=event['path'][-8..-7]
event['e_day']=event['path'][-6..-5]
"
}
mutate{
add_field => {
"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
}
remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
}
if[etype]=="VRB"
{
date {
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["message","etype"]
}
}else{
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["etype"]
}
}
}else if [type]=="cdndelivery_nodeschedule"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
#I can not find the VRB event here ,so I ignore it first
#match=>["message","%{NODESCHEDULE_PATTERN_VRB}"]
match=>["message","%{NODESCHEDULE_PATTERN_INF}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
add_tag=>["%{[type]}","grokked","%{[etype]}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
ruby {
#note 1 ruby need to be put before mutate
#note 2 if not run or correctly run ruby ,please install jruby and try it again
#in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
code => "
#IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# get date from path
event['e_year']=event['path'][-12..-9]
event['e_month']=event['path'][-8..-7]
event['e_day']=event['path'][-6..-5]
"
}
mutate{
add_field => {
"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
}
remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
}
if [etype]=="VRB" or [etype]=="INF"
{
date {
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["message","etype"]
}
}else{
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["etype"]
}
}
}else if [type]=="cdndelivery_storageschedule"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
#I can not find the VRB event here ,so I ignore it first
#match=>["message","%{STORAGESCHEDULE_PATTERN_VRB}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
add_tag=>["%{[type]}","grokked","%{[etype]}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
ruby {
#note 1 ruby need to be put before mutate
#note 2 if not run or correctly run ruby ,please install jruby and try it again
#in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
code => "
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# get date from path
event['e_year']=event['path'][-12..-9]
event['e_month']=event['path'][-8..-7]
event['e_day']=event['path'][-6..-5]
"
}
mutate{
add_field => {
"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
}
remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
}
if [etype]=="VRB"
{
date {
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["message","etype"]
}
}else{
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["etype"]
}
}
}else if [type]=="cdndelivery_notify"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
#I can not find the VRB event here ,so I ignore it first
#match=>["message","%{NOTIFY_PATTERN_VRB}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
add_tag=>["%{[type]}","grokked","%{[etype]}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
ruby {
#note 1 ruby need to be put before mutate
#note 2 if not run or correctly run ruby ,please install jruby and try it again
#in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
code => "
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# get date from path
event['e_year']=event['path'][-12..-9]
event['e_month']=event['path'][-8..-7]
event['e_day']=event['path'][-6..-5]
"
}
mutate{
add_field => {
"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
}
remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
}
if[etype]=="VRB"
{
date {
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["message","etype"]
}
}else{
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
remove_field=>["etype"]
}
}
}else if [type]=="statout"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
#I can not find the VRB event here ,so I ignore it first
#match=>["message","%{NOTIFY_PATTERN_VRB}"]
match=>["message","%{STATOUT_PATTERN}"]
add_tag=>["%{[type]}","grokked","statout"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
ruby {
#note 1 ruby need to be put before mutate
#note 2 if not run or correctly run ruby ,please install jruby and try it again
#in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
code => "
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# get date from path
event['e_year']=event['path'][-18..-15]
event['e_month']=event['path'][-14..-13]
event['e_day']=event['path'][-12..-11]
"
}
mutate{
add_field => {
"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
"timestamp" =>"%{cdn_date} %{e_time}"
}
remove_field=>["path","@version","@timestamp","type","e_type","message","e_year","e_month","e_day","e_time"]
}
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}
else if [type]=="dbagent_access"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
match=>["message","%{DBAGENT_ACCESS}"]
add_tag=>["%{[type]}","grokked","access"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
if [status]=="200"
{
#200 is normal http status, here we just need exception status,so drop 200
drop{}
}
#ruby{
# in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
# code=>"
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# "
#}
mutate{
gsub => [
"a_month", "Jan(?:uary)?", "01",
"a_month", "Feb(?:ruary)?", "02",
"a_month", "Mar(?:ch)?", "03",
"a_month", "Apr(?:il)?", "04",
"a_month", "May", "05",
"a_month", "Jun(?:e)?", "06",
"a_month", "Jul(?:y)?", "07",
"a_month", "Aug(?:ust)?", "08",
"a_month", "Sep(?:tember)?", "09",
"a_month", "Oct(?:ober)?", "10",
"a_month", "Nov(?:ember)?", "11",
"a_month", "Dec(?:ember)?", "12"
]
add_field =>{
"cdn_date" => "%{a_year}-%{a_month}-%{a_day}"
"timestamp" =>"%{cdn_date} %{time}"
}
remove_field=>["path","@version","@timestamp","type","time","a_day","a_month","a_year","message"]
convert=>["status","integer"]
}
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}else if [type]=="dbagent_error"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
match=>["message","%{DBAGENT_ERROR_3}"]
match=>["message","%{DBAGENT_ERROR_2}"]
match=>["message","%{DBAGENT_ERROR_1}"]
add_tag=>["%{[type]}","grokked","error"]
}
if "_grokparsefailure" in [tags]
{
#drop{}
}else
{
mutate{
add_field => {
"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
"timestamp" =>"%{cdn_date} %{time}"
}
remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","time"]
}
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}
}else if[type]=="media-info_HashValidService_info" or [type]=="media-info_HashValidService_error" or [type]=="media-info_PicKeyFramesService_info" or [type]=="media-info_PicKeyFramesService_error" or [type]=="media-info_PollCDNFileQueueService_CdnInterFaceLog_error_error" or [type]=="media-info_PollCDNFileQueueService_CdnInterFaceLog_error_info"
or [type]=="media-info_PollCDNFileQueueService_PollCDNFileQueue_error_error" or [type]=="media-info_PollCDNFileQueueService_PollCDNFileQueue_error_info" or [type]=="media-info_CdnPost_error" or [type]=="media-info_CdnPost_info" or [type]=="media-info_PassThroughService_error" or [type]=="media-info_PassThroughService_info" or [type]=="media-info_DispatchService_DispatchService_error" or [type]=="media-info_DispatchService_DispatchService_info" or [type]=="media-info_DispatchService_MessageQueue_error" or [type]=="media-info_DispatchService_MessageQueue_info" or [type]=="media-info_AssetIntefaceWebservice_error" or [type]=="media-info_AssetIntefaceWebservice_info" or [type]=="media-info_MgBoss_Asset_error" or [type]=="media-info_MgBoss_Asset_info" or [type]=="media-info_MgBoss_CDN_error" or [type]=="media-info_MgBoss_CDN_info" or [type]=="media-info_MgBoss_DataAccess_error" or [type]=="media-info_MgBoss_DataAccess_info" or [type]=="media-info_MgBoss_EFSQL_error" or [type]=="media-info_MgBoss_EFSQL_info" or [type]=="media-info_MgBoss_Power_error" or [type]=="media-info_MgBoss_Power_info" or [type]=="media-info_MgBoss_Sys_error" or [type]=="media-info_MgBoss_Sys_info" or [type]=="media-info_MgBoss_Template_error" or [type]=="media-info_MgBoss_Template_info"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
match=>["message","%{INFOERROR_PATTERN}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
ruby{
# in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
code=>"
tmpArray=event['type'].split('_');
if tmpArray.length==4
event['appname']=tmpArray[1]
event['category']=tmpArray[2]
elsif tmpArray.length==3
event['appname']=tmpArray[1]
end
event['index_type']=tmpArray[tmpArray.length-1]
"
}
mutate{
add_field => {"timestamp" =>"%{e_date} %{e_time}"}
remove_field => ["path","@version","@timestamp","message","e_time"]
remove_tag => ["multiline"]
}
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}else if [type]=="media-info_LiveProgramList"
{
grok{
break_on_match=>true
#condition 1
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
match=>["message","%{LOGOFONTIMERUN}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
ruby{
# in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
code=>"
event['appname']='LiveProgramList'
event['index_type']='LiveProgramList'
"
}
mutate{
add_field=>{"timestamp" =>"%{e_date} %{e_time}"}
remove_field=>["path","@version","@timestamp","message","e_time"]
}
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}else if [type]=="router"
{
grok{
break_on_match=>true
#condition 1
match=>["message","%{ROUTER_OK}"]
match=>["message","%{ROUTER_ERROR}"]
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
add_tag=>["%{[type]}","grokked","%{[status]}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
#geoip {
# source => "chost"
# target=>"cgeo"
# remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
# #database=>["path"]
#}
# ruby{
# in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
# code=>"
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
# "
# }
mutate{
add_field =>{
"date"=>"%{year}-%{month}-%{day}"
"timestamp" =>"%{date} %{time}"
}
remove_field=>["path","@version","@timestamp","type","time","day","month","year","message","time"]
}
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}else if [type]=="routerdll"
{
grok{
break_on_match=>true
#condition 1
match=>["message","%{ROUTERDLL_ERR}"]
patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
add_tag=>["%{[type]}","grokked","%{[etype]}"]
}
if "_grokparsefailure" in [tags]
{
drop{}
}
#geoip {
# source => "uip"
# target=>"cgeo"
# remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
# #database=>["path"]
#}
ruby{
# in linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
code=>"
x=event['path'].index(/_/)
# puts x
#puts event['path']
#puts event['type']
tmpDate=event['path'][x+1..x+1+8]
event['date']=tmpDate[0..3]+'-'+tmpDate[4..5]+'-'+tmpDate[6..7]
# IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
"
}
mutate{
add_field =>{
"timestamp" =>"%{date} %{time}"
}
remove_field=>["path","@version","@timestamp","type","time","a_day","a_month","a_year","message","time"]
}
date {
#use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
timezone=>"Etc/UTC"
match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
target=>"timestamp"
}
}
}
output {
#stdout { codec => rubydebug }
if "cdn" in [tags]
{
#this is a flash player error
# file{
# path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-prvcr.log"
# }
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn.json"
template_name=>"cdn"
template_overwrite=>true
index=>"cdn_%{cdn_date}"
index_type=>"%{[tags][2]}"
}
}else if "nginx_access" in [tags] or "nginx_error" in [tags]
{
# file{
# path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/nginx.log"
# }
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/nginx.json"
template_name=>"nginx"
template_overwrite=>true
index=>"nginx_%{cdn_date}"
index_type=>"%{[tags][2]}"
}
}else if "res-monitor" in [tags]
{
# file{
# path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/res-monitor.log"
# }
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/res-monitor.json"
template_name=>"res-monitor"
template_overwrite=>true
index=>"res-monitor_%{cdn_date}"
index_type=>"%{[tags][2]}"
}
}else if "cdndelivery_filecollect" in [tags]
{
# file{
# path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-delivery.log"
# }
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn-delivery.json"
template_name=>"cdn-delivery"
template_overwrite=>true
index=>"cdn-delivery_%{cdn_date}"
index_type=>"%{[tags][2]}-filecollect"
}
}else if "cdndelivery_nodeschedule" in [tags]
{
# file{
# path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-delivery.log"
# }
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn-delivery.json"
template_name=>"cdn-delivery"
template_overwrite=>true
index=>"cdn-delivery_%{cdn_date}"
index_type=>"%{[tags][2]}-nodeschedule"
}
}else if "cdndelivery_storageschedule" in [tags]
{
# file{
# path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-delivery.log"
# }
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn-delivery.json"
template_name=>"cdn-delivery"
template_overwrite=>true
index=>"cdn-delivery_%{cdn_date}"
index_type=>"%{[tags][2]}-storageschedule"
}
}else if "cdndelivery_notify" in [tags]
{
# file{
# path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-delivery.log"
# }
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn-delivery.json"
template_name=>"cdn-delivery"
template_overwrite=>true
index=>"cdn-delivery_%{cdn_date}"
index_type=>"%{[tags][2]}-notify"
}
}else if "statout" in [tags]
{
#file{
# path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/statout.log"
#}
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/statout.json"
template_name=>"statout"
template_overwrite=>true
index=>"statout_%{cdn_date}"
index_type=>"error"
}
}else if "dbagent_access" in [tags]
{
file{
path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/dbagent_access.log"
}
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/dbagent.json"
template_name=>"dbagent"
template_overwrite=>true
index=>"dbagent_%{cdn_date}"
index_type=>"access"
}
}else if "dbagent_error" in [tags]
{
file{
path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/dbagent_error.log"
}
elasticsearch
{
protocol=>"http"
cluster=>"hunantv_cdn"
host=>"localhost"
bind_host=>"localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/dbagent.json"
template_name=>"dbagent"
template_overwrite=>true
index=>"dbagent_%{cdn_date}"
index_type=>"error"
}
}
else if [type]=="media-info_HashValidService_info" or [type]=="media-info_HashValidService_error" or [type]=="media-info_PicKeyFramesService_info" or [type]=="media-info_PicKeyFramesService_error" or [type]=="media-info_PollCDNFileQueueService_CdnInterFaceLog_error_error" or [type]=="media-info_PollCDNFileQueueService_CdnInterFaceLog_error_info"
or [type]=="media-info_PollCDNFileQueueService_PollCDNFileQueue_error_error" or [type]=="media-info_PollCDNFileQueueService_PollCDNFileQueue_error_info" or [type]=="media-info_CdnPost_error" or [type]=="media-info_CdnPost_info" or [type]=="media-info_PassThroughService_error" or [type]=="media-info_PassThroughService_info" or [type]=="media-info_DispatchService_DispatchService_error" or [type]=="media-info_DispatchService_DispatchService_info" or [type]=="media-info_DispatchService_MessageQueue_error" or [type]=="media-info_DispatchService_MessageQueue_info" or [type]=="media-info_AssetIntefaceWebservice_error" or [type]=="media-info_AssetIntefaceWebservice_info" or [type]=="media-info_MgBoss_Asset_error" or [type]=="media-info_MgBoss_Asset_info" or [type]=="media-info_MgBoss_CDN_error" or [type]=="media-info_MgBoss_CDN_info" or [type]=="media-info_MgBoss_DataAccess_error" or [type]=="media-info_MgBoss_DataAccess_info" or [type]=="media-info_MgBoss_EFSQL_error" or [type]=="media-info_MgBoss_EFSQL_info" or [type]=="media-info_MgBoss_Power_error" or [type]=="media-info_MgBoss_Power_info" or [type]=="media-info_MgBoss_Sys_error" or [type]=="media-info_MgBoss_Sys_info" or [type]=="media-info_MgBoss_Template_error" or [type]=="media-info_MgBoss_Template_info" or [type]=="media-info_LiveProgramList"
{
#file{
# path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/output.log"
#}
elasticsearch
{
protocol=>"http"
cluster => "hunantv_cdn"
host => "localhost"
bind_host => "localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/media-info.json"
template_name=>"media-info"
template_overwrite=>true
index=>"media-info_%{e_date}"
index_type=>"%{[index_type]}"
}
}
else if "router" in [tags]
{
elasticsearch
{
protocol=>"http"
cluster => "hunantv_cdn"
host => "localhost"
bind_host => "localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/router.json"
template_name=>"router"
template_overwrite=>true
index=>"router_%{date}"
index_type=>"%{status}"
}
}else if "routerdll" in [tags]
{
elasticsearch
{
protocol=>"http"
cluster => "hunantv_cdn"
host => "localhost"
#bind_host => "localhost"
#port=>"9200"
codec=>"json"
template=>"/mnt/develop/elk/app/logstash-1.4.2/template/routerdll.json"
template_name=>"routerdll"
template_overwrite=>true
index=>"routerdll_%{date}"
}
}
}
CDN 正则表达式匹配文件
# cdn pattern doc
# created by xiaohelong 2014-08-28
CDN_DATE %{YEAR}-%{MONTHNUM}-%{MONTHDAY}
CDN_TIMESTAMP %{CDN_DATE:cdn_date}[T ]%{HOUR}:?%{MINUTE}:?\d{1,2}
UUID32 [0-9a-z]{32}
HTTPSTATUS_VALUE \d+
HTTPSTATUS status=%{HTTPSTATUS_VALUE:status}
CDN_UUID32 uuid=%{UUID32:uuid}
# we only set one eventype because if we match successfully,then we will go next.
CDN_EVENTTYPE_HTTP http
CDN_EVENTTYPE_MOVIEINFO MovieInfo
CDN_EVENTTYPE_PCVCR PCVCR
CDN_EVENTTYPE_VCR VCR
CDN_EVENTTYPE_prvcr prvcr
CDN_PATTERN_HTTP_UUID %{CDN_TIMESTAMP:timestamp}.*%{CDN_EVENTTYPE_HTTP:etype}://(%{IPV4:qhost}|%{HOSTNAME:qhost}).+%{CDN_EVENTTYPE_HTTP}://(%{IPV4:rhost}|%{HOSTNAME:rhost}).+%{CDN_UUID32}.+-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_HTTP %{CDN_TIMESTAMP:timestamp}.*%{CDN_EVENTTYPE_HTTP:etype}://(%{IPV4:qhost}|%{HOSTNAME:qhost}).+%{CDN_EVENTTYPE_HTTP}://(%{IPV4:rhost}|%{HOSTNAME:rhost}).+-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_MOVIEINFO_HTTP %{CDN_TIMESTAMP:timestamp}.*%{CDN_EVENTTYPE_MOVIEINFO:etype}\s+.*%{HTTPSTATUS}.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_MOVIEINFO %{CDN_TIMESTAMP:timestamp}.*%{CDN_EVENTTYPE_MOVIEINFO:etype}--.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_PCVCR_HTTP %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_PCVCR:etype}\s+.*%{HTTPSTATUS}.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_PCVCR %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_PCVCR:etype}--.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_VCR_HTTP %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_VCR:etype}\s+.*%{HTTPSTATUS}.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_VCR %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_VCR:etype}--.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_prvcr_HTTP %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_prvcr:etype}\s+.*%{HTTPSTATUS}.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_prvcr %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_prvcr:etype}--.*-(%{IPV4:chost}|%{HOSTNAME:chost})
#above complete cdn pattern
#begin nginx access part
NGINX_ACCESS (%{IPV4:chost}|%{HOSTNAME:chost}).+\[%{MONTHDAY:a_day}/%{MONTH:a_month}/%{YEAR:a_year}\:%{TIME:time}.*\"\s+%{NUMBER:status}.*
#(%{IPV4:chost}|%{HOSTNAME:chost}).+\[%{MONTHDAY}/%{MONTH}/%{YEAR}\:%{TIME:time}.*\"\s+%{NUMBER:status}.*
#end nginx access part
#begin nginx error part
NGINX_ERR_1301 1301
NGINX_ERROR_1301 %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s%{TIME:time}.+error.*\:\s+%{NGINX_ERR_1301:status}.*[0-9a-z]{32},.*time:%{INT:rtime}.*;t is %{INT:qtime}.*client: (%{IPV4:chost}|%{HOSTNAME:chost}).*
NGINX_ERROR %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s%{TIME:time}.+error.*\:\s+%{NUMBER:status}.*[0-9a-z]{32},.*client: (%{IPV4:chost}|%{HOSTNAME:chost}).*
#1301 %{YEAR}/%{MONTHNUM}/%{MONTHDAY}\s%{TIME}.+error.*\:\s+%{NUMBER}.*[0-9a-z]{32},%{DATA},.*time:%{INT}.*;t is %{INT}.*client: (%{IPV4:chost}|%{HOSTNAME:chost}).*
#not 1301 %{YEAR}/%{MONTHNUM}/%{MONTHDAY}\s%{TIME}.+error.*\:\s+%{NUMBER}.*[0-9a-z]{32},.*client: (%{IPV4:chost}|%{HOSTNAME:chost}).*
#end nginx error part
#begin ChenPU CDN Delivery and Resource Monitor part , all WRN,ERR use the same pattern to match
STR_ERR ((?i)err(info|or|msg)?)
CDNDELIVERY_EVENTTYPE_WRN WRN
CDNDELIVERY_EVENTTYPE_ERR ERR
#when match the WRN or ERR,please put ERRINFO pattern before no ERR
CDNDELIVERY_PATTERN_WRN_ERRINFO %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*%{CDNDELIVERY_EVENTTYPE_WRN:etype}--%{GREEDYDATA:info}%{STR_ERR}\s*(=|＝|:|：)\s*%{GREEDYDATA:errInfo}
CDNDELIVERY_PATTERN_WRN_NOERR %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*%{CDNDELIVERY_EVENTTYPE_WRN:etype}--%{GREEDYDATA:info}
CDNDELIVERY_PATTERN_ERR_ERRINFO %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*%{CDNDELIVERY_EVENTTYPE_ERR:etype}--%{GREEDYDATA:info}%{STR_ERR}\s*(=|＝|:|：)\s*%{GREEDYDATA:errInfo}
CDNDELIVERY_PATTERN_ERR_NOERR %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*%{CDNDELIVERY_EVENTTYPE_ERR:etype}--%{GREEDYDATA:info}
#begin res_monitor part
RESMONITOR_EVENTTYPE_VRB VRB
RESMONITOR_PATTERN_VRB %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*\s%{RESMONITOR_EVENTTYPE_VRB:etype}.*EpochTime":%{QUOTEDSTRING:EpochTime}.*NodeId":%{NUMBER:NodeId}.*CurrentConn":%{NUMBER:CurrentConn}.*CurrentBandwidth":%{NUMBER:CurrentBandwidth}.*DiskTotalSpace":%{NUMBER:DiskTotalSpace}.*DiskFreeSpace":%{NUMBER:DiskFreeSpace}.*CpuUsage":%{NUMBER:CpuUsage}.*MemUsage":%{NUMBER:MemUsage}.*WanUsage":%{NUMBER:WanUsage}.*LanUsage":%{NUMBER:LanUsage}.*IoUsage":%{NUMBER:IoUsage}.*LanIpState":%{NUMBER:LanIpState}.*WanIpState":%{NUMBER:WanIpState}.*
#end res_monitor part
#begin cdn delivery part
#begin filecollect part
FILECOLLECT_EVENTTYPE_VRB VRB
FILECOLLECT_PATTERN_VRB %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*\s%{FILECOLLECT_EVENTTYPE_VRB:etype}.*EpochTime":%{QUOTEDSTRING:EpochTime}.*NodeId":%{NUMBER:NodeId}.*IsInited":%{NUMBER:IsInited}.*FileName":%{QUOTEDSTRING:FileName}.*FileSize":%{INT:FileSize}.*FileHash":%{QUOTEDSTRING:FileHash}.*FileIsDel":%{INT:FileIsDel}.*
#end filecollect part
#begin nodeschedule part
NODESCHEDULE_EVENTTYPE_VRB VRB
NODESCHEDULE_EVENTTYPE_INF INF
NODESCHEDULE_PATTERN_VRB .*?
NODESCHEDULE_PATTERN_INF %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*\s+%{NODESCHEDULE_EVENTTYPE_INF:etype}--%{DATA}\s*ContentType=%{WORD:ContentType}.*FileId=%{WORD:FileId}.*FileName=%{DATA:FileName}，TorrentHash=.*%{WORD:TorrentHash}.*State=%{WORD:State}.*Progress=%{NUMBER:Progress}.*DownloadSpeed=%{NUMBER:DownloadSpeed}.*UploadSpeed=%{NUMBER:UploadSpeed}.*DownloadSpeedAvg=%{NUMBER:DownloadSpeedAvg}.*UploadSpeedAvg=%{NUMBER:UploadSpeedAvg}.*DownloadedBytes=%{NUMBER:DownloadedBytes}.*UploadedBytes=%{NUMBER:UploadedBytes}.*%{STR_ERR}=%{GREEDYDATA}.*
#end nodeschedule part
#begin storageschedule part
STORAGESCHEDULE_EVENTTYPE_VRB VRB
STORAGESCHEDULE_PATTERN_VRB .*?
#end storageschedule part
#begin notify part
NOTIFYSCHEDULE_EVENTTYPE_VRB VRB
NOTIFYSCHEDULE_PATTERN_VRB .*?
#end notify part
#end cdn delivery part
#end ChenPU CDN Delivery and Resource Monitor part
#begin shaochan statout
STATOUT_PATTERN %{TIME:e_time}\s+.*:%{GREEDYDATA:info}
#end shaochan statout
#Begin DBAGENT access part
DBAGENT_ACCESS (%{IPV4:chost}|%{HOSTNAME:chost}).+\[%{MONTHDAY:a_day}/%{MONTH:a_month}/%{YEAR:a_year}\:%{TIME:time}.*(POST|GET)\s*%{DATA:interface}\s+HTTP.*\s+%{INT:status}\s+%{INT}\s+.*
#End DBAGENT access part
#Begin DBAGENT error part
DBAGENT_ERROR_3 %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s+%{TIME:time}\s+\[error\]\s+%{INT:pid}#0:\s+\*%{INT}\s+%{INT:level}\s+msg:%{DATA:msg},\s+check time:%{INT:ctime},\s+query string:\s*call\s+%{WORD:interface}.*\(%{DATA:para}\),\s+client:\s+(%{IPV4:chost}|%{HOSTNAME:chost}),.*
DBAGENT_ERROR_2 %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s+%{TIME:time}\s+\[error\]\s+%{INT:pid}#0:\s+\*%{INT}\s+%{INT:level}\s+msg:%{DATA:msg},\s+check time:%{INT:ctime},\s+query string:\s*call\s+%{WORD:interface}.*\(%{DATA:para}\).*
DBAGENT_ERROR_1 %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s+%{TIME:time}\s+\[error\]\s+%{INT:pid}#0:\s+%{INT:level}\s+msg:%{GREEDYDATA:msg}$.*
#End DBAGENT error part
#Begin router part
ROUTER_REQUEST_URL http://%{IP:requestServer}/.*--%{GREEDYDATA:fileLoc}
ROUTER_OK %{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}\s*%{TIME:time},%{INT}.*-.*%{IP:chost}\s-\s%{WORD:status}\s-\s%{ROUTER_REQUEST_URL:requestURL}.*\s-\s%{NUMBER:isThirdCDN}\s-\s%{NUMBER:requestPolicyID}\s-\s%{NUMBER:rateLimited}
ROUTER_ERROR %{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}\s%{TIME:time},.*-.*%{IP:chost}\s-\s%{WORD:status}\s-\s%{GREEDYDATA:description}.*\s-\s%{NUMBER:isThirdCDN}\s-\s%{NUMBER:requestPolicyID}\s-\s%{NUMBER:rateLimited}
#End router part
#Begin routerdll part
ROUTERDLL_ERR %{TIME:time}\.%{INT}\s%{DATA:etype}--%{GREEDYDATA:info}\s*fid=%{GREEDYDATA:fid},\s*uip=%{IP:uip},\s*pno=%{INT:pno},\s*msec=%{NUMBER:msec},\s*statusDesc=%{GREEDYDATA:statusDesc},\s*errInfo=%{GREEDYDATA:errInfo},\s*userRegionId=%{INT:userRegionId},\s*svcRegionIds=%{GREEDYDATA:svcRegionIds},\shasFileNodeIds=%{GREEDYDATA:hasFileNodeIds},\s*routeType=%{GREEDYDATA:routeType},\s*isTestSpeed=%{INT:isTestSpeed},\s*lastSvrIps=%{GREEDYDATA:lastSvrIps}
#End routerdll part
#begin media-info pattern doc
# created by xiaohelong 2014-10-14
DATE_CN (%{YEAR}-%{MONTHNUM}-%{MONTHDAY})
#begin info part
INFOERROR_PATTERN .*【记录时间】\s*%{DATE_CN:e_date}\s*%{TIME:e_time},%{INT}.*\s*【线程编号】\[%{INT:tid}\]\s*【信息描述】\s*%{GREEDYDATA:info}
#end info part
#begin LogOfOnTimeRun
LOGOFONTIMERUN .*Date="%{DATE_CN:e_date}\s*%{TIME:e_time}.*Type="%{DATA:status}".*Note="%{DATA:info}".*
#end LogOfOnTimeRun
#end media-info
以下均为json模版文件
cdn.json 模版文件
{
"template": "cdn_*",
"settings": {
"index.number_of_shards": 5,
"number_of_replicas": 1
},
"mappings": {
"_default_": {
"_all": {
"enabled": "false"
},
"_source": {
"enabled": true,
"compress":true
},
"properties": {
"_id": {
"type": "string",
"index": "not_analyzed"
},
"host":{
"type":"string",
"index":"not_analyzed"
},
"message": {
"type": "string",
"index": "not_analyzed"
},
"chost": {
"dynamic": true,
"type": "ip",
"index": "not_analyzed"
},
"cgeo": {
"dynamic": true,
"type": "object",
"properties": {
"country_name": {
"type": "string",
"index": "not_analyzed"
},
"real_region_name": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
},
"http": {
"properties": {
"_timestamp": {
"enabled": true,
"type": "date",
"path": "timestamp",
"format": "dateOptionalTime"
},
"rhost": {
"type": "string",
"index": "not_analyzed"
},
"qhost": {
"type": "string",
"index": "not_analyzed"
},
"uuid": {
"dynamic": true,
"type": "string",
"index": "not_analyzed"
}
}
},
"MovieInfo": {
"properties": {
"_timestamp": {
"enabled": true,
"type": "date",
"path": "timestamp",
"format": "dateOptionalTime"
},
"status": {
"dynamic": true,
"type": "integer",
"index": "not_analyzed"
}
}
},
"PCVCR": {
"properties": {
"_timestamp": {
"enabled": true,
"type": "date",
"path": "timestamp",
"format": "dateOptionalTime"
},
"status": {
"dynamic": true,
"type": "integer",
"index": "not_analyzed"
}
}
},
"VCR": {
"properties": {
"_timestamp": {
"enabled": true,
"type": "date",
"path": "timestamp",
"format": "dateOptionalTime"
},
"rhost": {
"type": "string",
"index": "not_analyzed"
},
"qhost": {
"type": "string",
"index": "not_analyzed"
},
"uuid": {
"dynamic": true,
"type": "string",
"index": "not_analyzed"
}
}
},
"prvcr": {
"properties": {
"_timestamp": {
"enabled": true,
"type": "date",
"path": "timestamp",
"format": "dateOptionalTime"
},
"status": {
"dynamic": true,
"type": "integer",
"index": "not_analyzed"
}
}
}
}
{
"template": "cdn-delivery_*",
"settings": {
"index.number_of_shards": 5,
"number_of_replicas": 1
},
"mappings": {
"_default_": {
"_all": {
"enabled": "false"
},
"_source": {
"enabled": true,
"compress": true
},
"properties": {
"_id": {
"type": "string",
"index": "not_analyzed"
},
"timestamp": {
"enabled": true,
"type": "date",
"format": "dateOptionalTime"
},
"host": {
"type": "string",
"index": "not_analyzed"
}
}
},
"VRB-filecollect":{
"properties":{
"EpochTime":{
"type":"string",
"format":"not_analyzed"
},
"NodeId":{
"type":"integer"
},
"IsInited":{
"type":"boolean"
},
"FileName":{
"type":"string",
"index":"not_analyzed"
},
"FileSize":{
"type":"long"
},
"FileHash":{
"type":"string",
"index":"not_analyzed"
},
"FileIsDel":{
"type":"boolean"
}
}
},
"ERR-filecollect":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
},
"WRN-filecollect":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
},
"ERR-nodeschedule":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
},
"WRN-nodeschedule":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
},
"INF-nodeschedule":{
"properties":{
"ContentType":{
"type":"string",
"index":"not_analyzed"
},
"FileId":{
"type":"string",
"index":"not_analyzed"
},
"FileName":{
"type":"string",
"index":"not_analyzed"
},
"TorrentHash":{
"type":"string",
"index":"not_analyzed"
},
"State":{
"type":"string",
"index":"not_analyzed"
},
"Progress":{
"type":"float"
},
"DownloadSpeed":{
"type":"float"
},
"UploadSpeed":{
"type":"float"
},
"DownloadSpeedAvg":{
"type":"float"
},
"UploadSpeedAvg":{
"type":"float"
},
"DownloadedBytes":{
"type":"float"
},
"UploadedBytes":{
"type":"float"
}
}
},
"ERR-storageschedule":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
},
"WRN-storageschedule":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
},
"ERR-notify":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
},
"WRN-notify":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
}
}
}
{
"template": "dbagent_*",
"settings": {
"index.number_of_shards": 5,
"number_of_replicas": 0
},
"mappings": {
"_default_": {
"_all": {
"enabled": "false"
},
"_source": {
"enabled": true,
"compress": true
},
"properties": {
"_id": {
"type": "string",
"index": "not_analyzed"
},
"timestamp": {
"enabled": true,
"type": "date",
"format": "dateOptionalTime"
},
"host": {
"type": "string",
"index": "not_analyzed"
}
}
},
"access":{
"properties":{
"status":{
"type":"integer"
},
"chost": {
"type": "ip",
"index": "not_analyzed"
},
"interface":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
},
"error":{
"properties":{
"pid":{
"type":"integer"
},
"level":{
"type":"integer"
},
"msg":{
"type":"string",
"index":"not_analyzed"
},
"ctime":{
"dynamic":true,
"type":"long"
},
"interface":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
},
"para":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
},
"chost": {
"dynamic":true,
"type": "ip"
},
"message":{
"type":"string",
"index":"not_analyzed"
}
}
}
}
}
{
"template": "media-info_*",
"settings": {
"index.number_of_shards": 5,
"number_of_replicas": 1
},
"mappings": {
"_default_": {
"_all": {
"enabled": "false"
},
"_source": {
"enabled": true,
"compress": true
},
"properties": {
"_id": {
"type": "string",
"index": "not_analyzed"
},
"timestamp": {
"enabled": true,
"type": "date",
"format": "dateOptionalTime"
},
"host": {
"type": "string",
"index": "not_analyzed"
},
"appname":{
"type":"string",
"index":"not_analyzed"
},
"category":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
},
"info":{
"properties":{
"tid":{
"type":"integer"
},
"info":{
"type":"string",
"index":"analyzed"
}
}
},
"error":{
"properties":{
"tid":{
"type":"integer"
},
"info":{
"type":"string",
"index":"analyzed"
}
}
},
"LiveProgramList":{
"properties":{
"status":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"analyzed"
}
}
}
}
}
{
"template": "nginx_*",
"settings": {
"index.number_of_shards": 5,
"number_of_replicas": 1
},
"mappings": {
"_default_": {
"_all": {
"enabled": "false"
},
"_source": {
"enabled": true,
"compress": true
},
"properties": {
"_id": {
"type": "string",
"index": "not_analyzed"
},
"timestamp": {
"enabled": true,
"type": "date",
"format": "dateOptionalTime"
},
"host": {
"type": "string",
"index": "not_analyzed"
},
"message": {
"type": "string",
"index": "no"
},
"chost": {
"type": "ip",
"index": "not_analyzed"
},
"cgeo": {
"type": "object",
"properties": {
"country_name": {
"type": "string",
"index": "not_analyzed"
},
"real_region_name": {
"type": "string",
"index": "not_analyzed"
}
}
}
,
"status":{
"type":"integer"
}
}
},
"error":{
"properties":{
"interval":{
"dynamic":true,
"type":"integer"
}
}
}
}
}
{
"template": "nginx_access_*",
"settings" : {
"number_of_shards" : 5,
"number_of_replicas" : 1,
"index" : {
"query" : { "default_field" : "message" }
}
},
"mappings": {
"nginx_access": {
"_all": { "enabled": false },
"_source": {
"enabled": true,
"compress": true
},
"dynamic_templates": [
{
"string_template" : {
"match" : "*",
"mapping": { "type": "string", "index": "not_analyzed" },
"match_mapping_type" : "string"
}
}
],
"properties" : {
"@timestamp": { "type": "date", "format": "dateOptionalTime" },
"fid" : { "type" : "string", "index" : "not_analyzed" },
"filename" : { "type" : "string", "index" : "analyzed" },
"host" : { "type" : "string", "index" : "analyzed" },
"message" : { "type" : "string", "index" : "analyzed" },
"referer" : { "type" : "string", "index" : "analyzed" },
"request": { "type": "string", "index" : "analyzed" },
"sentbytes" : { "type" : "long" },
"status" : { "type" : "short" },
"type" : { "type" : "string", "index" : "not_analyzed" },
"user_agent" : { "type" : "string", "index" : "analyzed" },
"userip" : { "type" : "string", "index" : "not_analyzed" },
"uuid" : { "type" : "string", "index" : "not_analyzed" },
"verb" : { "type" : "string", "index" : "not_analyzed" },
"limitrate" : { "type" : "integer" },
"nid" : { "type" : "short" },
"payload" : { "type" : "string", "index" : "analyzed" },
"sign" : { "type" : "string", "index" : "not_analyzed"},
"urgid" : { "type" : "short" },
"srgid" : { "type" : "short" },
"srgids" : { "type" : "string", "index" : "analyzed" },
"t" : { "type" : "string", "index" : "not_analyzed" },
"geoip" : {
"dynamic" : true,
"path" : "full",
"properties" : {
"location" : {
"type" : "geo_point"
}
},
"type" : "object"
}
}
}
}
}
{
"template": "res-monitor_*",
"settings": {
"index.number_of_shards": 5,
"number_of_replicas": 1
},
"mappings": {
"_default_": {
"_all": {
"enabled": "false"
},
"_source": {
"enabled": true,
"compress": true
},
"properties": {
"_id": {
"type": "string",
"index": "not_analyzed"
},
"timestamp": {
"enabled": true,
"type": "date",
"format": "dateOptionalTime"
},
"host": {
"type": "string",
"index": "not_analyzed"
}
}
},
"VRB":{
"properties":{
"EpochTime":{
"type":"string",
"format":"not_analyzed"
},
"NodeId":{
"type":"integer"
},
"CurrentConn":{
"type":"integer"
},
"CurrentBandwidth":{
"type":"integer"
},
"DiskTotalSpace":{
"type":"long"
},
"DiskFreeSpace":{
"type":"long"
},
"CpuUsage":{
"type":"integer"
},
"MemUsage":{
"type":"integer"
},
"WanUsage":{
"type":"integer"
},
"LanUsage":{
"type":"integer"
},
"IoUsage":{
"type":"integer"
},
"LanIpState":{
"type":"integer"
},
"WanIpState":{
"type":"integer"
}
}
},
"ERR":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"type":"string",
"index":"not_analyzed"
}
}
},
"WRN":{
"properties":{
"message":{
"type":"string",
"index":"not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"dynamic":true,
"type":"string",
"index":"not_analyzed"
}
}
}
}
}
{
"template": "router_*",
"settings": {
"index.number_of_shards": 5,
"number_of_replicas": 1
},
"mappings": {
"_default_": {
"_all": {
"enabled": "false"
},
"_source": {
"enabled": true,
"compress": true
},
"properties": {
"_id": {
"type": "string",
"index": "not_analyzed"
},
"timestamp": {
"enabled": true,
"type": "date",
"format": "dateOptionalTime"
},
"host": {
"type": "string",
"index": "not_analyzed"
},
"chost":{
"type":"ip",
"index":"not_analyzed"
},
"isThirdCDN":{
"type":"boolean"
},
"requestPolicyID":{
"type":"integer"
},
"rateLimited":{
"type":"long"
}
}
},
"ok":{
"properties":{
"requestURL":{
"type":"string",
"index":"not_analyzed"
},
"requestServer":{
"type":"ip",
"index":"not_analyzed"
},
"fileLoc":{
"type":"string",
"index":"not_analyzed"
}
}
},
"error":{
"properties":{
"description":{
"type":"string",
"index":"not_analyzed"
}
}
}
}
}
{
"template": "routerdll_*",
"settings": {
"index.number_of_shards": 5,
"number_of_replicas": 1
},
"mappings": {
"_default_": {
"_all": {
"enabled": "false"
},
"_source": {
"enabled": true,
"compress": true
},
"properties": {
"_id": {
"type": "string",
"index": "not_analyzed"
},
"timestamp": {
"enabled": true,
"type": "date",
"format": "dateOptionalTime"
},
"host": {
"type": "string",
"index": "not_analyzed"
}
}
},
"ERR":{
"properties":{
"info":{
"type":"string",
"index":"not_analyzed"
},
"fid":{
"type":"string",
"index":"not_analyzed"
},
"uip":{
"type":"ip",
"index":"not_analyzed"
},
"pno":{
"type":"integer"
},
"msec":{
"type":"float"
},
"statusDesc":{
"type":"string",
"index":"not_analyzed"
},
"errInfo":{
"type":"string",
"index":"not_analyzed"
},
"userRegionId":{
"type":"integer"
},
"svcRegionIds":{
"type":"string",
"index":"not_analyzed"
},
"hasFileNodeIds":{
"type":"string",
"index":"not_analyzed"
},
"routeType":{
"type":"string",
"index":"not_analyzed"
},
"isTestSpeed":{
"type":"boolean"
},
"lastSvrIps":{
"type":"string",
"index":"not_analyzed"
}
}
}
}
}
{
"template": "statout_*",
"settings": {
"index.number_of_shards": 5,
"number_of_replicas": 1
},
"mappings": {
"_default_": {
"_all": {
"enabled": "false"
},
"_source": {
"enabled": true,
"compress": true
},
"properties": {
"_id": {
"type": "string",
"index": "not_analyzed"
},
"timestamp": {
"enabled": true,
"type": "date",
"format": "dateOptionalTime"
},
"host": {
"type": "string",
"index": "not_analyzed"
},
"info":{
"type":"string",
"index":"not_analyzed"
}
}
}
}
}
1078
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
