3. Logstash 网络设备日志；输入-格式化-输出

=================================================================

1. 日志输入接口

 

input{ # 输入接口

 

udp{ #启用UDP协议

type => "syslog" # 类型为syslog

host => "100.76.37.69" # 本机对外服务器IP地址

port => 514 # 启用UDP 514 端口，一般接收 网络设备日志。

}

 

}

============================================================================

2. 日志策略

filter{ # 策略

 

==============================================================================

2.1 针对于华为防火墙日志格式化

if "HW" and "FW" in [message]{ # 判断 HW 和 FW 字段同时在本条日志中出现

 

grok{ #日志格式化

match => {

"message" => "\S+[0-9]{1,4}-\d\d-\d\d %{TIME} %{HOSTNAME:Log_Device_Name} %%01(?<Log_Total>[A-Z]{1,9})/(?<Log_Level>\d)/(?<Log_Type>\S+):"

}

}

 

grok{

match => {

"message" =>"\S+[0-9]{1,4}-\d\d-\d\d %{TIME} (?<Log_Company>[A-Z]{1,4})-(?<Log_Area>[A-Z]{1,3})-(?<Log_Position>\S{1,6})-(?<Log_Function>[A-Z]{1,6})-(?<

Log_Model>\S{1,3}[0-9]{1,6}-[0-9]{1,8}\S+)"

}

 

}

===================================================================

2.1.1 字段重命名 及怎删改

 

if "HW" and "FW" in [Log_Device_Name]{

mutate{

 

rename => {"host" => "Log_Ip"} # 修改原日志 host 字段为 Log_Ip字段

add_field => {"Log_Device_Type" => "FW"} #增加 FW 字段

add_field => {"Log_City" => "BJ"}

remove_field => ["@version"]

remove_field => ["_id"]

 

 

}

}

 

}

=========================================================================================

2.2 华为交换机日志格式

if "HW" and "SW" in [message]{

 

grok{

match => {

"message" => "%{MONTH} +%{MONTHDAY} %{YEAR} %{TIME} %{HOSTNAME:Log_Device_Name} %%01(?<Log_Total>[A-Z]{1,9})/(?<Log_Level>\d)/(?<Log_Type>\S+):"

 

}

}

 

grok{

match => {

"message" =>"%{MONTH} +%{MONTHDAY} %{YEAR} %{TIME} (?<Log_Company>[A-Z]{1,4})-(?<Log_Area>[A-Z]{1,3})-(?<Log_Position>\S{1,6})-(?<Log_Function>[A-Z]{1,6

})-(?<Log_Model>\S{1,3}[0-9]{1,6}-[0-9]{1,8}\S+)"

}

}

2.2.1 字段重命名 及怎删改

if "SW" and "HW" in [Log_Device_Name]{

mutate{

 

rename => {"host" => "Log_Ip"}

add_field => {"Log_Device_Type" => "SW"}

add_field => {"Log_City" => "BJ"}

remove_field => ["@version"]

remove_field => ["_id"]

 

}

}

}

 

=================================================================================

2.3 防火墙VPN 日志格式

 

if "VPN" in [message]{

 

grok{

match => {

"message" => "\S+[0-9]{1,4}-\d\d-\d\d %{TIME} %{HOSTNAME:Log_Device_Name} %%01(?<Log_Total>[A-Z]{1,9})/(?<Log_Level>\d)/(?<Log_Type>\S+):"

 

}

}

 

grok{

match => {

"message" =>"\S+[0-9]{1,4}-\d\d-\d\d %{TIME} (?<Log_Company>[A-Z]{1,4})-(?<Log_Area>[A-Z]{1,3})-(?<Log_Position>\S{1,6})-(?<Log_Function>[A-Z]{1,6})-(?<

Log_Model>\S{1,3}[0-9]{1,6}-[0-9]{1,8}\S+)"

}

}

========================================================================================

2.3.1 字段重命名 及怎删改

if "VPN" and "HW" in [Log_Device_Name]{

mutate{

 

rename => {"host" => "Log_Ip"}

add_field => {"Log_Device_Type" => "FW"}

add_field => {"Log_City" => "BJ"}

remove_field => ["@version"]

remove_field => ["_id"]

 

}

}

}

 

============================================================================================================================

2.4 H3C 日志格式化

 

if "H3C" in [message]{

grok{

match => {

"message" => "%{MONTH} +%{MONTHDAY} %{TIME} %{YEAR} %{HOSTNAME:Log_Device_Name} %%10(?<Log_Total>[A-Z]{1,7})\S(?<Log_Level>\d)/(?<Log_Type>\S+)"

 

}

}

grok{

match => {

"message" => "%{MONTH} +%{MONTHDAY} %{TIME} %{YEAR} (?<Log_Company>[A-Z]{1,4})-(?<Log_City>[A-Z]{1,4})-(?<Log_Area>[A-Z]{1,3})-(?<Log_Position>\S{1,6}

)-(?<Log_Function>[A-Z]{1,9})-(?<Log_Model>\S+{1,3}[0-9]{1,6}-[0-9]{1,8}\S+)"

}

}

==========================================================

2.4.1 字段重命名 及怎删改

mutate{

 

rename => {"host" => "Log_Ip"}

add_field => {"Log_Device_Type" => "SW"}

remove_field => ["@version"]

remove_field => ["_id"]

}

}

 

======================================================================================================

2.5 EMC 存储日志格式化

if "XIO_XY_XMS" in [message]{

 

grok{

match => {

 

"message" => "\S+%{MONTH} +%{MONTHDAY} %{TIME} (?<Log_Device_Name>[A-Z]{1,6}_[A-Z]{1,6}_[A-Z]{1,6}) ([a-z]{1,6}:) \d+: \S+"

}

 

}

===================================================

2.5.1 字段重命名 及怎删改

 

mutate{

 

rename => {"host" => "Log_Ip"}

add_field => {"Log_Device_Type" => "Storage"}

add_field => {"Log_City" => "BJ"}

add_field => {"Log_Area" => "XY"}

add_field => {"Log_Position" => "1F-A11"}

add_field => {"Log_Company" => "JP"}

remove_field => ["@version"]

remove_field => ["_id"]

}

 

 

}

 

if "XIO_YZ_XMS" in [message]{

 

grok{

match => {

 

"message" => "\S+%{MONTH} +%{MONTHDAY} %{TIME} (?<Log_Device_Name>[A-Z]{1,6}_[A-Z]{1,6}_[A-Z]{1,6}) ([a-z]{1,6}:) \d+: \S+"

}

 

}

 

mutate{

 

rename => {"host" => "Log_Ip"}

add_field => {"Log_Device_Type" => "Storage_DB_Orcal"}

add_field => {"Log_City" => "BJ"}

add_field => {"Log_Area" => "YZ"}

add_field => {"Log_Position" => "B1-I01"}

add_field => {"Log_Company" => "JP"}

remove_field => ["@version"]

remove_field => ["_id"]

}

 

 

}

======================================================================================================

2.6 DEll NAS日志格式化

if [host] == "100.119.8.50"{

mutate{

 

rename => {"host" => "Log_Ip"}

add_field => {"Log_Device_Type" => "Storage_NAS"}

add_field => {"Log_City" => "BJ"}

add_field => {"Log_Company" => "JDB"}

remove_field => ["@version"]

remove_field => ["_id"]

}

}

#############################################################################################################

}

 

3. 日志输出至Elasticsearch集群

output{

==========================================

3.1 solarwinds 日志输出

if [host] == "1.65.15.1"{

elasticsearch{

hosts => "1.65.17.102:9200"

manage_template => true

index => "Solarwinds-%{+YYYY-MM}"

 

}

}

===========================================================

3.2 数据中心网络设备日志输出（网络组 集群）

if [host] != "1.65.15.1"{

elasticsearch{

hosts => ["100.76.37.64:9200","100.76.37.65:9200","100.76.37.66:9200"]

manage_template => true

index => "logstash_network-%{+YYYY-MM-dd}"

}

}

 

=================================================================

3.3 DB部门 Elasticsearch 集群

elasticsearch{

hosts => ["100.115.3.1:9201","100.115.3.2:9201","100.115.3.3:9201"]

manage_template => true

index => "logstash_network-%{+YYYY-MM-dd}"

}

}

 

[root@Logstash_37_69 config]#