logstash 6.x 收集syslog日志

1、logstash端

关闭logstash所在机器的rsyslog，释放514端口号

[root@node1 config]# systemctl stop rsyslog
[root@node1 config]# systemctl status rsyslog
● rsyslog.service - System Logging Service
   Loaded: loaded (/usr/lib/systemd/system/rsyslog.service; enabled; vendor preset: enabled)
   Active: inactive (dead) since Thu 2018-04-26 14:32:34 CST; 1min 58s ago
  Process: 3915 ExecStart=/usr/sbin/rsyslogd -n $SYSLOGD_OPTIONS (code=exited, status=0/SUCCESS)
 Main PID: 3915 (code=exited, status=0/SUCCESS)

Apr 26 14:25:16 node1 systemd[1]: Starting System Logging Service...
Apr 26 14:25:16 node1 systemd[1]: Started System Logging Service.
Apr 26 14:32:34 node1 systemd[1]: Stopping System Logging Service...
Apr 26 14:32:34 node1 systemd[1]: Stopped System Logging Service.
[root@node1 config]#

编写logstash配置文件

[root@node1 logstash-6.2.3]# vi config/local_syslog.conf
[root@node1 logstash-6.2.3]# cat config/local_syslog.conf
input {
 syslog {
    type => "rsyslog"
    port => "514"
  }
}
output{
  stdout{
    codec => rubydebug
  }
}

启动logstash

[root@node1 logstash-6.2.3]# bin/logstash -f config/local_syslog.conf
Sending Logstash's logs to /var/log/logstash which is now configured via log4j2.properties
[2018-04-26T14:39:57,627][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/opt/logstash-6.2.3/modules/netflow/configuration"}
[2018-04-26T14:39:57,650][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/opt/logstash-6.2.3/modules/fb_apache/configuration"}
[2018-04-26T14:39:58,301][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2018-04-26T14:39:59,346][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"6.2.3"}
[2018-04-26T14:40:00,022][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}
[2018-04-26T14:40:04,438][INFO ][logstash.pipeline        ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[2018-04-26T14:40:04,901][INFO ][logstash.pipeline        ] Pipeline started succesfully {:pipeline_id=>"main", :thread=>"#<Thread:0x518728c7 run>"}
[2018-04-26T14:40:04,989][INFO ][logstash.inputs.syslog   ] Starting syslog udp listener {:address=>"0.0.0.0:514"}
[2018-04-26T14:40:05,013][INFO ][logstash.inputs.syslog   ] Starting syslog tcp listener {:address=>"0.0.0.0:514"}
[2018-04-26T14:40:05,034][INFO ][logstash.agent           ] Pipelines running {:count=>1, :pipelines=>["main"]}

查看端口号

[root@node1 config]# netstat -anp|grep 514
tcp6       0      0 :::514                  :::*                    LISTEN      4260/java           
udp        0      0 0.0.0.0:514             0.0.0.0:*                           4260/java           
unix  2      [ ACC ]     STREAM     LISTENING     15141    822/mcelog           /var/run/mcelog-client
unix  2      [ ]         DGRAM                    15147    828/chronyd          
[root@node1 config]#

发现514端口已经被logstash占有

2、syslog端

切换到另一台服务器node2上，配置syslog

[root@node2 ~]# vi /etc/rsyslog.conf

添加一行*.* @@node1:514，把日志输送到远端的logstash上。

# remote host is: name/ip:port, e.g. 192.168.0.1:514, port optional
#*.* @@remote-host:514
*.* @@node1:514

重新rsyslogd程序

[root@node2 ~]# systemctl restart rsyslog

3、logstash端收集数据

这时发现logstash端已经收集到node2的syslog日志数据

[2018-04-26T14:45:18,361][INFO ][logstash.inputs.syslog   ] new connection {:client=>"10.17.12.157:55204"}
{
    "severity_label" => "Informational",
    "facility_label" => "system",
         "timestamp" => "Apr 26 14:39:23",
          "severity" => 6,
              "host" => "10.17.12.157",
           "message" => "Stopping System Logging Service...\n",
          "@version" => "1",
           "program" => "systemd",
        "@timestamp" => 2018-04-26T06:39:23.000Z,
              "type" => "rsyslog",
          "priority" => 30,
         "logsource" => "node2",
          "facility" => 3
}
{
    "severity_label" => "Informational",
    "facility_label" => "system",
         "timestamp" => "Apr 26 14:39:23",
          "severity" => 6,
              "host" => "10.17.12.157",
           "message" => "Stopped System Logging Service.\n",
          "@version" => "1",
           "program" => "systemd",
        "@timestamp" => 2018-04-26T06:39:23.000Z,
              "type" => "rsyslog",
          "priority" => 30,
         "logsource" => "node2",
          "facility" => 3
}
{
    "severity_label" => "Notice",
    "facility_label" => "security/authorization",
         "timestamp" => "Apr 26 14:39:23",
          "severity" => 5,
              "host" => "10.17.12.157",
           "message" => "Unregistered Authentication Agent for unix-process:4601:59761164 (system bus name :1.2556, object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.utf8) (disconnected from bus)\n",
          "@version" => "1",
           "program" => "polkitd",
        "@timestamp" => 2018-04-26T06:39:23.000Z,
              "type" => "rsyslog",
          "priority" => 85,
               "pid" => "762",
         "logsource" => "node2",
          "facility" => 10
}
{
    "severity_label" => "Informational",
    "facility_label" => "system",
         "timestamp" => "Apr 26 14:40:01",
          "severity" => 6,
              "host" => "10.17.12.157",
           "message" => "Started Session 1235 of user root.\n",
          "@version" => "1",
           "program" => "systemd",
        "@timestamp" => 2018-04-26T06:40:01.000Z,
              "type" => "rsyslog",
          "priority" => 30,
         "logsource" => "node2",
          "facility" => 3
}
{
    "severity_label" => "Informational",
    "facility_label" => "system",
         "timestamp" => "Apr 26 14:40:01",
          "severity" => 6,
              "host" => "10.17.12.157",
           "message" => "Starting Session 1235 of user root.\n",
          "@version" => "1",
           "program" => "systemd",
        "@timestamp" => 2018-04-26T06:40:01.000Z,
              "type" => "rsyslog",
          "priority" => 30,
         "logsource" => "node2",
          "facility" => 3
}
{
    "severity_label" => "Informational",
    "facility_label" => "clock",
         "timestamp" => "Apr 26 14:40:01",
          "severity" => 6,
              "host" => "10.17.12.157",
           "message" => "(root) CMD (/usr/lib64/sa/sa1 1 1)\n",
          "@version" => "1",
           "program" => "CROND",
        "@timestamp" => 2018-04-26T06:40:01.000Z,
              "type" => "rsyslog",
          "priority" => 78,
               "pid" => "4640",
         "logsource" => "node2",
          "facility" => 9
}
{
    "severity_label" => "Notice",
    "facility_label" => "security/authorization",
         "timestamp" => "Apr 26 14:45:18",
          "severity" => 5,
              "host" => "10.17.12.157",
           "message" => "Registered Authentication Agent for unix-process:4786:59796608 (system bus name :1.2559 [/usr/bin/pkttyagent --notify-fd 5 --fallback], object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.utf8)\n",
          "@version" => "1",
           "program" => "polkitd",
        "@timestamp" => 2018-04-26T06:45:18.000Z,
              "type" => "rsyslog",
          "priority" => 85,
               "pid" => "762",
         "logsource" => "node2",
          "facility" => 10
}
{
    "severity_label" => "Informational",
    "facility_label" => "system",
         "timestamp" => "Apr 26 14:45:18",
          "severity" => 6,
              "host" => "10.17.12.157",
           "message" => "Starting System Logging Service...\n",
          "@version" => "1",
           "program" => "systemd",
        "@timestamp" => 2018-04-26T06:45:18.000Z,
              "type" => "rsyslog",
          "priority" => 30,
         "logsource" => "node2",
          "facility" => 3
}
{
    "severity_label" => "Informational",
    "facility_label" => "system",
         "timestamp" => "Apr 26 14:45:18",
          "severity" => 6,
              "host" => "10.17.12.157",
           "message" => "Started System Logging Service.\n",
          "@version" => "1",
           "program" => "systemd",
        "@timestamp" => 2018-04-26T06:45:18.000Z,
              "type" => "rsyslog",
          "priority" => 30,
         "logsource" => "node2",
          "facility" => 3
}
{
    "severity_label" => "Notice",
    "facility_label" => "security/authorization",
         "timestamp" => "Apr 26 14:45:18",
          "severity" => 5,
              "host" => "10.17.12.157",
           "message" => "Unregistered Authentication Agent for unix-process:4786:59796608 (system bus name :1.2559, object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.utf8) (disconnected from bus)\n",
          "@version" => "1",
           "program" => "polkitd",
        "@timestamp" => 2018-04-26T06:45:18.000Z,
              "type" => "rsyslog",
          "priority" => 85,
               "pid" => "762",
         "logsource" => "node2",
          "facility" => 10
}
{
    "severity_label" => "Informational",
    "facility_label" => "syslogd",
         "timestamp" => "Apr 26 14:45:18",
          "severity" => 6,
              "host" => "10.17.12.157",
           "message" => "[origin software=\"rsyslogd\" swVersion=\"7.4.7\" x-pid=\"4792\" x-info=\"http://www.rsyslog.com\"] start\n",
          "@version" => "1",
           "program" => "rsyslogd",
        "@timestamp" => 2018-04-26T06:45:18.000Z,
              "type" => "rsyslog",
          "priority" => 46,
         "logsource" => "node2",
          "facility" => 5
}