centos7部署logstash推送到elasticsearch

1、准备基础插件

yum -y install net-tools  install wget java lrzsz unzip zip vim

2、下载安装包

cd /data/es
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.6.2.tar.gz

3、解压

tar -zxvf logstash-7.6.2.tar.gz

4、修改配置文件信息

说明：调用elasticsearch如果没有账号密码可以注释删除，根据自己实际情况更改

/data/es/logstash-7.6.2/config/logstash.yml

增加：

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://ip地址:9200" ]
xpack.monitoring.elasticsearch.username: "elastic"
xpack.monitoring.elasticsearch.password: "自己设置的密码"

新增配置文件

/data/es/logstash-7.6.2/bin/smart.conf

input {
       file{
	path => "/data/es/logs/*.log"
	codec => json
	start_position => "beginning"
	type => "smart"
           }
}

filter {
    grok {
        pattern_definitions => {
            "QUALIFIED" => "[a-zA-Z0-9$_.]+"
        }

        match => {
            "message" => "%{TIMESTAMP_ISO8601:logdate}%{SPACE}\[%{USERNAME:logthread}\]%{SPACE}%{WORD:loglevel}%{SPACE}%{QUALIFIED:logclass:text}%{SPACE}-%{SPACE}%{GREEDYDATA:logmsg:text}"

        }

    }
}


output {
    elasticsearch {
        hosts =>["http://ip地址设置自己的:9200"]
        index => "smart"
        user => "elastic"
        password => "自己设置的密码"
    }

}

5、启动logstash服务

cd /data/es/logstash-7.6.2/bin
./logstash -f smart.conf

6、去kibana服务看下配置相关索引

查看：

 其他说明：

1、如果想了解ELK部署可参看http://t.csdn.cn/FVCp7

2、很多时候我们的错误日志是分多行的，就比如java错误日志，但是默认logstash是按照每行进行一个事件，这样错误就不连贯，未能达到我们的需求，所以可以做下更改

2023-06-07 18:52:32.485 ERROR 15200 --- [http-nio-9090-exec-5] o.s.w.bind.annotation.ExceptionHandler   : 运行时异常：

org.springframework.jdbc.BadSqlGrammarException: 
### Error querying database.  Cause: java.sql.SQLSyntaxErrorException: Unknown column org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542)
	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:143)
	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
	at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:374)
	at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
	at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:893)
	at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1707)
	at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
	at java.lang.Thread.run(Thread.java:745)

修改配置文件，在input增加 

       codec => multiline {
                     pattern => "^\d+"
                     negate => "true"
                     what => "previous"
                     charset => "UTF-8"
                     auto_flush_interval => 3
                   }

input {
       file{
	path => "/data/es/logs/*.log"
	#codec => json
	start_position => "beginning"
	type => "smart"
                 codec => multiline {
                     pattern => "^\d+"
                    negate => "true"
                      what => "previous"
                        charset => "UTF-8"
                       auto_flush_interval => 3
                       }
           }
}

filter {
    grok {
        pattern_definitions => {
            "QUALIFIED" => "[a-zA-Z0-9$_.]+"
        }
   
        match => {
            "message" => "%{TIMESTAMP_ISO8601:logdate}%{SPACE}\[%{USERNAME:logthread}\]%{SPACE}%{WORD:loglevel}%{SPACE}%{QUALIFIED:logclass:text}%{SPACE}-%{SPACE}%{GREEDYDATA:logmsg:text}"

        }

    }
}


output {
    elasticsearch {
        hosts =>["http://ip:9200"]
        index => "smart"
        user => "elastic"
        password => "你设置的密码"
    }

}