Logstash同步日志文件到Elasticsearch

Logstash

logstash下载

logstash下载地址

logstash版本与elasticsearch版本一致

多文件形式解决方案

C:\Users\admin\Desktop\a 目录下

1.创建生成日志的测试脚本

createlog.sh

#!/bin/bash
icnt=1;
while (true)
do
  echo '{"id":'${icnt}',"name":"test","loginTime":1620269208}' >> login_log_20210506.log
  echo '{"id":'${icnt}',"name":"test","logoutTime":1620269208}' >> logout_log_20210506.log
  icnt=$((icnt+1));
done

生成日志文件

login_log_20210506.log

{"id":1,"name":"test","loginTime":1620269208}
{"id":2,"name":"test","loginTime":1620269208}
{"id":3,"name":"test","loginTime":1620269208}

logout_log_20210506.log

{"id":1,"name":"test","logoutTime":1620269208}
{"id":2,"name":"test","logoutTime":1620269208}
{"id":3,"name":"test","logoutTime":1620269208}

2.编写logstash配置文件a.conf

# 输入
input {

  # 文件形式输入
  file {
    # 指定输入文件 login*.log为所有日志文件
    path => [ "C:/Users/admin/Desktop/a/login*.log" ]
    # 监听文件的起始位置
    start_position => "beginning"
    type => "loginlog"
    codec => json {
      charset => "UTF-8"
    }   
  }

  # 文件形式输入
  file {
    # 指定输入文件 logout*.log为所有日志文件
    path => [ "C:/Users/admin/Desktop/a/logout*.log" ]
    start_position => "beginning"
    type => "logoutlog"
    codec => json {
      charset => "UTF-8"
    }   
  }

}

filter {
  grok {
    # 截取文件名filename作为临时变量,不会输出到es中
    match => ["path","%{GREEDYDATA}/%{GREEDYDATA:[@metadata][filename]}\.log"]
  }

  # 一行日志为一个message，将message转为json数据
  json {
    source => "message" 
  } 

  # 类型为loginlog
  if[type] == "loginlog"{
    mutate {
      # 默认字段类型为字符串,转换字段类型
      convert => { "id" => "integer" }
      convert => { "loginTime" => "integer" }
    }
  }

  # 类型为bulletlog
  if[type] == "logoutlog"{
    mutate {
      # 默认字段类型为字符串,转换字段类型
      convert => { "id" => "integer" }
      convert => { "logoutTime" => "integer" }
    }
  }


  mutate {
    # 移除指定字段，不让其输出到es
    remove_field => ["@version"]
    remove_field => ["@timestamp"]
    remove_field => ["host"]
    remove_field => ["path"]
  }
}

# 输出到es
output {
  elasticsearch {
    # 指定es
    hosts => "127.0.0.1:9200"
    # 将filename作为索引名称
    index => "%{[@metadata][filename]}"
  }
  stdout { codec => json_lines }
}

3.运行logstash 并指定使用a.conf配置文件

logstash.bat -f a.conf

4.连接elasticsearch查看生成的索引信息

http://localhost:9200/login_log_20210506/_search
http://localhost:9200/logout_log_20210506/_search