logstash安装及使用

最新推荐文章于 2024-02-17 14:32:35 发布

叶子不落

最新推荐文章于 2024-02-17 14:32:35 发布

阅读量223

点赞数

分类专栏： elasticsearch 文章标签： elasticsearch linux

本文链接：https://blog.csdn.net/FAN_1998/article/details/107409091

版权

elasticsearch 专栏收录该内容

2 篇文章 0 订阅

订阅专栏

#解压logstash包

tar -zxf logstash-6.2.2.tar.gz

#移到soft文件下创建logstash622文件夹

mv logstash-6.2.2 soft/logstash622

#进去至bin目录

cd soft/logstash622/bin/

#空白输入输出

./logstash -e 'input { stdin {} } output { stdout {} }'

#环境变量

./logstash -e 'input { stdin {} } output { stdout { codec => rubydebug}  }'

#创建config文件夹

mkdir /opt/config

#进去config

cd /opt/config/

编辑文本

vi mylog.conf      
↓
input {
    stdin {}
}
output {
    stdout {
        codec => rubydebug
    }
}

#加载环境变量文件

./logstash -f /opt/config/mylog.conf

#创建一个文本

vi aa.txt

（随便输入字）

#进去config

cd /opt/config/

编辑文本 (全量读取：没有更改的情况下只可以读取一次)

vi mylog.conf 
↓
input {
    file {
        path => "/opt/aa.txt"
        start_position => "beginning"
    }
}
output {
    stdout {
        codec => rubydebug
    }
}

#加载环境变量文件

./logstash -f /opt/config/mylog.conf

#全量读取：随时读取

input {
    file {
        path => "/opt/aa.txt"
        start_position => "beginning"
        sincedb_path => "/dev/null"
    }
}
output {
    stdout {
        codec => rubydebug
    }
}

#读取json文件

input {
    file {
        path => "/opt/aa.txt"
        start_position => "beginning"
        sincedb_path => "/dev/null"
        codec => json
    }
}
output {
    stdout {
        codec => rubydebug
    }
}

同级读取json日志文件

input {
    file {
        path => "/opt/aa.txt"
        start_position => "beginning"
        sincedb_path => "/dev/null"
        codec => json
    } 
}
filter {
        mutate {
            add_field => { "@adv" => "%{cm}" }
        }
        json {
            source => "@adv"
            remove_field => [ "@adv","cm" ]
        }
}
output {
    stdout {
        codec => rubydebug
    }
}

(
78461|event_login|958975260460|192.168.56.233
40265|event_login|960670860460|192.168.56.247
37170|event_login|962614860460|192.168.56.133
)

↓

正则表达式的匹配

input {
    file {
        path => "/opt/aa.txt"
        start_position => "beginning"
        sincedb_path => "/dev/null"
    }
}
filter {
    grok {
        match => { "message" => "%{NUMBER:user_id}\|%{WORD:event_name}\|%{NUMBER:times}\|%{IP:client_address}" }
        remove_field => ["message"]
    }
}
output {
    stdout {
        codec => rubydebug
    }
}

input {
    file {
        path => "/opt/aa.txt"
        start_position => "beginning"
        sincedb_path => "/dev/null"
    }
}
filter {
    grok {
        match => { "message" => "(?<userid>[0-9]+)\|(?<event_name>[a-zA-Z_]+)\|(?<times>[0-9]+)\|(?<clientip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})" }
        remove_field => ["message"]
    }
}
output {
    stdout {
        codec => rubydebug
    }
}

多类型文件解析

input {
    file {
        path => "/opt/aa.txt"
        start_position => "beginning"
        sincedb_path => "/dev/null"
        type => "system"
    }
    file {
        path => "/opt/bb.txt"
        start_position => "beginning"
        sincedb_path => "/dev/null"
        codec => json
        type => "action"
    }
}
filter {
    if [type] == "system" {

        grok {
            match => { "message" => "(?<userid>[0-9]+)\|(?<event_name>[a-zA-Z_]+)\|(?<times>[0-9]+)\|(?<clientip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})" }
            remove_field => ["message"]
        }
    }else {
        mutate {
                add_field => { "@adv" => "%{cm}" }
        }
        json {
                source => "@adv"
                remove_field => [ "@adv","cm" ] 
        }
    }
}