项目：ELKB日志采集系统部署实现（arm环境）

root@ES-201:~# vim /etc/profile
export JAVA_HOME=/usr/local/jdk
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin

# 生效变量

root@ES-201:~# source /etc/profile

# 验证安装

root@ES-201:~# java -version
java version "21.0.2" 2024-01-16 LTS
Java(TM) SE Runtime Environment (build 21.0.2+13-LTS-58)
Java HotSpot(TM) 64-Bit Server VM (build 21.0.2+13-LTS-58, mixed mode, sharing)

根据官方文档安装elasticsearch

# 下载并安装公共密钥

root@ES-200:~# wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
root@ES-200:~# apt-get install apt-transport-https

# 保存库定义到 /etc/apt/sources.list.d/elastic-8.x.list 列表中

root@ES-200:~# echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-8.x.list

# 安装后，会自动创建elasticsearch用户

root@ES-200:~# apt-get update && apt-get install elasticsearch

## 启动服务前，简单配置修改，将 xpack.security.enabled 和 xpack.security.http.ssl.enabled 修改为false
## 默认是https 修改为http

root@ES-200:/etc/elasticsearch# vim elasticsearch.yml
.....
xpack.security.enabled: false
....
xpack.security.http.ssl:
enabled: false
keystore.path: certs/http.p12

# 配置内核参数

root@ES-201:~# vim /etc/security/limits.conf
...
* soft nofile 65536
* hard nofile 65536
* soft nproc 2048
* hard nproc 2048
* soft memlock unlimited
* hard memlock unlimited

# 启动服务

root@ES-200:~# systemctl start elasticsearch.service

# 查看服务状态，占用端口的情况

root@ES-200:~# systemctl status elasticsearch.service
● elasticsearch.service - Elasticsearch
Loaded: loaded (/lib/systemd/system/elasticsearch.service; disabled; vendor preset: enabled)
Active: active (running) since Fri 2024-05-17 02:35:36 UTC; 9s ago
Docs: https://www.elastic.co
Main PID: 1559 (java)
Tasks: 80 (limit: 2191)
.....

root@ES-200:~# netstat -ntl
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State
tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN
tcp6 0 0 ::1:9300 :::* LISTEN
tcp6 0 0 127.0.0.1:9300 :::* LISTEN
tcp6 0 0 :::9200 :::* LISTEN
tcp6 0 0 :::22 :::* LISTEN

# 测试请求elasticsearch服务返回的情况，如下返回则正常

root@ES-200:/etc/elasticsearch# curl -X GET http://192.168.10.200:9200
{
"name" : "ES-200",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "P4LE8GyYTbiTTjYEuz2h7g",
"version" : {
"number" : "8.13.4",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "da95df118650b55a500dcc181889ac35c6d8da7c",
"build_date" : "2024-05-06T22:04:45.107454559Z",
"build_snapshot" : false,
"lucene_version" : "9.10.0",
"minimum_wire_compatibility_version" : "7.17.0",
"minimum_index_compatibility_version" : "7.0.0"
},
"tagline" : "You Know, for Search"
}

到此，elasticsearch部署完成！！！

（二）部署ES-head可视化es插件

ES-head可视化es插件，elasticsearch-head可理解为是一个数据可视化工具，是集群管理工具、数据可视化、增删改查工具；可以在浏览器进行访问页面化

# 需要配合nodejs环境才能正常使用，安装nodejs

root@ES-200:~/elasticsearch-head-5.0.0# apt install nodejs
root@ES-200:~/elasticsearch-head-5.0.0# node -v
v12.22.9

# 安装nodejs的管理器npm

root@ES-200:~/elasticsearch-head-5.0.0# apt install npm

# 安装ES-head依赖包，包在package.json下

root@ES-200:~# cd elasticsearch-head-5.0.0/
root@ES-200:~/elasticsearch-head-5.0.0# npm install
.....
npm ERR! code 1
npm ERR! path /root/elasticsearch-head-5.0.0/node_modules/phantomjs-prebuilt
npm ERR! command failed
npm ERR! command sh -c node install.js
npm ERR! PhantomJS not found on PATH
npm ERR! Unexpected platform or architecture: linux/arm64
npm ERR! It seems there is no binary available for your platform/architecture
npm ERR! Try to install PhantomJS globally

npm ERR! A complete log of this run can be found in:
npm ERR! /root/.npm/_logs/2024-05-17T03_14_50_182Z-debug-0.log

# 补包，自动安装失败的插件

root@ES-200:~/elasticsearch-head# npm install phantomjs-prebuilt@2.1.16 --ignore-script
....
added 526 packages, and audited 527 packages in 24s
22 packages are looking for funding
run `npm fund` for details
47 vulnerabilities (3 low, 9 moderate, 27 high, 8 critical)
To address issues that do not require attention, run:
npm audit fix
To address all issues possible (including breaking changes), run:
npm audit fix --force
Some issues need review, and may require choosing
a different dependency.
Run `npm audit` for details.

# 安装完成后，后台grunt启动，发布ES-head 到前端web

root@ES-200:~/elasticsearch-head# nohup npm run start &
[1] 2125
root@ES-200:~/elasticsearch-head# nohup: ignoring input and appending output to 'nohup.out'
回车按下
root@ES-200:~/elasticsearch-head# jobs
[1]+ Running nohup npm run start &

# 查看占用端口

root@ES-200:~/elasticsearch-head# netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:9100 0.0.0.0:* LISTEN 2137/grunt
tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 705/systemd-resolve
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 742/sshd: /usr/sbin
tcp6 0 0 127.0.0.1:9300 :::* LISTEN 2061/java
tcp6 0 0 :::9200 :::* LISTEN 2061/java
tcp6 0 0 :::22 :::* LISTEN 742/sshd: /usr/sbin
tcp6 0 0 ::1:9300 :::* LISTEN 2061/java

# elasticsearch默认不允许跨域访问，需要开启，否则访问不了
# 在配置文件中elasticsearch.yml末尾添加

root@ES-200:~# vim /etc/elasticsearch/elasticsearch.yml
...
# 允许ES跨域访问
http.cors.enabled: true
http.cors.allow-origin: "*"

web访问：http://192.168.10.200:9100/
连接：http://192.168.10.200:9200/

至此，ES-head完成配置！！！！

ES-head插件显示的问题：

如果没有数据展示，且浏览器分析发现 406 错误
elasticsearch-head无法显示数据问题处理

root@ES-200:~# vim /root/elasticsearch-head/_site/vendor.js
第6886行：
   contentType: "application/x-www-form-urlencoded",
   修改为：contentType: "application/json;charset=UTF-8",

第7573行;
    var inspectData = s.contentType === "application/x-www-form-urlencoded" &&
   修改为：
    var inspectData = s.contentType === "application/json;charset=UTF-8" &&

脚本模拟插入数据到elasticsearch中，后续很多地方模拟都需要用到这个脚本

root@ES-200:~# vim my-doc.sh
#!/bin/bash
read -p "输入要新增的文档个数: " num

for ((i=1;i<=$num;i++))
do
m_name=`shuf -n 1 -e jackson smith peter natasha hulk bruce jack mary terry mike john tony stack steven roges niko rabom bili david`
m_age=`shuf -n 1 -i 20-30`
m_hobby=`shuf -n 1 -e cat dog duck reading run fly eat bike fish shot football `

curl -XPOST '192.168.10.201:9200/class/student/'$i'' -H 'Content-Type: application/json' -d '{ "name" : "'$m_name'","age": "'$m_age'","hobby" : "'$m_hobby'"}'
done

# 创建索引

# 执行脚本，模拟插入一条数据

root@ES-200:~# ./my-doc.sh
输入要新增的文档个数: 1

# 浏览器查看数据

# 查看具体数据

（三）kibana部署

官方部署文档：https://www.elastic.co/guide/en/kibana/current/deb.html

本次部署使用版本：kibana-8.13.4

根据官方文档操作即可
# 下载并安装公共密钥

root@kibana-203:~# wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
root@kibana-203:~# sudo apt-get install apt-transport-https

# 保存库定义到 /etc/apt/sources.list.d/elastic-8.x.list 列表中

root@kibana-203:~# echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-8.x.list

# 安装kibana

root@kibana-203:~# sudo apt-get update && sudo apt-get install kibana

# 配置文件修改，解开注释

root@kibana-203:~# vim /etc/kibana/kibana.yml
server.port: 5601
server.host: "192.168.10.203"
server.name: "kibana-203"
elasticsearch.hosts: ["http://192.168.10.200:9200","http://192.168.10.201:9200","http://192.168.10.202:9200"]
i18n.locale: "zh-CN"

# 启动服务，并设置自启

root@kibana-203:~# systemctl start kibana.service
root@kibana-203:~# systemctl enable kibana.service

# 如果服务起不来，排查kibana服务，使用命令定位分析问题

root@ES-200:~# journalctl -xeu kibana.service

# 查看占用端口

root@kibana-203:~# netstat -anutlp | grep "5601"
tcp 0 0 192.168.10.203:5601 0.0.0.0:* LISTEN 2075/node

web访问：http://192.168.10.203:5601/app/home#/

到这，安装成功！！！

（四）filebeat部署

Filebea安装

官网部署文档：https://www.elastic.co/guide/en/beats/filebeat/8.13/setup-repositories.html#_apt

# 下载并安装公共密钥

root@nginx-40:~# wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -

# 安装apt-transport-https包

root@nginx-40:~# sudo apt-get install apt-transport-https

# 保存库定义到 /etc/apt/sources.list.d/elastic-8.x.list 列表中

root@nginx-40:~# echo "deb https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-8.x.list

# 安装filebeat

root@nginx-40:~# sudo apt-get update && sudo apt-get install filebeat

# 将enabled设置为true，启用模块

root@nginx-40:~# vim /etc/filebeat/filebeat.yml

# 启动日志采集器

root@nginx-40:~# systemctl start filebeat.service
root@nginx-40:~# systemctl enable filebeat.service

# 可能运行一会端口突然就没有

root@nginx-40:~# netstat -antulp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 724/systemd-resolve
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 773/sshd: /usr/sbin
tcp 0 0 192.168.10.40:22 192.168.10.1:57077 ESTABLISHED 16559/sshd: root@pt
tcp 0 196 192.168.10.40:22 192.168.10.1:55717 ESTABLISHED 892/sshd: root@pts/
tcp6 0 0 :::33060 :::* LISTEN 791/mysqld
tcp6 0 0 :::22 :::* LISTEN 773/sshd: /usr/sbin
tcp6 0 0 :::3306 :::* LISTEN 791/mysqld
udp 0 0 127.0.0.53:53 0.0.0.0:* 724/systemd-resolve
udp 0 0 127.0.0.1:48255 127.0.0.53:53 ESTABLISHED 16630/filebeat
udp 0 0 192.168.10.40:48408 114.114.114.114:53 ESTABLISHED 724/systemd-resolve

# 查看进程运行

root@nginx-40:~# ps aux | grep "filebeat"
root 16630 0.7 8.7 1900164 175644 ? Ssl 09:41 0:00 /usr/share/filebeat/bin/filebeat --environment systemd -c /etc/filebeat/filebeat.yml --path.home /usr/share/filebeat --path.config /etc/filebeat --path.data /var/lib/filebeat --path.logs /var/log/filebeat
root 16655 0.0 0.0 3432 1792 pts/0 S+ 09:43 0:00 grep --color=auto filebeat

# 查看服务运行状态，running正常运行

root@nginx-40:~# systemctl status filebeat.service

简单说一说Filebeat的原理

Inputs组件：

Inputs负责定义Filebeat要监视的日志文件或位置。在配置文件中，用户可以指定一个或多个输入源，每个输入源可以是一个具体的文件路径、一个目录

Harvesters组件：

对于每个输入的日志文件，Filebeat都会启动一个Harvester（收割机）来读取文件内容。Harvester会逐行读取文件内容，并将读取到的数据发送到libbeat（Filebeat的一个内部组件，负责数据的聚合和转发）
每个文件都会启动一个独立的Harvester，这意味着Filebeat可以同时监视并读取多个日志文件。Harvester会保持文件描述符的打开状态，以便在文件更新时能够立即读取到新内容

libbeat组件：

libbeat是Filebeat的一个内部组件，负责接收来自Harvester的日志数据，并进行聚合和预处理。然后，libbeat会将处理后的数据发送到用户指定的输出目标

Outputs组件：

Outputs定义了Filebeat将日志数据转发到的目标系统。Filebeat支持多种输出目标，如Elasticsearch、Logstash等，可以在配置文件中指定输出目标的地址、端口等信息

Filebeat简单配置使用

示例一：配置采集nginx

修改nginx的日志的格式为json，否则filebeat无法识别！！

# 需要修改nginx配置文件 nginx.conf 两处地方

root@nginx-40:~# cat /usr/local/nginx/conf/nginx.conf
user nginx nginx;
worker_processes 1;
error_log logs/error.log;
pid logs/nginx.pid;

events {
worker_connections 1024;
}

http {
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
log_format json '{ "time_local": "$time_local", '
'"remote_addr": "$remote_addr", '
'"referer": "$http_referer", '
'"method": "$request_method", '
'"uri": "$request_uri", '
'"status": $status, '
'"bytes": $body_bytes_sent, '
'"user_agent": "$http_user_agent", '
'"x_forwarded": "$http_x_forwarded_for" '
' }';
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
charset utf-8;

server {
listen 80;
server_name www.news.com;
access_log /data/web-data/logs/new.com.access.log json;
location / {
root /data/web-data/news.com;
index index.html;
}
}
}

# filebeat配置文件，注意是yml形式，格式很严格
root@nginx-40:~# cat /etc/filebeat/filebeat.yml

filebeat.inputs:
- type: log
enabled: true
paths:
- /data/web-data/logs/new.com.access.log # 监视的日志路径
tags: ["news.com"]
fields:
log_source: nginx-22_news.com.access.log # # 向文档中添加自定义字段
fields_under_root: true ## 将自己添加的filed字段，放在 json文档根
json.keys_under_root: true # # 将文档中含有json格式中的key 提取到文档的根位置
json.overwrite_keys: true # 有重复的内容，就覆盖

filebeat.config.modules:
path: ${path.config}/modules.d/*.yml # 指定了Filebeat应该在哪里查找模块配置文件
reload.enabled: false # 控制Filebeat是否应该监视模块配置文件的更改，并在检测到更改时重新加载这些配置

output.elasticsearch:
hosts: ["192.168.10.200:9200"] # 连接ES主机，如果是集群，就写全部的IP地址
indices:
- index: "nginx-22_news.com.access.log-%{+yyyy.MM.dd}" # ES索引创建
when.contains:
tags: "news.com" # 必须和上面的日志tags一致，进行匹配

测试请求nginx，到 ES 中观察是否有索引产生，如果长时间没有索引产生，试着重启 filebeat ，或者
刷新一下网页，促使网页日志文件有新的行产生。

日志信息如下：

# 经过 Json 处理后的访问日志，单独成为 key ，后期可以单独进行分析。

# 没有经过json 处理的日志，形式如下

示例二：继续添加日志采集

# 添加测试日志test.log

root@nginx-40:~# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /data/web-data/logs/new.com.access.log
tags: ["news.com"]
fields:
log_source: nginx-22_news.com.access.log
fields_under_root: true
json.keys_under_root: true
json.overwrite_keys: true

- type: log
enabled: true
paths:
- /data/web-data/logs/test.log
tags: ["test"]
fields:
log_source: test.log
fields_under_root: true
json.keys_under_root: true
json.overwrite_keys: true

filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false

output.elasticsearch:
hosts: ["192.168.10.200:9200","192.168.10.201:9200","192.168.10.202:9200"]
indices:
- index: "nginx-22_news.com.access.log-%{+yyyy.MM.dd}"
when.contains:
tags: "news.com"

- index: "test.log-%{+yyyy.MM.dd}"
when.contains:
tags: "test"

# 重启filebeat服务，生效配置

root@nginx-40:~# systemctl restart filebeat.service

注意：重启过程中，可能因为自己配置文件写错了服务起不来，可以查看进程是否在运行排查

root@nginx-40:~# ps aux | grep "filebeat"

json.keys_under_root: true
json.overwrite_keys: true

filebeat.yal中配置这两个字段目的是将json数据提取出来再外面，而不是全放在message字段中，针对json格式
将文档中含有json格式中的key 提取到文档的根位置
有重复的内容，就覆盖

# 模拟日志，插入日志的脚本

root@nginx-40:/data/web-data/logs# cat autoproductlog.sh

#!/bin/bash
# 能够模拟常规格式或 json格式的web日志
# version: v2.0
# 用法: ./auto_log.sh 1.log [json]
# 加参数 json就生成 json格式的,不加就是常规格式

if [ $# -lt 1 ]
then
echo "EORRO: Filname is Missing"
echo "Useage: ./auto_log.sh filename [json]"
exit 1
fi

read -p "请输入要生成的日志条目数： " num

for ((i=1;i<=$num;i++))
do
ip_1=`shuf -n 1 -e 100 30 202`
ip_2=`shuf -n 1 -e 80 20 100`
ip_3=`shuf -n 1 -i 50-60`
ip_4=`shuf -n 1 -i 10-30`
my_ip="${ip_1}.${ip_2}.${ip_3}.${ip_4}"
##referer来源地址
ref=$( shuf -n 1 -e http://www.news.com http://www.taobao.com http://www.google.com http://www.baidu.com)
##组装URI
nohtml=`shuf -n 1 -i 100-110`
mime=`shuf -n 1 -e jpeg gif html js png jpg`
uri1=`shuf -n 1 -e images html css js app news shop user`
uri="/${uri1}/${nohtml}.${mime}"
##传输量
size=`shuf -n 1 -i 200-600000`
##状态码
code=`shuf -n 1 -e 200 301 302 304 404 401 500 502`
##客户端浏览器
client=`shuf -n 1 -e Chrome IE QQBrose r Firefox UC`

## 生成随机访问时间
HH=`shuf -n 1 -i 0-23`
if [ $HH -lt 10 ]; then time_hh=0${HH}; else time_hh=$HH; fi
MM=`shuf -n 1 -i 0-59`
if [ $MM -lt 10 ]; then time_mm=0${MM}; else time_mm=$MM; fi
SS=`shuf -n 1 -i 0-59`
if [ $SS -lt 10 ]; then time_ss=0${SS}; else time_ss=$SS; fi
DD=`shuf -n 1 -i 1-30`
if [ $DD -lt 10 ]; then m_dd=0${DD}; else m_dd=$DD; fi
m_month=`shuf -n 1 -e Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec`
m_year=2022

## 请求时长
m_time="0.`shuf -n 1 -i 1-9`00"

## 客户端操作系统
m_sys=`shuf -n 1 -e Windows Android MacOS IOS Linux`

if [ "$2" == "json" ]
then
cat >> ./$1 <<EOF
{ "http_time": "${m_dd}/${m_month}/${m_year}:${time_hh}:${time_mm}:${time_ss} +0800", "remote_addr": "$my_ip", "referer": "$ref", "request": "GET $uri HTTP/1.1", "status": $code, "bytes": $size,"AgentVersion":"$m_sys $client","request_time": "$m_time" }
EOF
else
cat >> ./$1 <<EOF
${my_ip} - - [${m_dd}/${m_month}/${m_year}:${time_hh}:${time_mm}:${time_ss} +0800] "GET ${uri} HTTP/1.1" ${code} ${size} "${ref}" ${m_sys} ${client} ${m_time}
EOF
fi

done
echo "成功插入日志${num}条"

# 使用示例：

root@nginx-40:/data/web-data/logs# ./autoproductlog.sh test.log
请输入要生成的日志条目数： 1

# 用json格式插入

root@nginx-40:/data/web-data/logs# ./autoproductlog.sh test.log json
请输入要生成的日志条目数： 1

# ES生成索引

# 日志数据

示例三：监控java日志的配置

# 监控Java日志

root@nginx-40:~# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /data/web-data/logs/new.com.access.log
tags: ["news.com"]
fields:
log_source: nginx-22_news.com.access.log
fields_under_root: true
json.keys_under_root: true
json.overwrite_keys: true

- type: log
enabled: true
paths:
- /data/web-data/logs/test.log
tags: ["test"]
fields:
log_source: test.log
fields_under_root: true
json.keys_under_root: true
json.overwrite_keys: true

- type: log
enabled: true
paths:
- /data/web-data/logs/java.log
tags: ["java"]
fields:
log_source: java.log
#multiline.pattern: '^[0-9]{4}[-][0-9]{2}[-][0-9]{2}[ ][0-9]{2}:[0-9]{2}:[0-9]{2}'
multiline.pattern: '^[0-9]{2}[-][A-Z][a-z]{2}[-][0-9]{4}[ ][0-9]{2}:[0-9]{2}:[0-9]{2}'
multiline.negate: true
multiline.match: after

filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false

output.elasticsearch:
hosts: ["192.168.10.200:9200"]
indices:
- index: "nginx-22_news.com.access.log-%{+yyyy.MM.dd}"
when.contains:
tags: "news.com"

- index: "test.log-%{+yyyy.MM.dd}"
when.contains:
tags: "test"

- index: "java.log-%{+yyyy.MM.dd}"
when.contains:
tags: "java"

（四）logstash部署

官方文档部署：https://www.elastic.co/guide/en/logstash/8.13/installing-logstash.html#_apt

logstash安装

# 下载并安装公共密钥

root@logstash-204:~# wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elastic-keyring.gpg

# 安装apt-transport-https包

root@logstash-204:~# sudo apt-get install apt-transport-https

# 保存库定义到 /etc/apt/sources.list.d/elastic-8.x.list 列表中，添加安装源到本地

root@logstash-204:~# echo "deb [signed-by=/usr/share/keyrings/elastic-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-8.x.list

# 安装 logstash

root@logstash-204:~# sudo apt-get update && sudo apt-get install logstash

# 启动服务

root@logstash-204:/etc/logstash# systemctl start logstash

# 如果找不到logstash，可以使用whereis搜索命令所在

root@logstash-204:/etc/logstash# whereis logstash
logstash: /etc/logstash /usr/share/logstash
root@logstash-204:/etc/logstash# ls -l /usr/share/logstash/bin/logstash
-rwxr-xr-x 1 root root 2149 May 6 13:02 /usr/share/logstash/bin/logstash

# 命令行输出，测试
# 输入一下命令，可以进入交互界面，输入hello world 会输出相应内容

root@logstash-204:/etc/logstash# /usr/share/logstash/bin/logstash -e 'input { stdin { } } output { stdout {codec=>rubydebug} }'

# 官方编写文档：https://www.elastic.co/guide/en/logstash/current/configuration.html

# 编写配置文件，读取配置文件的方式

root@logstash-204:/etc/logstash# vim conf.d/elk.conf
input{
stdin{}
}

filter{
}

output{
stdout{}
}

# 读取配置文件启动，发生修改自动重载 --config.reload.automatic

[root@logstash-224 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/elk.conf --config.reload.automatic

logstash简单介绍

Logstash的数据处理过程主要包括：Inputs、Filters、Outputs 三部分，另外在Inputs和Outputs中可以使用Codecs对数据格式进行处理。
这四个部分均以插件形式存在，用户通过定义pipeline配置文件，设置需要使用input,filter,output,codec插件，以实现特定的数据采集，数据处理，数据输出等功能

        Inputs：用于从数据源获取数据，常见的插件如 file, syslog, redis, beats 等
        Filters：用于处理数据如格式转换，数据派生等，常见的插件如grok, mutate, drop, clone, geoip等
        Outputs：用于数据输出，常见的插件如 elastcisearch，file, graphite, statsd等
        Codecs：Codecs(编码插件)不是一个单独的流程，而是在输入和输出等插件中用于数据转换的模块，用于对数据进行编码处理，
常见的插件如json，multiline

logstash简单使用

# 简单配置一下logstash文件

root@logstash-204:/etc/logstash/conf.d# vim elk.conf
input {
file {
path => "/etc/logstash/conf.d/test.log"
exclude =>"*.gz"
start_position => "beginning"
ignore_older => 0
sincedb_path => "/dev/null"
type => "test_01"
}
}

filter{}

output {
stdout {}
}

# 使用文件启动logstash检索日志发生修改自动重载 --config.reload.automatic

root@logstash-204:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/elk.conf --config.reload.automatic
....
阻塞，等待日志写入后输出

# 打开bash新的会话，编写简易的脚本写日志

root@logstash-204:/etc/logstash/conf.d# cat product_log.sh
#!/bin/bash
num=`shuf -n 1 -i 2-254`
msg='192.168.10.'$num' - - [11/Jun/2020:11:45:33 +0800] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0"'
echo $msg >> /etc/logstash/conf.d/test.log

#执行脚本后，logstash输出结果

# logstash文件配置文件 elk.conf - 增加filter过滤，保存自动重载

.....
filter{
grok{
match => {"message" => "%{IPV4:my_ip}"}
}
}
.......

#执行脚本后，logstash输出结果

# logstash文件配置文件 elk.conf - 继续增加filter条件 - 获取时间字段，保存重载

.....
filter{
grok{
match => {"message" => "%{IPV4:my_ip}\ -\ -\ \[%{HTTPDATE:my_time}\]"}
}
}
.......

# 解释：%{括号内的内容，就是正则匹配到的内容}，将匹配到的内容给 my_ip 和 my_time 字段

# 执行脚本后，logstash输出结果

# logstash文件配置文件 elk.conf - 继续增加filter条件 - 删除字段，保存重载

.....
filter{
grok{
match => {"message" => "%{IPV4:my_ip}\ -\ -\ \[%{HTTPDATE:my_time}\]"}
remove_field => ["original"] # 删除 message 字段
}
}
.......

# 执行脚本后，logstash输出结果 - 不在有 message 字段

# logstash文件配置文件 elk.conf - 继续增加filter条件 - 继续匹配日志中其他内容保存到对应字段，保存重载

.....
filter{
grok{
match => {"message" => "%{IPV4:my_ip}\ -\ -\ \[%{HTTPDATE:my_time}\]\ %{QS:my_request}\ %{NUMBER:my_status}\ %{NUMBER:my_bytes}"}
remove_field => ["original"] # 删除 message 字段
}
}
.......

# 执行脚本后，logstash输出结果

# 自定义类型
示例：自定义grok pattern
我们通过pattern_definitions参数，以键值对的方式定义pattern名称和内容。
也可以通过pattern_dir参数，以文件的形式读取pattern。

filter {
   grok {
       match => {
           "message" => "%{SERVICE:my_service}"
       }
       pattern_definitions => {
           "SERVICE" => "[a-z0-9]{10,11}"
       }
   }
}

# logstash文件配置文件 elk.conf - 增加date插件

.....
filter{
grok{
match => {"message" => "%{IPV4:my_ip}\ -\ -\ \[%{HTTPDATE:my_time}\]\ %{QS:my_request}\ %{NUMBER:my_status}\ %{NUMBER:my_bytes}"}
remove_field => ["original"] # 删除 message 字段
}
date{
match => ["my_time","dd/MMM/yyyy:HH:mm:ss Z"] # 自定义日期时间格式
target => "@timestamp"
}
}
.......

# 执行脚本后，logstash输出结果

# logstash文件配置文件 elk.conf - 增加mutate插件

.....
filter{
grok{
match => {"message" => "%{IPV4:my_ip}\ -\ -\ \[%{HTTPDATE:my_time}\]\ %{QS:my_request}\ %{NUMBER:my_status}\ %{NUMBER:my_bytes}"}
remove_field => ["original"] # 删除 message 字段
}
date{
match => ["my_time","dd/MMM/yyyy:HH:mm:ss Z"] # 自定义日期时间格式
target => "@timestamp"
}
mutate{
convert => ["my_ip","string"] # 使用string类型
gsub => ["my_request","HTTP","http"] # 将HTTP替换为http
split => ["message"," "] # 分隔符分割字符串为数值功能
add_field => ["Browser","%{[message][20]}"] # 增加字段 Browser
rename => ["my_ip","my_IP"] # 将 my_ip 字段重命名为 my_IP
#remove_field => ["message"] # 删除字段message
}
}
.......

# 执行脚本后，logstash输出结果

# logstash文件配置文件 elk.conf - 增加Geoip插件

.....
filter{
grok{
match => {"message" => "%{IPV4:my_ip}\ -\ -\ \[%{HTTPDATE:my_time}\]\ %{QS:my_request}\ %{NUMBER:my_status}\ %{NUMBER:my_bytes}"}
remove_field => ["original"] # 删除 message 字段
}
date{
match => ["my_time","dd/MMM/yyyy:HH:mm:ss Z"] # 自定义日期时间格式
target => "@timestamp"
}
mutate{
convert => ["my_ip","string"] # 使用string类型
gsub => ["my_request","HTTP","http"] # 将HTTP替换为http
split => ["message"," "] # 分隔符分割字符串为数值功能
add_field => ["Browser","%{[message][20]}"] # 增加字段 Browser
rename => ["my_ip","my_IP"] # 将 my_ip 字段重命名为 my_IP
#remove_field => ["message"] # 删除字段message
}
geoip{
source => "my_IP" # IP来源
target => "[geoip-data]" # 存储GeoIP数据的字段
}
}
.......

# 执行脚本后，logstash输出结果

# 最终配置文件样例 - elk.conf

root@logstash-204:/etc/logstash/conf.d# cat elk.conf
input {
file {
path => "/etc/logstash/conf.d/test.log"
exclude =>"*.gz"
start_position => "beginning"
ignore_older => 0
sincedb_path => "/dev/null"
type => "test_01"
}
}

filter{
grok{
match => {"message" => "%{IPV4:my_ip}\ -\ -\ \[%{HTTPDATE:my_time}\]\ %{QS:my_request}\ %{NUMBER:my_status}\ %{NUMBER:my_bytes}"}
remove_field => ["message"]
}
date{
match => ["my_time","dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
}
mutate{
convert => ["my_ip","string"]
gsub => ["my_request","HTTP","http"]
split => ["message"," "]
add_field => ["Browser","%{[message][20]}"]
rename => ["my_ip","my_IP"]
#remove_field => ["message"]
}
geoip{
source => "my_IP"
target => "[geoip-data]"
}
}

output {
stdout {}
}

三、方案一：实现ELB的流程

（一）整体的流程图

（二）开始配置

# 将接收input，用5044来监听，output 传出日志到ES中

root@logstash-204:/etc/logstash/conf.d# cat elk.conf
input {
beats {
port => 5044
}
}

filter{
grok{
pattern_definitions => {
"MY_CODE" => "[2345][0-9][0-9]"
}
match => {"message" => "%{IPV4:my_ip}\ -\ -\ \[%{HTTPDATE:my_time}\]\ %{QS:my_request}\ %{NUMBER:my_status}\ %{NUMBER:my_bytes}"}
}
date{
match => ["my_time","dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
}
mutate{
convert => ["my_ip","string"]
gsub => ["my_request","HTTP","http"]
split => ["message"," "]
gsub => ["message","\"",""]
add_field => ["Browser","%{[message][13]}"]
add_field => ["my_byte","%{[message][9]}"]
add_field => ["my_sec","%{[message][14]}"]
rename => ["my_ip","my_IP"]
remove_field => ["message"]
}

mutate{
convert => { "my_byte" => "integer" }
convert => { "my_sec" => "float" }
}

geoip{
source => "my_IP"
target => "[geoip-data]"
fields => ["city_name","region_name","country_name","ip"]
}
}

output {
elasticsearch {
hosts => ["http://192.168.10.200:9200"]
#index => "web-test_new-%{+YYYY.MM.dd}"
index => "web-test_new-%{+YYYY}"
}
#stdout{
# codec => rubydebug
#}
}

# 启动logstash，阻塞监听5044，等待日志

root@logstash-204:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/elk.conf --config.reload.automatic

# filebeat 配置文件修改将日志传输至 logstash 服务

root@nginx-40:/data/web-data/logs# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /data/web-data/logs/test.log
tags: ["test"]
fields:
log_source: test.log
fields_under_root: true
json.keys_under_root: true
json.overwrite_keys: true

filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false

output.logstash:
hosts: ["192.168.10.204:5044"]

# 重启filebeat服务，生效配置

root@nginx-40:/data/web-data/logs# systemctl restart filebeat.service

# 准备生成日志的脚本

root@nginx-40:/data/web-data/logs# cat autoproductlog.sh

#!/bin/bash
# 能够模拟常规格式或 json格式的web日志
# version: v2.0
# 用法: ./auto_log.sh 1.log [json]
# 加参数 json就生成 json格式的,不加就是常规格式

if [ $# -lt 1 ]
then
echo "EORRO: Filname is Missing"
echo "Useage: ./auto_log.sh filename [json]"
exit 1
fi

read -p "请输入要生成的日志条目数： " num

for ((i=1;i<=$num;i++))
do
ip_1=`shuf -n 1 -e 100 30 202`
ip_2=`shuf -n 1 -e 80 20 100`
ip_3=`shuf -n 1 -i 50-60`
ip_4=`shuf -n 1 -i 10-30`
my_ip="${ip_1}.${ip_2}.${ip_3}.${ip_4}"
##referer来源地址
ref=$( shuf -n 1 -e http://www.news.com http://www.taobao.com http://www.google.com http://www.baidu.com)
##组装URI
nohtml=`shuf -n 1 -i 100-110`
mime=`shuf -n 1 -e jpeg gif html js png jpg`
uri1=`shuf -n 1 -e images html css js app news shop user`
uri="/${uri1}/${nohtml}.${mime}"
##传输量
size=`shuf -n 1 -i 200-600000`
##状态码
code=`shuf -n 1 -e 200 301 302 304 404 401 500 502`
##客户端浏览器
client=`shuf -n 1 -e Chrome IE QQBrose r Firefox UC`

## 生成随机访问时间
HH=`shuf -n 1 -i 0-23`
if [ $HH -lt 10 ]; then time_hh=0${HH}; else time_hh=$HH; fi
MM=`shuf -n 1 -i 0-59`
if [ $MM -lt 10 ]; then time_mm=0${MM}; else time_mm=$MM; fi
SS=`shuf -n 1 -i 0-59`
if [ $SS -lt 10 ]; then time_ss=0${SS}; else time_ss=$SS; fi
DD=`shuf -n 1 -i 1-30`
if [ $DD -lt 10 ]; then m_dd=0${DD}; else m_dd=$DD; fi
m_month=`shuf -n 1 -e Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec`
m_year=2022

## 请求时长
m_time="0.`shuf -n 1 -i 1-9`00"

## 客户端操作系统
m_sys=`shuf -n 1 -e Windows Android MacOS IOS Linux`

## 生成ERROR字样测试数据
level=`shuf -n 1 -e ERROR INFO DEBUG`
message="出错了啦，呼叫管理员！！"

if [ "$2" == "json" ]
then
cat >> ./$1 <<EOF
{ "http_time": "${m_dd}/${m_month}/${m_year}:${time_hh}:${time_mm}:${time_ss} +0800", "remote_addr": "$my_ip", "referer": "$ref", "request": "GET $uri HTTP/1.1", "status": $code, "bytes": $size,"AgentVersion":"$m_sys $client","request_time": "$m_time" }
EOF
elif [ "$2" == "test" ];then
cat >> ./$1 <<EOF
{ "http_time": "${m_dd}/${m_month}/${m_year}:${time_hh}:${time_mm}:${time_ss} +0800", "remote_addr": "$my_ip", "referer": "$ref", "request": "GET $uri HTTP/1.1", "status": $code, "bytes": $size,"AgentVersion":"$m_sys $client","request_time": "$m_time", "message": "${level} ${message}"}
EOF
else
cat >> ./$1 <<EOF
${my_ip} - - [${m_dd}/${m_month}/${m_year}:${time_hh}:${time_mm}:${time_ss} +0800] "GET ${uri} HTTP/1.1" ${code} ${size} "${ref}" ${m_sys} ${client} ${m_time}
EOF
fi

done
echo "成功插入日志${num}条"

# 测试插入一条标准日志输出

root@nginx-40:/data/web-data/logs# ./autoproductlog.sh test.log
请输入要生成的日志条目数： 1
成功插入日志1条

# logstash 输出情况

# ES增加索引，新增数据一条

四、方案二：实现ELKB的流程

（一）整体的流程图

（二）使用kafka单机部署

# 官网下载地址：https://kafka.apache.org/downloads
按要求虚拟环境：

CPU：2

内存： 2G

# 下载安装包

root@kafka-205:/usr/local/src# wget https://downloads.apache.org/kafka/3.7.0/kafka_2.13-3.7.0.tgz

# 下载安装jdk

root@kafka-205:/usr/local# tar -xf jdk-21_linux-aarch64_bin.tar.gz
root@kafka-205:/usr/local# mv jdk-21.0.2 jdk

# 加入环境变量

root@kafka-205:/usr/local# vim /etc/profile
...
# jdk
export JAVA_HOME=/usr/local/jdk
export CLASSPATH=$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib:/test
export PATH=$JAVA_HOME/bin:$PATH

# 测试安装

root@kafka-205:/usr/local# java -version
java version "21.0.2" 2024-01-16 LTS
Java(TM) SE Runtime Environment (build 21.0.2+13-LTS-58)
Java HotSpot(TM) 64-Bit Server VM (build 21.0.2+13-LTS-58, mixed mode, sharing)

# 解压kafka，放在/usr/local中

root@kafka-205:/usr/local/src# tar -xf kafka_2.13-3.7.0.tgz -C /usr/local/
root@kafka-205:/usr/local# mv kafka_2.13-3.7.0 kafka

# 加入环境变量

root@kafka-205:/usr/local# vim /etc/profile
...
export PATH=$PATH:/usr/local/kafka/bin

root@kafka-205:/usr/local# source /etc/profile # 生效文件

# 修改zookeeper配置文件

root@kafka-205:/usr/local# cat /usr/local/kafka/config/zookeeper.properties
dataDir=/tmp/zookeeper
clientPort=2181
maxClientCnxns=0
tickTime=2000
initLimit=10
syncLimit=5
admin.enableServer=false
server.1=kafka-205:2888:3888

# 启动zookeeper服务，默认前台方式运行，需要在后台启动，添加 -daemon 参数即可

[root@kafka-50 /usr/local/kafka]# zookeeper-server-start.sh -daemon /usr/local/kafka/config/zookeeper.properties
root@kafka-205:/usr/local# netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 732/sshd: /usr/sbin
tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 703/systemd-resolve
tcp6 0 0 :::43317 :::* LISTEN 2040/java
tcp6 0 0 :::2181 :::* LISTEN 2040/java
tcp6 0 0 :::22 :::* LISTEN 732/sshd: /usr/sbin
udp 0 0 127.0.0.53:53 0.0.0.0:* 703/systemd-resolve

# 修改kafka服务配置文件

root@kafka-205:/usr/local# cat /usr/local/kafka/config/server.properties
broker.id=60
listeners=PLAINTEXT://kafka-205:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
#zookeeper.connect=kafka-60:2181,kafka-61:2181,kafka-62:2181
zookeeper.connect=kafka-205:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

# 启动kafka服务，后台运行 -daemon

root@kafka-205:/usr/local# kafka-server-start.sh -daemon /usr/local/kafka/config/server.properties

# 查看占用端口

root@kafka-205:/usr/local# netstat -ntulp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 732/sshd: /usr/sbin
tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 703/systemd-resolve
tcp6 0 0 :::2181 :::* LISTEN 2564/java
tcp6 0 0 :::42081 :::* LISTEN 2564/java
tcp6 0 0 :::22 :::* LISTEN 732/sshd: /usr/sbin
tcp6 0 0 :::39891 :::* LISTEN 3004/java
tcp6 0 0 192.168.10.205:9092 :::* LISTEN 3004/java
udp 0 0 127.0.0.53:53 0.0.0.0:* 703/systemd-resolve

到这，kafka成功安装！！！

（三）kafka配置

# 注意：Kafka 如果之前有过集群配置，先修改配置文件为单机，清理目录，重启 Kafka 和 zookeeper

[root@kafka-60 /tmp/zookeeper]#rm -rf /tmp/zookeeper/version-2/*
[root@kafka-60 /tmp/zookeeper]# rm -rf /tmp/kafka-logs/
[root@kafka-60 /tmp/zookeeper]# zookeeper-server-start.sh -daemon /usr/local/kafka/config/zookeeper.properties
[root@kafka-60 /tmp/zookeeper]# kafka-server-start.sh -daemon /usr/local/kafka/config/server.properties

# 创建主题 web-test ，1分区，1副本

root@kafka-205:/usr/local# kafka-topics.sh --create --bootstrap-server kafka-205:9092 --replication-factor 1 --partitions 1 --topic web-test
Created topic web-test.

# 再开启一个会话，运行一个命令行的消费，用来观察日志的接收

root@kafka-205:~# kafka-console-consumer.sh --bootstrap-server kafka-205:9092 --topic web-test
阻塞等待日志....

（四）Logstath 配置

# 添加 kafka-205 主机解析

root@logstash-204:/etc/logstash/conf.d# vim /etc/hosts
192.168.10.205 kafka-205

# 修改配置文件

root@logstash-204:/etc/logstash/conf.d# cat elk.conf
input {
# beats {
# port => 5044
# }
kafka{
bootstrap_servers => "192.168.10.205:9092"
topics=>["web-test"]
codec => json { charset => "UTF-8" }
}
}

filter{
grok{
pattern_definitions => {
"MY_CODE" => "[2345][0-9][0-9]"
}
match => {"message" => "%{IPV4:my_ip}\ -\ -\ \[%{HTTPDATE:my_time}\]\ %{QS:my_request}\ %{NUMBER:my_status}\ %{NUMBER:my_bytes}"}
}
date{
match => ["my_time","dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
}
mutate{
convert => ["my_ip","string"]
gsub => ["my_request","HTTP","http"]
split => ["message"," "]
gsub => ["message","\"",""]
add_field => ["Browser","%{[message][13]}"]
add_field => ["my_byte","%{[message][9]}"]
add_field => ["my_sec","%{[message][14]}"]
rename => ["my_ip","my_IP"]
remove_field => ["message"]
}

mutate{
convert => { "my_byte" => "integer" }
convert => { "my_sec" => "float" }
}

geoip{
source => "my_IP"
target => "[geoip-data]"
fields => ["city_name","region_name","country_name","ip"]
}
}

output {
elasticsearch {
hosts => ["http://192.168.10.200:9200"]
#index => "web-test_new-%{+YYYY.MM.dd}"
index => "web-test_new-%{+YYYY}"
}
#stdout{
# codec => rubydebug
#}
}

# 启动logstash，会启用消费者模式，并且能够自动创建消费者组，阻塞等待日志

root@logstash-204:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/elk.conf --config.reload.automatic

（三）Filebeat配置

# 添加 kafka-205 主机解析

root@nginx-40:/data/web-data/logs# vim /etc/hosts
192.168.10.205 kafka-205

# 修改配置文件

root@nginx-40:/data/web-data/logs# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /data/web-data/logs/test.log
tags: ["test"]
fields:
log_source: test.log
fields_under_root: true
json.keys_under_root: true
json.overwrite_keys: true

filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false

output.kafka:
hosts: ["192.168.10.205:9092"]
topic: "web-test"

# 重启服务，生效

root@nginx-40:/data/web-data/logs# systemctl restart filebeat.service

使用脚本生产日志 - 脚本上面有提供

root@nginx-40:/data/web-data/logs# ./autoproductlog.sh test.log
请输入要生成的日志条目数： 1

# 查看kafka，消费者情况

# 查看ES

到此，成功配置ELKB日志系统分析！！！！

补充kibana：添加数据视图

最后有几句话要说

这些只是部署的一个流程，如果深入掌握甚至精通还是要花费不少的精力去学习的，我也只是浅学了一下kafka和elasticsearch的一些基本使用！！！

当然，感谢阅读，谢谢在茫茫海量文章中你能刷新我，也你们工作顺利，蒸蒸日上！！

小果冻啦

关注

26
点赞
踩
13

收藏

觉得还不错? 一键收藏
0
评论
项目：ELKB日志采集系统部署实现（arm环境）

Logstash的数据处理过程主要包括：Inputs、Filters、Outputs 三部分，另外在Inputs和Outputs中可以使用Codecs对数据格式进行处理。这四个部分均以插件形式存在，用户通过定义pipeline配置文件，设置需要使用input,filter,output,codec插件，以实现特定的数据采集，数据处理，数据输出等功能Inputs：用于从数据源获取数据，常见的插件如 file, syslog, redis, beats 等。
复制链接

扫一扫