时间同步:
yum -y install ntpdate
ntpdate ntp1.aliyun.com
安装java:
yum install java-1.8.0-openjdk -y
部署ES
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.4.1.rpm
rpm -ivh elasticsearch-6.4.1.rpm
##安装插件插件
sudo bin/elasticsearch-plugin install ingest-geoip
sudo bin/elasticsearch-plugin install ingest-user-agent
#修改es配置文件
[root@elk-01 ~]# grep '^[a-Z]' /etc/elasticsearch/elasticsearch.yml
cluster.name: olda-cluster #集群名称
node.name: elk-01 #节点名称
node.master: true #当前节点为集群master
node.data: true #当前为数据节点
path.data: /var/lib/elasticsearch #数据存放位置
path.logs: /var/log/elasticsearch #日志存放位置
network.host: 10.0.0.10 #监听的IP地址
http.port: 9200 #监听的端口
discovery.zen.ping.unicast.hosts: ["elk-01"] #集群节点的主机名或IP地址
启动ES服务:
systemctl start elasticsearch.service
#开机自动启动
systemctl enable elasticsearch.service
#查看集群状态:
curl '10.0.0.10:9200/_cluster/health?pretty'
#如果状态是yellow
curl -H "Content-Type: application/json" -XPUT 'http://10.0.0.10:9200/_settings' -d '
{
"index" : {
"number_of_replicas" : 0
}
}'
添加ES Pipeline Grok:
对日志格式解析,代替logstash
##日志格式:
##error 2020-11-23 20:02:03 /php/phplib/yii/vendor/yiisoft/yii2/BaseYii.php 401 logid=d1fa0143d89b72e1adec09c7488cdcda app=resource caller_url=/audio/historywords/ user_ip=192.168.33.1 local_ip=192.168.33.10 service_name= port=8080 method=GET errorno=0 interface=/audio/historywords/ msg=errorcode:500 errormsg:Page not found.
PUT _ingest/pipeline/app-resource
{
"description" : "app-resource",
"processors" : [
{
"grok": {
"field": "message",
"patterns": [
"""(?<log_level>[A-Za-z]+) (?<time>[1-9]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])\s+(20|21|22|23|[0-1]\d):[0-5]\d:[0-5]\d) %{UNIXPATH:path} %{NUMBER:err_line_number} logid=%{WORD:logid} app=%{WORD:app_name} caller_url=%{URIPATHPARAM:caller_url} user_ip=%{IPV4:user_ip} local_ip=%{IPV4:local_ip} service_name=(?<service_name>(\s|[\w]+)) port=%{NUMBER:port} method=(?<method>(\s|[\w]+)) errorno=%{NUMBER:errorno} interface=%{URIPATHPARAM:interface} msg=(?<msg>(\s|.+))"""
]
}
},
{
####转换时间字段格式
"date" : {
"ignore_failure" : true,
"field" : "time",
"formats" : [
"dd/MMM/yyyy:HH:mm:ss Z"
],
"timezone" : "Asia/Shanghai"
}
}
]
}
gork函数详解:gork表达式大全
部署KA:
下载并安装源码包:
wget https://artifacts.elastic.co/downloads/kibana/kibana-6.4.1-x86_64.rpm
rpm -ivh kibana-6.4.1-x86_64.rpm
KA配置:
#默认日志存放在系统日志
grep '^[a-Z]' /etc/kibana/kibana.yml
server.port: 5601 #kibana监听的端口
server.host: "10.0.0.10" #kibana监听的IP地址
elasticsearch.url: "http://10.0.0.10:9200" #kibana连接ES集群master节点地址
启动KA:
systemctl start kibana.service
访问测试
部署FB:
下载并安装源码包:
https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.4.1-x86_64.rpm
rpm -ivh filebeat-6.4.1-x86_64.rpm
FB配置:
如果使用filebeat内置的模版,开启相应的模块即可,这里使用自定义模版
更多内容见:filebeat模块与配置
#=========================== Filebeat inputs =============================
filebeat.inputs:
- type: log
enabled: true
paths:
- /home/work/dsp/log/app/resource/resource.log.wf
fields:
type: "app-resource-log"
close_renamed: true
close_removed: true
scan_frequency: 10s
#=========================== Filebeat inputs =============================
filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml
# Set to true to enable config reloading
reload.enabled: false
# Period on which files under path should be checked for changes
#reload.period: 10s
#==================== Elasticsearch template setting ==========================
setup.ilm.enabled: false
#自定义模版,在ES中创建索引时的规则
#setup.template.name: "php-nginx" # 模版名称
#setup.template.pattern: "php-nginx-*" #模版匹配规则 output.elasticsearch-indices-index匹配
#setup.template.settings: # 配置生成索引的分片与副本数
# index.number_of_shards: 3
# index.number_of_replicas: 1
#setup.template.overwrite: true
#setup.template.enabled: true
setup.template.name: "app-resource"
setup.template.pattern: "app-resource-*"
setup.template.settings:
index.number_of_shards: 3
index.number_of_replicas: 0
setup.template.overwrite: true
setup.template.enabled: true
#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
# Array of hosts to connect to.
hosts: ["localhost:9200"]
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
index: "php-log-%{+yyyy.MM.dd}"
indices:
#- index: "php-nginx-%{+yyyy.MM.dd}"
# when.equals:
# fields.type: "php-nginx-access"
- index: "app-resource-%{+yyyy.MM.dd}"#对应ES索引名称并匹配刚才创建的模版
when.equals:
fields.type: "app-resource-log"
pipelines:
#- pipeline: "php-nginx-access"
# when.equals:
# fields.type: "php-nginx-access"
- pipeline: "app-resource"#对应ES创建的grok名称
when.equals:
fields.type: "app-resource-log"
FB调试:
#输出filebeat debug信息
filebeat -e
#FB日志
tail -f /var/log/filebeat/filebeat
#查看FB是否收集日志
curl '10.0.0.10:9200/_cat/indices?v'
见证奇迹的时刻
可以根据pipeline中预处理的字段进行搜索