-
这些组件理想情况下使用同一版本,较为兼容
-
配置好要拉取文件
pwd
/var/log/linwei_all
ll
total 19420
-rw-r--r-- 1 root root 3307971 Nov 1 19:26 access.a.log
-rw-r--r-- 1 root root 3307971 Nov 1 14:54 access.b.log
-rw-r--r-- 1 root root 3307971 Nov 1 14:54 access.c.log
-rw-r--r-- 1 root root 3307971 Nov 1 14:54 access.d.log
-rw-r--r-- 1 root root 3307971 Nov 1 14:54 access.f.log
-rw-r--r-- 1 root root 3307971 Nov 1 14:54 access.log
nginx日志格式为
log_format main escape=default '$remote_addr -[$time_local] -[$host] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" request_time[$request_time] upstream_response_time[$upstream_response_time] upstream_addr[$upstream_addr] logId[$http_x_logid] imid[$http_imid] statistic[$statistic] Bfe_logid[$http_bfe_logid] CLIENTIP[$http_clientip] device_id[$cookie_device_id] X-Deviceid[$http_x_deviceid] route[$http_route] product[$http_product] subsys[$http_subsys]';
uninitialized_variable_warn off;
-
filebeat
-
配置文件
-
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/linwei_all/*.log
output.kafka:
hosts: ["172.31.9.52:9092"]
enabled: true
topic: test2
#####将filebeat的默认输出取消
codec.format:
string: "%{[message]}"
-
zookeeper
-
配置文件
pwd
/app/tools
egrep -v '^#|^$' server1/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/app/tools/data/zk1
dataLogDir=/app/tools/logs/zk1
clientPort=2181
maxClientCnxns=60
server.1=172.31.9.52:2888:3888
server.2=172.31.9.52:2889:3889
server.3=172.31.9.52:2890:3890
[root@bjkjy-bkm-neisou-spider.bjkjy.baidu.com tools]# egrep -v '^#|^$' server2/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/app/tools/data/zk2
dataLogDir=/app/tools/logs/zk2
clientPort=2182
maxClientCnxns=60
server.1=172.31.9.52:2888:3888
server.2=172.31.9.52:2889:3889
server.3=172.31.9.52:2890:3890
egrep -v '^#|^$' server3/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/app/tools/data/zk3
dataLogDir=/app/tools/logs/zk3
clientPort=2183
maxClientCnxns=60
server.1=172.31.9.52:2888:3888
server.2=172.31.9.52:2889:3889
server.3=172.31.9.52:2890:3890
-
kafka
-
配置文件
-
pwd
/app/tools/kafka/config
[root@bjkjy-bkm-neisou-spider.bjkjy.baidu.com config]# egrep -v '^#|^$' server.properties
broker.id=1
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=../data
num.partitions=3
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.31.9.52:2181,172.31.9.52:2182,172.31.9.52:2183
zookeeper.connection.timeout.ms=18000
-
-
查看topic主题
-
bin/kafka-topics.sh --bootstrap-server kafkahost:9092 --list
__consumer_offsets
kingmouse
logstash-kafka-topic
talking
test
test2
-
-
查看消费者
-
pwd
/app/tools/kafka/bin
[root@bjkjy-bkm-neisou-spider.bjkjy.baidu.com bin]# ./kafka-consumer-groups.sh --bootstrap-server kafkahost:9092 --describe --group kafkatest
GROUP TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG CONSUMER-ID HOST CLIENT-ID
kafkatest test2 0 66927 66927 0 kafkatest-0-4ff79b8c-4a60-4f1e-adbd-6790aef112e4 /172.31.9.52 kafkatest-0
kafkatest test2 1 66750 66750 0 kafkatest-0-4ff79b8c-4a60-4f1e-adbd-6790aef112e4 /172.31.9.52 kafkatest-0
kafkatest test2 2 66890 66890 0 kafkatest-0-4ff79b8c-4a60-4f1e-adbd-6790aef112e4 /172.31.9.52 kafkatest-0
-
logstash
-
jdk环境下载jdk11.大部分jdk版本不兼容
-
配置文件
-
cat logstash.conf
input{
kafka{
bootstrap_servers => "172.31.9.52:9092"
client_id => "kafkatest"
group_id => "kafkatest"
auto_offset_reset => "latest"
consumer_threads => "1"
decorate_events => true
topics => ["test2"]
}
# file{
# path => "/var/log/linwei_all/*.log"
# type => "log"
# }
}
filter {
grok {
patterns_dir => "/etc/logstash/grok/patterns"
##配置日志匹配规则
match => {
"message" => '%{IP:remote_addr} -\[%{HTTPDATE:time_local}\] -\[%{USER:host}\] %{QUOTEDSTRING:request} %{INT:status} %{INT:body_bytes_sent} %{QUOTEDSTRING:http_referer} %{QUOTEDSTRING:http_user_agent} %{QUOTEDSTRING:http_x_forwarded_for} request_time\[%{SECOND:request_time}\] upstream_response_time\[%{SECOND:upstream_response_time}\] upstream_addr\[%{DATA:upstream_addr}\] logId\[%{INT:http_x_logid}\] imid\[%{INT:http_imid}\] statistic\[%{DATA:statistic}\] Bfe_logid\[%{INT:http_bfe_logid}\] CLIENTIP\[%{IP:http_clientip}\] device_id\[%{DATA:cookie_device_id}\] X-Deviceid\[%{DATA:http_x_deviceid}\] route\[%{DATA:http_route}\] product\[%{DATA:http_product}\] subsys\[%{DATA:http_subsys}\]'
}
}
}
output {
elasticsearch {
hosts => "http://172.31.9.52:9200"
index => "logs_linwei_data"
timeout => 300
}
stdout {
codec=> rubydebug
}
}
-
es
-
配置文件
-
pwd
/etc/elasticsearch
cat elasticsearch.yml
node.name: node-1
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
#bootstrap.memory_lock: true
network.host: '0.0.0.0'
http.port: 9200
discovery.seed_hosts: ["172.31.9.52"]
cluster.initial_master_nodes: ["172.31.9.52"]
-
kibana
-
配置文件
-
pwd
/etc/kibana
cat kibana.yml
server.port: 8089
server.host: "172.31.9.52"
elasticsearch.hosts: [ "http://172.31.9.52:9200" ]
kibana.index: ".kibana"
logging.dest: /var/log/kibana/kibana.log
logging.quiet: false
i18n.locale: "zh-CN"