filebeat将nginx的数据抽取到推送到logstash的beats插件;
logstash将filebeat的输入推送到kafka
1.kafka环境准备
tar xvf kafka_2.12-2.7.0.tgz
[esadmin@oracle1 soft]$ ln -sv kafka_2.12-2.7.0 kafka
‘kafka’ -> ‘kafka_2.12-2.7.0’
2.检查Logstash 插件
[esadmin@oracle1 app]$ ./bin/logstash-plugin list --verbose |grep kafka
2024-06-30T17:19:54.605+08:00 [main] WARN FilenoUtil : Native subprocess control requires open access to the JDK IO subsystem
Pass '--add-opens java.base/sun.nio.ch=org.jruby.dist --add-opens java.base/java.io=org.jruby.dist' to enable.
logstash-integration-kafka (10.4.0)
├── logstash-input-kafka
└── logstash-output-kafka
3.kafka配置
mkdir -p /esdb/soft/kafka/logs
vi /esdb/soft/kafka/config/server.properties
log.dirs=/esdb/soft/kafka/logs/kafka-0-logs
num.partitions=16
host.name=192.168.1.7
advertised.host.name=192.168.1.7
listeners=PLAINTEXT://192.168.1.7:9092
advertised.listeners=PLAINTEXT://192.168.1.7:9092
socket.send.buffer.bytes=1024000
socket.receive.buffer.bytes=1024000
socket.request.max.bytes=1048576000
partition.assignment.strategy=roundrobin
num.network.threads=28
num.io.threads=28
4.
--zookeeper安装。zookeeper是kafka安装包自带的,伪集群。
mkdir -p /esdb/soft/kafka/zookeeper
vi /esdb/soft/kafka/config/zookeeper.properties
dataDir /esdb/soft/kafka/zookeeper
--启动zookeeper
nohup /esdb/soft/kafka/bin/zookeeper-server-start.sh /esdb/soft/kafka/config/zookeeper.properties &
--启动kafka;
nohup /esdb/soft/kafka/bin/kafka-server-start.sh /esdb/soft/kafka/config/server.properties &
5.配置nginx-to-kafka
vi /esdb/logstash/app/config/pipelines/nginx-to-kakfa.conf
input {
beats {
port => "5044"
codec => "json"
client_inactivity_timeout => 36000
}
}
output {
if "sspu-nginx" in [tags] {
kafka {
bootstrap_servers => "192.168.1.7:9092"
topic_id => "sspu-nginx"
codec => "json"
}
}
}
vi /esdb/logstash/app/config/pipelines.yml
- pipeline.id: nginx-to-kafka
pipeline.batch.size: 3000
pipeline.batch.delay: 5
queue.type: persisted
path.queue: /esdb/logstash/data
queue.page_capacity: 250mb
queue.max_bytes: 1024mb
queue.checkpoint.acks: 1024
queue.checkpoint.writes: 1024
queue.checkpoint.interval: 1000
pipeline.workers: 2
path.config: "/esdb/logstash/app/config/pipelines/nginx-to-kakfa.conf"
--启动Logstash;
nohup /esdb/logstash/app/bin/logstash &
7.配置filebeat抽取nginx的数据到logstash;
vi /esdb/filebeat-nginx/sspu-nginx.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /usr/local/nginx/logs/access-json.log
tags: ["sspu-nginx"]
fields_under_root: true
backoff: "1s"
tail_files: false
output.logstash:
hosts: ["192.168.1.7:5044"]
loadbalance: true
worker: 2
compression_level: 3
[2024-07-01T09:47:08,026][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.9.2", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc Java HotSpot(TM) 64-Bit Server VM 17.0.11+7-LTS-207 on 17.0.11+7-LTS-207 +indy +jit [linux-x86_64]"}
[2024-07-01T09:47:11,632][INFO ][org.reflections.Reflections] Reflections took 71 ms to scan 1 urls, producing 22 keys and 45 values
[2024-07-01T09:47:15,363][INFO ][org.apache.kafka.clients.producer.ProducerConfig][sspu-nginx] ProducerConfig values:
acks = 1
batch.size = 16384
bootstrap.servers = [192.168.1.7:9092]
buffer.memory = 33554432
client.dns.lookup = default
client.id =
compression.type = none
connections.max.idle.ms = 540000
delivery.timeout.ms = 120000
enable.idempotence = false
interceptor.classes = []
key.serializer = class org.apache.kafka.common.serialization.StringSerializer
linger.ms = 0
max.block.ms = 60000
max.in.flight.requests.per.connection = 5
max.request.size = 1048576
metadata.max.age.ms = 300000
metric.reporters = []
metrics.num.samples = 2
metrics.recording.level = INFO
metrics.sample.window.ms = 30000
partitioner.class = class org.apache.kafka.clients.producer.internals.DefaultPartitioner
receive.buffer.bytes = 32768
reconnect.backoff.max.ms = 50
reconnect.backoff.ms = 50
request.timeout.ms = 40000
retries = 2147483647
retry.backoff.ms = 100
sasl.client.callback.handler.class = null
sasl.jaas.config = null
sasl.kerberos.kinit.cmd = /usr/bin/kinit
sasl.kerberos.min.time.before.relogin = 60000
sasl.kerberos.service.name = null
sasl.kerberos.ticket.renew.jitter = 0.05
sasl.kerberos.ticket.renew.window.factor = 0.8
sasl.login.callback.handler.class = null
sasl.login.class = null
sasl.login.refresh.buffer.seconds = 300
sasl.login.refresh.min.period.seconds = 60
sasl.login.refresh.window.factor = 0.8
sasl.login.refresh.window.jitter = 0.05
sasl.mechanism = GSSAPI
security.protocol = PLAINTEXT
security.providers = null
send.buffer.bytes = 131072
ssl.cipher.suites = null
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
ssl.endpoint.identification.algorithm = https
ssl.key.password = null
ssl.keymanager.algorithm = SunX509
ssl.keystore.location = null
ssl.keystore.password = null
ssl.keystore.type = JKS
ssl.protocol = TLS
ssl.provider = null
ssl.secure.random.implementation = null
ssl.trustmanager.algorithm = PKIX
ssl.truststore.location = null
ssl.truststore.password = null
ssl.truststore.type = JKS
transaction.timeout.ms = 60000
transactional.id = null
value.serializer = class org.apache.kafka.common.serialization.StringSerializer
logstash需要root用户启动。
[2024-07-01T09:47:15,455][INFO ][org.apache.kafka.common.utils.AppInfoParser][sspu-nginx] Kafka version: 2.4.1
[2024-07-01T09:47:15,456][INFO ][org.apache.kafka.common.utils.AppInfoParser][sspu-nginx] Kafka commitId: c57222ae8cd7866b
[2024-07-01T09:47:15,457][INFO ][org.apache.kafka.common.utils.AppInfoParser][sspu-nginx] Kafka startTimeMs: 1719798435452
[2024-07-01T09:47:15,999][INFO ][logstash.javapipeline ][sspu-nginx] Starting pipeline {:pipeline_id=>"sspu-nginx", "pipeline.workers"=>2, "pipeline.batch.size"=>3000, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>6000, "pipeline.sources"=>["/esdb/logstash/app/config/pipelines/nginx-to-kakfa.conf"], :thread=>"#<Thread:0x60574704 run>"}
[2024-07-01T09:47:16,181][INFO ][org.apache.kafka.clients.Metadata][sspu-nginx] [Producer clientId=producer-1] Cluster ID: XcNNpGCORkSwUpBjVjteVQ
[2024-07-01T09:47:17,067][INFO ][logstash.javapipeline ][sspu-nginx] Pipeline Java execution initialization time {"seconds"=>1.06}
[2024-07-01T09:47:17,096][INFO ][logstash.inputs.beats ][sspu-nginx] Beats inputs: Starting input listener {:address=>"0.0.0.0:5044"}
[2024-07-01T09:47:17,132][INFO ][logstash.javapipeline ][sspu-nginx] Pipeline started {"pipeline.id"=>"sspu-nginx"}
[2024-07-01T09:47:17,297][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:"sspu-nginx"], :non_running_pipelines=>[]}
[2024-07-01T09:47:17,342][INFO ][org.logstash.beats.Server][sspu-nginx][e1f78b4ba9c376953b8cb2626ceb582a819cb4df6c94774e4efdfc5ef14aaff5] Starting server on port: 5044
[2024-07-01T09:47:17,729][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
--logstash启动报错:这个报错可以忽略,不影响数据的同步。
[2024-07-01T10:24:19,174][INFO ][org.apache.kafka.clients.Metadata][nginx-to-kafka] [Producer clientId=producer-1] Cluster ID: XcNNpGCORkSwUpBjVjteVQ
Sending Logstash logs to /esdb/logstash/logs which is now configured via log4j2.properties
2024-07-01 10:29:15,844 main ERROR No ScriptEngine found for language JavaScript. Available languages are: ruby, jruby
2024-07-01 10:29:15,869 main ERROR No ScriptEngine found for language JavaScript. Available languages are: ruby, jruby
2024-07-01 10:29:16,045 main ERROR No ScriptEngine found for language JavaScript. Available languages are: ruby, jruby
vi /esdb/logstash/app/config/logstash.yml
---放开这两个值。明确定义Logstash的端口。
http.host: 192.168.1.7
http.port: 9600
8.启动filebeat;
启动filebeat;
nohup /esdb/filebeat-nginx/filebeat -e -c /esdb/filebeat-nginx/sspu-nginx.yml &
[root@oracle1 ~]# jps
72865 Kafka
45909 Elasticsearch
72486 QuorumPeerMain (kafka-zookeeper)
88248 Logstash
46155 Elasticsearch
88669 Jps
62557 Bootstrap (tomcat)
46351 Elasticsearch
/esdb/soft/kafka/bin/kafka-topics.sh --list --zookeeper 192.168.1.7:2181
__consumer_offsets
10.nginx写入数据
[root@oracle1 config]# ab -n10000 -c2 http://192.168.1.7/
This is ApacheBench, Version 2.3 <$Revision: 1430300 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/
Benchmarking 192.168.1.7 (be patient)
Completed 1000 requests
Completed 2000 requests
Completed 3000 requests
Completed 4000 requests
Completed 5000 requests
Completed 6000 requests
Completed 7000 requests
Completed 8000 requests
Completed 9000 requests
Completed 10000 requests
Finished 10000 requests
Server Software: nginx/1.20.2
Server Hostname: 192.168.1.7
Server Port: 80
Document Path: /
Document Length: 153 bytes
Concurrency Level: 2
Time taken for tests: 3.527 seconds
Complete requests: 10000
Failed requests: 0
Write errors: 0
Non-2xx responses: 10000
Total transferred: 3030000 bytes
HTML transferred: 1530000 bytes
Requests per second: 2834.93 [#/sec] (mean)
Time per request: 0.705 [ms] (mean)
Time per request: 0.353 [ms] (mean, across all concurrent requests)
Transfer rate: 838.85 [Kbytes/sec] received
Connection Times (ms)
min mean[+/-sd] median max
Connect: 0 0 0.2 0 6
Processing: 0 1 1.1 0 65
Waiting: 0 0 1.1 0 65
Total: 0 1 1.2 0 66
Percentage of the requests served within a certain time (ms)
50% 0
66% 0
75% 1
80% 1
90% 1
95% 2
98% 2
99% 3
100% 66 (longest request)
11.kafka查看数据
[root@oracle1 config]# /esdb/soft/kafka/bin/kafka-topics.sh --list --zookeeper 192.168.1.7:2181
__consumer_offsets
sspu-nginx
--有16个分区。
[root@oracle1 kafka-0-logs]# pwd
/esdb/soft/kafka/logs/kafka-0-logs
[root@oracle1 kafka-0-logs]# ll |grep sspu-nginx
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-0
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-1
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-10
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-11
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-12
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-13
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-14
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-15
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-2
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-3
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-4
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-5
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-6
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-7
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-8
drwxrwxr-x 2 esadmin esadmin 141 Jul 1 11:12 sspu-nginx-9