[root@swarm02 data]# tar -zxvf kafka_2.12-2.2.0.tgz
[root@swarm02 data]# mv kafka_2.12-2.2.0 kafka
[root@swarm02 data]# ll
total 55696
drwxr-xr-x 6 elk elk 4096 May 5 03:47 elk
drwxr-xr-x 6 root root 83 Mar 9 14:46 kafka
-rw-r--r-- 1 root root 57028557 May 6 10:02 kafka_2.12-2.2.0.tgz
drwxr-xr-x 3 elk elk 78 Apr 30 04:28 skyworking
[root@swarm02 data]#
[root@swarm02 kafka]# pwd
/data/kafka
[root@swarm02 kafka]# mkdir logs
vim /config/server.properties
log.dirs=/data/kafka/logs
zookeeper.connect=localhost:2181
启动
/data/kafka/bin/kafka-server-start.sh /data/kafka/config/server.properties &
停止
/data/kafka/bin/kafka-server-stop.sh
搭建zookeeper
[root@swarm02 data]# tar -zxvf zookeeper-3.4.6.tar.gz
[root@swarm02 data]# mv zookeeper-3.4.6 zookeeper
进入Zookeeper的config目录下
拷贝zoo_sample.cfg文件重命名为zoo.cfg,然后修改dataDir属性
[root@swarm02 data]# mv zookeeper-3.4.6 zookeeper
[root@swarm02 data]# ll
total 72988
drwxr-xr-x 6 elk elk 4096 May 5 03:47 elk
drwxr-xr-x 7 root root 94 May 6 10:08 kafka
-rw-r--r-- 1 root root 57028557 May 6 10:02 kafka_2.12-2.2.0.tgz
drwxr-xr-x 3 elk elk 78 Apr 30 04:28 skyworking
drwxr-xr-x 10 elk elk 4096 Feb 20 2014 zookeeper
-rw-r--r-- 1 root root 17699306 May 6 10:15 zookeeper-3.4.6.tar.gz
[root@swarm02 data]#
[root@swarm02 data]# cd zookeeper/
[root@swarm02 zookeeper]# mkdir data
[root@swarm02 zookeeper]# mkdir log
[root@swarm02 zookeeper]# ll
total 1532
drwxr-xr-x 2 elk elk 141 Feb 20 2014 bin
-rw-rw-r-- 1 elk elk 82446 Feb 20 2014 build.xml
-rw-rw-r-- 1 elk elk 80776 Feb 20 2014 CHANGES.txt
drwxr-xr-x 2 elk elk 88 May 6 10:31 conf
drwxr-xr-x 10 elk elk 122 Feb 20 2014 contrib
drwxr-xr-x 2 root root 6 May 6 10:33 data
drwxr-xr-x 2 elk elk 4096 Feb 20 2014 dist-maven
drwxr-xr-x 6 elk elk 4096 Feb 20 2014 docs
-rw-rw-r-- 1 elk elk 1953 Feb 20 2014 ivysettings.xml
-rw-rw-r-- 1 elk elk 3375 Feb 20 2014 ivy.xml
drwxr-xr-x 4 elk elk 4096 Feb 20 2014 lib
-rw-rw-r-- 1 elk elk 11358 Feb 20 2014 LICENSE.txt
drwxr-xr-x 2 root root 6 May 6 10:33 log
-rw-rw-r-- 1 elk elk 170 Feb 20 2014 NOTICE.txt
-rw-rw-r-- 1 elk elk 1770 Feb 20 2014 README_packaging.txt
-rw-rw-r-- 1 elk elk 1585 Feb 20 2014 README.txt
drwxr-xr-x 5 elk elk 44 Feb 20 2014 recipes
drwxr-xr-x 8 elk elk 145 Feb 20 2014 src
-rw-rw-r-- 1 elk elk 1340305 Feb 20 2014 zookeeper-3.4.6.jar
-rw-rw-r-- 1 elk elk 836 Feb 20 2014 zookeeper-3.4.6.jar.asc
-rw-rw-r-- 1 elk elk 33 Feb 20 2014 zookeeper-3.4.6.jar.md5
-rw-rw-r-- 1 elk elk 41 Feb 20 2014 zookeeper-3.4.6.jar.sha1
[root@swarm02 zookeeper]#
[root@swarm02 zookeeper]# cd conf/
[root@swarm02 conf]# ll
total 12
-rw-rw-r-- 1 elk elk 535 Feb 20 2014 configuration.xsl
-rw-rw-r-- 1 elk elk 2161 Feb 20 2014 log4j.properties
-rw-rw-r-- 1 elk elk 922 Feb 20 2014 zoo_sample.cfg
[root@swarm02 conf]# cp zoo_sample.cfg zoo.cfg
[root@swarm02 conf]# ll
total 16
-rw-rw-r-- 1 elk elk 535 Feb 20 2014 configuration.xsl
-rw-rw-r-- 1 elk elk 2161 Feb 20 2014 log4j.properties
-rw-r--r-- 1 root root 922 May 6 10:28 zoo.cfg
-rw-rw-r-- 1 elk elk 922 Feb 20 2014 zoo_sample.cfg
[root@swarm02 conf]#
[root@swarm02 conf]# vim zoo.cfg
# 数据的存放目录
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/log
# 端口,默认就是2181
clientPort=2181
/data/zookeeper/bin/zkServer.sh start
/data/zookeeper/bin/zkServer.sh stop
/data/zookeeper/bin/zkServer.sh restart
/data/zookeeper/bin/zkServer.sh status
netstat -tunlp|egrep "(2181|9092)"
[root@swarm02 config]# vim logstash-kaka.conf
input {
kafka {
topics_pattern => "cetc-.*"
consumer_threads => 10
decorate_events => true
auto_offset_reset => "latest"
bootstrap_servers => "10.0.14.29:9092"
codec => "json"
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => [ "10.0.14.29:9200" ]
index => "%{[@metadata][kafka][topic]}"
}
}
/data/elk/logstash/bin/logstash -f /data/elk/logstash/config/logstash-kaka.conf
nohup /data/elk/logstash/bin/logstash -f /data/elk/logstash/config/logstash-kafka.conf &
nohup /data/elk/logstash/bin/logstash -f /data/elk/logstash/config/logstash-kafka.conf &
Sending Logstash logs to /data/elk/logstash/logs which is now configured via log4j2.properties
[2019-05-06T11:16:28,581][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2019-05-06T11:16:28,604][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"6.5.4"}
[2019-05-06T11:16:29,567][INFO ][logstash.config.source.local.configpathloader] No config files found in path {:path=>"/data/elk/logstash/conf/logstash-kaka.conf"}
[2019-05-06T11:16:29,577][ERROR][logstash.config.sourceloader] No configuration found in the configured sources.
[2019-05-06T11:16:29,966][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
netstat -tunlp|egrep "(9600)"
[root@swarm02 ~]# /data/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --describe --topic applog
http://192.168.20.101:9600
http://192.168.20.101:9600/_node
http://192.168.20.101:9600/_node/stats
http://192.168.20.101:9600/_node/hot_threads
# curl http://192.168.20.101:9600/_node/stats?pretty
API:Logstash提供了丰富的api来查看Logstash的当前状态