kafka+zookeeper + logstash 安装

最新推荐文章于 2023-05-19 00:26:40 发布

weixin_34192993

最新推荐文章于 2023-05-19 00:26:40 发布

阅读量400

点赞数

文章标签：大数据开发工具 python

原文链接：https://my.oschina.net/zhongwenhao/blog/3049358

版权

2019独角兽企业重金招聘Python工程师标准>>>


[root@swarm02 data]# tar -zxvf kafka_2.12-2.2.0.tgz 

[root@swarm02 data]# mv kafka_2.12-2.2.0 kafka
[root@swarm02 data]# ll
total 55696
drwxr-xr-x 6 elk  elk      4096 May  5 03:47 elk
drwxr-xr-x 6 root root       83 Mar  9 14:46 kafka
-rw-r--r-- 1 root root 57028557 May  6 10:02 kafka_2.12-2.2.0.tgz
drwxr-xr-x 3 elk  elk        78 Apr 30 04:28 skyworking
[root@swarm02 data]# 


[root@swarm02 kafka]# pwd
/data/kafka

[root@swarm02 kafka]# mkdir logs

vim /config/server.properties

log.dirs=/data/kafka/logs
zookeeper.connect=localhost:2181

启动
/data/kafka/bin/kafka-server-start.sh /data/kafka/config/server.properties &
停止
/data/kafka/bin/kafka-server-stop.sh



搭建zookeeper
[root@swarm02 data]# tar -zxvf zookeeper-3.4.6.tar.gz
[root@swarm02 data]# mv zookeeper-3.4.6 zookeeper


进入Zookeeper的config目录下
拷贝zoo_sample.cfg文件重命名为zoo.cfg，然后修改dataDir属性


[root@swarm02 data]# mv zookeeper-3.4.6 zookeeper
[root@swarm02 data]# ll
total 72988
drwxr-xr-x  6 elk  elk      4096 May  5 03:47 elk
drwxr-xr-x  7 root root       94 May  6 10:08 kafka
-rw-r--r--  1 root root 57028557 May  6 10:02 kafka_2.12-2.2.0.tgz
drwxr-xr-x  3 elk  elk        78 Apr 30 04:28 skyworking
drwxr-xr-x 10 elk  elk      4096 Feb 20  2014 zookeeper
-rw-r--r--  1 root root 17699306 May  6 10:15 zookeeper-3.4.6.tar.gz
[root@swarm02 data]# 



[root@swarm02 data]# cd zookeeper/


[root@swarm02 zookeeper]# mkdir data
[root@swarm02 zookeeper]# mkdir log
[root@swarm02 zookeeper]# ll
total 1532
drwxr-xr-x  2 elk  elk      141 Feb 20  2014 bin
-rw-rw-r--  1 elk  elk    82446 Feb 20  2014 build.xml
-rw-rw-r--  1 elk  elk    80776 Feb 20  2014 CHANGES.txt
drwxr-xr-x  2 elk  elk       88 May  6 10:31 conf
drwxr-xr-x 10 elk  elk      122 Feb 20  2014 contrib
drwxr-xr-x  2 root root       6 May  6 10:33 data
drwxr-xr-x  2 elk  elk     4096 Feb 20  2014 dist-maven
drwxr-xr-x  6 elk  elk     4096 Feb 20  2014 docs
-rw-rw-r--  1 elk  elk     1953 Feb 20  2014 ivysettings.xml
-rw-rw-r--  1 elk  elk     3375 Feb 20  2014 ivy.xml
drwxr-xr-x  4 elk  elk     4096 Feb 20  2014 lib
-rw-rw-r--  1 elk  elk    11358 Feb 20  2014 LICENSE.txt
drwxr-xr-x  2 root root       6 May  6 10:33 log
-rw-rw-r--  1 elk  elk      170 Feb 20  2014 NOTICE.txt
-rw-rw-r--  1 elk  elk     1770 Feb 20  2014 README_packaging.txt
-rw-rw-r--  1 elk  elk     1585 Feb 20  2014 README.txt
drwxr-xr-x  5 elk  elk       44 Feb 20  2014 recipes
drwxr-xr-x  8 elk  elk      145 Feb 20  2014 src
-rw-rw-r--  1 elk  elk  1340305 Feb 20  2014 zookeeper-3.4.6.jar
-rw-rw-r--  1 elk  elk      836 Feb 20  2014 zookeeper-3.4.6.jar.asc
-rw-rw-r--  1 elk  elk       33 Feb 20  2014 zookeeper-3.4.6.jar.md5
-rw-rw-r--  1 elk  elk       41 Feb 20  2014 zookeeper-3.4.6.jar.sha1
[root@swarm02 zookeeper]# 


[root@swarm02 zookeeper]# cd conf/
[root@swarm02 conf]# ll
total 12
-rw-rw-r-- 1 elk elk  535 Feb 20  2014 configuration.xsl
-rw-rw-r-- 1 elk elk 2161 Feb 20  2014 log4j.properties
-rw-rw-r-- 1 elk elk  922 Feb 20  2014 zoo_sample.cfg
[root@swarm02 conf]# cp zoo_sample.cfg zoo.cfg
[root@swarm02 conf]# ll
total 16
-rw-rw-r-- 1 elk  elk   535 Feb 20  2014 configuration.xsl
-rw-rw-r-- 1 elk  elk  2161 Feb 20  2014 log4j.properties
-rw-r--r-- 1 root root  922 May  6 10:28 zoo.cfg
-rw-rw-r-- 1 elk  elk   922 Feb 20  2014 zoo_sample.cfg
[root@swarm02 conf]# 

[root@swarm02 conf]# vim zoo.cfg
# 数据的存放目录
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/log
# 端口，默认就是2181
clientPort=2181


/data/zookeeper/bin/zkServer.sh start
/data/zookeeper/bin/zkServer.sh stop
/data/zookeeper/bin/zkServer.sh restart
/data/zookeeper/bin/zkServer.sh status





netstat -tunlp|egrep "(2181|9092)"






[root@swarm02 config]# vim  logstash-kaka.conf

input {
     kafka {
        topics_pattern => "cetc-.*"    
        consumer_threads => 10
        decorate_events => true
        auto_offset_reset => "latest"
        bootstrap_servers => "10.0.14.29:9092"
        codec => "json"
    }
}

output {
  stdout {  codec => rubydebug }
  elasticsearch {
    hosts => [ "10.0.14.29:9200" ]
    index => "%{[@metadata][kafka][topic]}"
  }

}







/data/elk/logstash/bin/logstash -f /data/elk/logstash/config/logstash-kaka.conf

nohup /data/elk/logstash/bin/logstash -f /data/elk/logstash/config/logstash-kafka.conf &

nohup /data/elk/logstash/bin/logstash -f /data/elk/logstash/config/logstash-kafka.conf &


Sending Logstash logs to /data/elk/logstash/logs which is now configured via log4j2.properties
[2019-05-06T11:16:28,581][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2019-05-06T11:16:28,604][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"6.5.4"}
[2019-05-06T11:16:29,567][INFO ][logstash.config.source.local.configpathloader] No config files found in path {:path=>"/data/elk/logstash/conf/logstash-kaka.conf"}
[2019-05-06T11:16:29,577][ERROR][logstash.config.sourceloader] No configuration found in the configured sources.
[2019-05-06T11:16:29,966][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}



netstat -tunlp|egrep "(9600)"

[root@swarm02 ~]# /data/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --describe  --topic applog


http://192.168.20.101:9600

http://192.168.20.101:9600/_node

http://192.168.20.101:9600/_node/stats

http://192.168.20.101:9600/_node/hot_threads


# curl http://192.168.20.101:9600/_node/stats?pretty

API：Logstash提供了丰富的api来查看Logstash的当前状态

转载于:https://my.oschina.net/zhongwenhao/blog/3049358