kafka集群

kafka集群

一.zookeeper集群

1.必备包

jdk-8u144-linux-x64.rpm

zookeeper-3.4.9.tar.gz

 

2.zk部署

ln -s /application/zookeeper-3.4.9 /usr/local/zookeeper

cd /usr/local/zookeeper/conf

cp zoo_sample.cfg zoo.cfg

[root@hostname conf]# cat zoo.cfg

# The number of milliseconds of each tick

tickTime=2000

# The number of ticks that the initial

# synchronization phase can take

initLimit=10

# The number of ticks that can pass between

# sending a request and getting an acknowledgement

syncLimit=5

# the directory where the snapshot is stored.

# do not use /tmp for storage, /tmp here is just

# example sakes.

dataDir=/data/zookeeper

#dataDir=/application/data/zookeeper

# the port at which the clients will connect

clientPort=2181

# the maximum number of client connections.

# increase this if you need to handle more clients

#maxClientCnxns=60

#

# Be sure to read the maintenance section of the

# administrator guide before turning on autopurge.

#

# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance

#

# The number of snapshots to retain in dataDir

#autopurge.snapRetainCount=3

# Purge task interval in hours

# Set to "0" to disable auto purge feature

#autopurge.purgeInterval=1

server.1=IP:2888:3888

#IP换成hostname可能会出问题

 

tickTime:心跳时间

dataDir=/application/zookeeper/data

 

在zoo.cfg最后添加

server.1=ip:2888:3888

server.2=ip:2888:3888

server.3=ip:2888:3888

#IP换成hostname可能会出问题

Server.A=B:C:D

A:是一个数字，表示第几号服务器

B:服务器IP，#IP换成hostname可能会出问题

C:集群中Leader服务器交换信息的端口

D:Leader挂了以后重新选举Leader的端口

 

分别在3台服务器上

echo “1” > /application/zookeeper/myid

echo “2” > /application/zookeeper/myid

echo “3” > /application/zookeeper/myid

#myid文件就是dataDir指定的位置

 

vi   %zookeeper%/zkServer.sh

JVMPAPRM="-Xms1G -Xmx4G -Xmn2G"

该选项要提供内存信息由开发提供参数

 

alias zkServer.sh="/usr/local/zookeeper/bin/zkServer.sh"

 

zk简单总结:

jdk-8u144-linux-x64.rpm

zoo.cfg

server.1=ip:2888:3888

server.2=ip:2888:3888

server.3=ip:2888:3888

dataDir=/application/zookeeper/data

 

分别在3台服务器上

echo “1” > /application/zookeeper/myid

echo “2” > /application/zookeeper/myid

echo “3” > /application/zookeeper/myid

#myid文件就是dataDir指定的位置

 

alias zkServer.sh="/usr/local/zookeeper/bin/zkServer.sh"

 

 

二.kafka集群

vi server.properties

broker.id=0

#broker.id的值需要改,可以改成IP为尾号

 

Log.dirs=

#Log.dirs存储topic相关信息

 

num.partitions=1

#不修改

 

zookeeper.connect=ip:port

#必须写IP地址

#例:172.17.82.68:2181,172.17.82.69:2181,172.17.82.70:2181

#例:172.17.82.68:2181,172.17.82.69:2181,172.17.82.70:2181/kafkagroup

#例:172.17.82.68:2181,172.17.82.69:2181,172.17.82.70:2181/abcdefghijk

#/kafkagroup 或 /任意字母代表./zkCli.sh下执行ls / 时，会出现一个zookeeper文件夹，用于存储kafka信息

Zookeeper.connection.timeout.ms=6000

#6000=6秒，等待链接6秒后失效

三．开机自动启动

 

vi /etc/init.d/autozkfka.sh

内容如下

#!/bin/bash

# chkconfig: 2345 10 90

# description: autozkfka.sh

/usr/local/zookeeper/bin/zkServer.sh start /usr/local/zookeeper/conf/zoo.cfg

/application/kafka/bin/kafka-server-start.sh /application/kafka/config/zookeeper.properties

 

chmod +x /etc/init.d/autozkfka.sh

 

chkconfig --add /etc/init.d/autozkfka.sh

chkconfig autozkfka.sh on

vi %kafka%/bin/kafka-server-start.sh

 

 

三.Kafka-manager

nohup ./kafka-manager &

or

echo "nohup /usr/local/kafka-manager-1.3.3.7/bin/kafka-manager -Dconfig.file=/usr/local/kafka-manager-1.3.3.7/conf/application.conf -Dhttp.port=8090 &" >> /etc/rc.local

转载于:https://my.oschina.net/u/3125863/blog/1817569