1. ZooKeeper
2. Kafka
2.1 解压安装
# 确保scala已经安装好,本文安装的是2.11.7
tar -xf kafka_2.11-0.9.0.1.tgz
cd kafka_2.11-0.9.0.1
mkdir logs
vim ~/.bash_profile
export KAFKA_HOME=/home/zkpk/kafka_2.11-0.9.0.1
export PATH=$PATH:$KAFKA_HOME/bin
source ~/.bash_profile
2.2 配置
2.2.1 server.properties
只设置了以下4项,其他使用默认值
# 当前机器在集群中的唯一标识,和zookeeper的myid性质一样
broker.id=0
host.name=hsm01
# 消息存放的目录,这个目录可以配置为“,”逗号分割的表达式,
# 上面的num.io.threads要大于这个目录的个数这个目录,如果配置多个目录,
# 新创建的topic他把消息持久化的地方是,当前以逗号分割的目录中,
# 那个分区数最少就放那一个
log.dirs=/home/zkpk/kafka_2.11-0.9.0.1/logs
# 配置自定义的ZooKeeper
zookeeper.connect=hsm01:2181,hss01:2181,hss02:2181/kafka
2.2.2 复制到其他节点
scp -r ~/kafka_2.11-0.9.0.1/ hss01:~/
scp -r ~/kafka_2.11-0.9.0.1/ hss02:~/
# 修改broker.id与host.name
# 配置环境变量
2.3 启动
kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties
2.4 测试
# 创建Topic
kafka-topics.sh --create --zookeeper hsm01:2181/kafka --replication-factor 1 --partitions 1 --topic shuaige
# 创建一个broker,发布者
kafka-console-producer.sh --broker-list hsm01:9092 --topic shuaige
# 创建一个订阅者
kafka-console-consumer.sh --zookeeper hsm01:2181/kafka --topic shuaige --from-beginning
# 查看主题
kafka-topics.sh --zookeeper hsm01:2181/kafka --list
# 查看主题详情
kafka-topics.sh --describe --zookeeper localhost:2181 --topic test
# 删除主题
kafka-topics.sh --zookeeper localhost:2181 --delete --topic test
2.5 参考
3. Flume
3.1 解压安装
# /home/zkpk目录
tar -xf apache-flume-1.6.0-bin.tar.gz
mv apache-flume-1.6.0-bin/ flume-1.6.0
# 配置环境变量
vim .bash_profile
export FLUME_HOME=/home/zkpk/flume-1.6.0
export PATH=$PATH:$FLUME_HOME/bin
3.2 配置(与kafka整合)
kafkasink只有在1.6.0以上的flume版本才有。
3.2.1 flume-env.sh
JAVA_HOME=/opt/jdk1.8.0_45
3.2.2 kafka-sogolog.properties
# configure agent
a1.sources = f1
a1.channels = c1
a1.sinks = k1
# configure the source
a1.sources.f1.type = netcat
a1.sources.f1.bind = localhost
a1.sources.f1.port = 3333
# configure the sink (kafka)
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.topic = sogolog
a1.sinks.k1.brokerList = hsm01:9092,hss01:9092/kafka
a1.sinks.k1.requiredAcks = 0
a1.sinks.k1.batchSize = 20
# configure the channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# bind the source and sink to the channel
a1.sources.f1.channels = c1
a1.sinks.k1.channel = c1
3.3 启动
启动ZooKeeper服务
$ZOOKEEPER_HOME/bin/zkServer.sh start
启动kafka
# 启动服务
kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties
# 创建Topic
kafka-topics.sh --create --zookeeper hsm01:2181/kafka --replication-factor 1 --partitions 1 --topic sogolog
# 创建一个订阅者
kafka-console-consumer.sh --zookeeper hsm01:2181/kafka --top