大数据组件配置

Zookeeper环境配置

[root@hadoop122 ~]# cd /opt/module/zookeeper-3.4.10/conf
[root@hadoop122 conf]# mv zoo_sample.cfg  zoo.cfg 
[root@hadoop122 conf]# vim zoo.cfg 
dataDir=/opt/module/zookeeper-3.4.10/zkData
[root@hadoop122 zookeeper-3.4.10]# mkdir zkData
[root@hadoop122 zookeeper-3.4.10]# cd zkData/
[root@hadoop122 zkData]# vim myid
2

# 分发到hadoop123,hadoop124
[root@hadoop122 module]# xsync zookeeper-3.4.10

#修改myid
[root@hadoop123 zookeeper-3.4.10]# cd zkData/
[root@hadoop123 zkData]# vim myid 
3
#修改myid
[root@hadoop124 zookeeper-3.4.10]# cd zkData/
[root@hadoop123 zkData]# vim myid 
4

# 这个是standlone模式
[root@hadoop122 zookeeper-3.4.10]# bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /opt/module/zookeeper-3.4.10/bin/../conf/zoo.cfg
Mode: standalone


# 改为cluster模式
[root@hadoop122 zookeeper-3.4.10]# vim conf/zoo.cfg 
#######################cluster##########################
server.2=hadoop102:2888:3888
server.3=hadoop103:2888:3888
server.4=hadoop104:2888:3888

# 同步
[root@hadoop122 zookeeper-3.4.10]# xsync conf/zoo.cfg 

编写脚本控制Zookeeper服务的开启和关闭

#!/bin/bash
case $1 in
"start") {
	for i in hadoop122 hadoop123 hadoop124
	do
		ssh $i "source /etc/profile; /opt/module/zookeeper-3.4.10/bin/zkServer.sh start"
	done
};;
"stop") {
	for i in hadoop122 hadoop123 hadoop124
	do
		ssh $i "source /etc/profile; /opt/module/zookeeper-3.4.10/bin/zkServer.sh stop"
	done
};;
"status") {
	for i in hadoop122 hadoop123 hadoop124
	do
		ssh $i "source /etc/profile; /opt/module/zookeeper-3.4.10/bin/zkServer.sh status"
	done
};;
esac

Kafka相关配置

Kafka安装

1.安装kafka，并解压

[root@hadoop122 module]# mv kafka_2.11-0.11.0.0/ kafka

2.修改配置文件

[root@hadoop122 kafka]# cd config/
[root@hadoop122 config]# vim server.properties
#broker 的全局唯一编号，不能重复
broker.id=0

#删除 topic 功能使能
delete.topic.enable=true

#kafka 运行日志存放的路径
log.dirs=/opt/module/kafka/logs

#配置连接 Zookeeper 集群地址
zookeeper.connect=hadoop102:2181,hadoop103:2181,hadoop104:2181

3.配置环境变量

[root@hadoop122 config]# vim /etc/profile
#KAFKA_HOME
export KAFKA_HOME=/opt/module/kafka
export PATH=$PATH:$KAFKA_HOME/bin
[root@hadoop122 config]$ source /etc/profile

4.分发安装包

[root@hadoop122 module]# xsync kafka/
[root@hadoop122 module]# xsync /etc/profile

分别在 hadoop123 和 hadoop124 上修改配置文件/opt/module/kafka/config/server.properties中的 broker.id=1、broker.id=2 ,刷新环境变量， source /etc/prf4ofile, 注：broker.id 不得重复

5.kafka 群起脚本

#! /bin/bash
case $1 in
"start"){
	for i in hadoop122 hadoop123 hadoop124
	do
		echo " --------启动 $i Kafka-------"
		ssh $i "source /etc/profile; /opt/module/kafka/bin/kafka-server-start.sh -daemon /opt/module/kafka/config/server.properties "
	done
};;
"stop"){
	for i in hadoop122 hadoop123 hadoop124
	do
		echo " --------停止 $i Kafka-------"
		ssh $i "source /etc/profile; /opt/module/kafka/bin/kafka-server-stop.sh stop"
	done
};;
esac

6.查看是否启动成功

[root@hadoop122 bin]# kf.sh start
 --------启动 hadoop122 Kafka-------
 --------启动 hadoop123 Kafka-------
 --------启动 hadoop124 Kafka-------
[root@hadoop122 bin]# xcall.sh jps
================== hadoop122 ==================
2848 Jps
2770 Kafka
2067 QuorumPeerMain
1468 NameNode
1597 DataNode
================== hadoop123 ==================
1361 DataNode
2291 Jps
1541 QuorumPeerMain
2221 Kafka
================== hadoop124 ==================
1649 QuorumPeerMain
1362 DataNode
2322 Kafka
2392 Jps
1464 SecondaryNameNode

Kafka 命令行操作

创建 topic

[root@hadoop122 kafka]# bin/kafka-topics.sh --zookeeper hadoop122:2181 --create --topic first --replication-factor 3 --partitions 1
Created topic "first".

# 选项说明：
--topic 定义 topic 名
--replication-factor 定义副本数
--partitions 定义分区数

查看topic

[root@hadoop122 kafka]# bin/kafka-topics.sh --zookeeper hadoop122:2181 --list
first

删除topic

[root@hadoop122 kafka]# bin/kafka-topics.sh --zookeeper hadoop122:2181 --delete --topic first
Topic first is marked for deletion.
Note: This will have no impact if delete.topic.enable is not set to true.

[root@hadoop122 kafka]# bin/kafka-topics.sh --zookeeper hadoop122:2181 --list

# 需要 server.properties 中设置 delete.topic.enable=true 否则只是标记删除

生产者产生消息

[root@hadoop122 kafka]# bin/kafka-console-producer.sh --broker-list hadoop122:9092 --topic hello
>hello

消费者消费消息

# 方式一：旧的
[root@hadoop122 kafka]# bin/kafka-console-consumer.sh --zookeeper hadoop122:2181 --topic hello --from-beginning
hello

# 方式二：推荐
[root@hadoop122 kafka]# bin/kafka-console-consumer.sh --bootstrap-server hadoop122:9092 --topic hello --from-beginning
hello

看某个 Topic 的详情

[root@hadoop122 kafka]# bin/kafka-topics.sh  --zookeeper hadoop122:2181 --describe --topic hello

修改分区数

[root@hadoop122 kafka]# bin/kafka-topics.sh  --zookeeper hadoop102:2181 --alter --topic hello --partitions 6