hadoop生态搭建(3节点)-08.kafka配置

如果之前没有安装jdk和zookeeper,安装了的请直接跳过

https://www.oracle.com/technetwork/java/javase/downloads/java-archive-javase8-2177648.html

# ==================================================================安装 jdk

mkdir -p /usr/java
tar -zxvf ~/jdk-8u111-linux-x64.tar.gz -C /usr/java
rm -r ~/jdk-8u111-linux-x64.tar.gz

http://archive.apache.org/dist/zookeeper/

# ==================================================================安装 zookeeper
# zookeeper集群搭建要至少3台服务器,服务器都要部署zookeeper

tar -zxvf ~/zookeeper-3.4.12.tar.gz -C /usr/local
rm -r ~/zookeeper-3.4.12.tar.gz

# https://www.scala-lang.org/download/2.12.4.html
# ==================================================================安装 scala

tar -zxvf ~/scala-2.12.4.tgz -C /usr/local
rm –r ~/scala-2.12.4.tgz

# http://archive.apache.org/dist/kafka/0.10.2.1/

# ==================================================================安装 kafka

tar -zxvf ~/kafka_2.12-0.10.2.1.tgz -C /usr/local
rm –r ~/kafka_2.12-0.10.2.1.tgz

# 环境变量
# ==================================================================node1 node2 node3

vi /etc/profile

# 在export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL下添加
export JAVA_HOME=/usr/java/jdk1.8.0_111
export ZOOKEEPER_HOME=/usr/local/zookeeper-3.4.12
export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.6
export MYSQL_HOME=/usr/local/mysql
export HBASE_HOME=/usr/local/hbase-1.2.4
export HIVE_HOME=/usr/local/hive-2.1.1
export SCALA_HOME=/usr/local/scala-2.12.4
export KAFKA_HOME=/usr/local/kafka_2.12-0.10.2.1

export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$MYSQL_HOME/bin:$HBASE_HOME/bin:$HIVE_HOME/bin:$SCALA_HOME/bin:$KAFKA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native

# ==================================================================node1

# 使环境变量生效
source /etc/profile


# 查看配置结果
echo $KAFKA_HOME

# ==================================================================node2 node3

mkdir -p /usr/java

# ==================================================================node1

# jdk

# zookeeper

# https://www.cnblogs.com/zcf5522/p/9753539.html

# scala
scp -r $SCALA_HOME node2:/usr/local/
scp -r $SCALA_HOME node3:/usr/local/

# ==================================================================配置 kafka
mkdir $KAFKA_HOME/logs

# 配置 kafka server.properties
vi $KAFKA_HOME/config/server.properties

broker.id=1

delete.topic.enable=true
auto.create.topics.enable=true

listeners=PLAINTEXT://node1:9092
#listeners=SASL_PLAINTEXT://node1:9092

log.dirs=/usr/local/kafka_2.12-0.10.2.1/logs

zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka

# 配置ACL入口类
#authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer
# 本例使用SASL_PLAINTEXT
#security.inter.broker.protocol=SASL_PLAINTEXT
#sasl.mechanism.inter.broker.protocol=PLAIN
#sasl.enabled.mechanisms=PLAIN
# 设置本例中admin为超级用户
#super.users=User:admin;User:kafka


scp -r $KAFKA_HOME node2:/usr/local/
scp -r $KAFKA_HOME node3:/usr/local/

# ==================================================================node2 node3

# 使环境变量生效
source /etc/profile


# 查看配置结果
echo $KAFKA_HOME

# ==================================================================node2

vi $KAFKA_HOME/config/server.properties

broker.id=2
listeners=PLAINTEXT://node2:9092
#listeners=SASL_PLAINTEXT://node2:9092

# ==================================================================node3

vi $KAFKA_HOME/config/server.properties

broker.id=3
listeners=PLAINTEXT://node3:9092
#listeners=SASL_PLAINTEXT://node3:9092

# kafka集群启动

# ==================================================================node1 node2 node3
# 启动zookeeper
zkServer.sh start
zkServer.sh status
# zkServer.sh stop


# ==================================================================node1
# 需要手动在ZooKeeper中创建路径/kafka,使用如下命令连接到任意一台ZooKeeper服务器
zkCli.sh

create /kafka ''


# ==================================================================node1 node2 node3
# 启动 kafka
kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties
# kafka-server-start.sh $KAFKA_HOME/config/server.properties &


# 验证启动进程
jps


# ==================================================================kafka集群测试1
# kafka-topics.sh --delete --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic clustertopic1
# 创建topic
kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181/kafka --replication-factor 3 --partitions 1 --topic clustertopic1

# 查看topic列表
kafka-topics.sh --list --zookeeper node1:2181,node2:2181,node3:2181/kafka

# 查询topic状态
kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic clustertopic1

# 生产消息
kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic clustertopic1
> This is a message
> This is another message

# 消费消息
kafka-console-consumer.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --from-beginning --topic clustertopic1


# 停止
kafka-server-stop.sh
zkServer.sh stop


# ==================================================================kafka集群测试2
# kafka-topics.sh --delete --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic clustertopic2
# 创建topic
kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181/kafka --replication-factor 3 --partitions 3 --topic clustertopic2

# 查看topic列表
kafka-topics.sh --list --zookeeper node1:2181/kafka

# 查询topic状态
kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic clustertopic2

# 生产消息 在一台服务器上创建一个生产者
kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic clustertopic2
> This is a message
> This is another message

# 消费消息,在另外一个节点启动消费进程
kafka-console-consumer.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --from-beginning --topic clustertopic2


# 停止
kafka-server-stop.sh
zkServer.sh stop



shutdown -h now
# 快照 kafka

 

 

# 以下配置可以暂时不用配置 

# ==================================================================使用 SASL/PLAIN 认证

# ==================================================================node1
# 配置kafka server端(每个broker)
vi $KAFKA_HOME/config/server.properties

broker.id=1

delete.topic.enable=true
auto.create.topics.enable=true

#listeners=PLAINTEXT://node1:9092
listeners=SASL_PLAINTEXT://node1:9092

log.dirs=/usr/local/kafka_2.12-0.10.2.1/logs

zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka

# SASL机制
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.mechanism.inter.broker.protocol=PLAIN
sasl.enabled.mechanisms=PLAIN

# 完成身份验证的类
authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer
# 如果没有找到ACL(访问控制列表)配置,则允许任何操作。
allow.everyone.if.no.acl.found=true

# 设置admin为超级用户
super.users=User:admin;User:kafka


# ==================================================================node2
vi $KAFKA_HOME/config/server.properties

broker.id=2

delete.topic.enable=true
auto.create.topics.enable=true

#listeners=PLAINTEXT://node2:9092
listeners=SASL_PLAINTEXT://node2:9092

log.dirs=/usr/local/kafka_2.12-0.10.2.1/logs

zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka

# SASL机制
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.mechanism.inter.broker.protocol=PLAIN
sasl.enabled.mechanisms=PLAIN

# 完成身份验证的类
authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer
# 如果没有找到ACL(访问控制列表)配置,则允许任何操作。
allow.everyone.if.no.acl.found=true

# 设置admin为超级用户
super.users=User:admin;User:kafka


# ==================================================================node3
vi $KAFKA_HOME/config/server.properties

broker.id=3

delete.topic.enable=true
auto.create.topics.enable=true

#listeners=PLAINTEXT://node3:9092
listeners=SASL_PLAINTEXT://node3:9092

log.dirs=/usr/local/kafka_2.12-0.10.2.1/logs

zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka

# SASL机制
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.mechanism.inter.broker.protocol=PLAIN
sasl.enabled.mechanisms=PLAIN

# 完成身份验证的类
authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer
# 如果没有找到ACL(访问控制列表)配置,则允许任何操作。
allow.everyone.if.no.acl.found=true

# 设置admin为超级用户
super.users=User:admin;User:kafka


# ==================================================================node1
# user_userName必须配置admin用户
vi $KAFKA_HOME/config/kafka_server_jaas.conf

KafkaServer {
 org.apache.kafka.common.security.plain.PlainLoginModule required
 username="admin"
 password="Admin*123456"
 user_admin="Admin*123456"
 user_kafka="Kafka*123456"
 user_producer="Producer*123456"
 user_consumer="Consumer*123456";
};


# 这个配置定义了3个用户(kafka、producer和consumer)。在KafkaServer部分,username和password是broker用于初始化连接到其他的broker,
# 在这个例子中,kafka用户为broker间的通讯,useruserName定义了所有连接到broker和broker验证的所有的客户端连接包括其他broker的用户密码。
# useruserName必须配置admin用户,否则报错(useruserName必须配置kafka用户未验证)


# JAAS文件作为每个broker的jvm参数,在kafka-server-start.sh脚本中增加如下配置
vi $KAFKA_HOME/bin/kafka-server-start.sh

if [ "x$KAFKA_OPTS" = "x" ]; then
    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/kafka_server_jaas.conf"
fi


# 配置kafka client端 PLAIN机制下kafka生产者/消费者如何生产/消费
# kafka_client_jaas 可以分别建立不同如 producer_jaas.conf 和 consumer_jaas.conf
vi $KAFKA_HOME/config/kafka_client_jaas.conf

KafkaClient {
 org.apache.kafka.common.security.plain.PlainLoginModule required
 username="kafka"
 password="Kafka*123456";
};


# vi $KAFKA_HOME/config/producer_jaas.conf
#KafkaClient {
# org.apache.kafka.common.security.plain.PlainLoginModule required
# username="producer"
# password="Producer*123456";
#};

#vi $KAFKA_HOME/config/consumer_jaas.conf
#KafkaClient {
# org.apache.kafka.common.security.plain.PlainLoginModule required
# username="consumer"
# password="Consumer*123456";
#};


# KafkaClient部分,username和password是客户端用来配置客户端连接broker的用户,在这个例子中,客户端使用kafka 用户连接到broker。
# 修改 producer 和 consumer 的配置文件(配置注意空格不然报错)
vi $KAFKA_HOME/config/producer.properties

security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN


vi $KAFKA_HOME/config/consumer.properties

security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN


# 在启动 producer 和 consumer 时,分别添加jvm参数
vi $KAFKA_HOME/bin/kafka-console-consumer.sh

if [ "x$KAFKA_OPTS" = "x" ]; then
    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/kafka_client_jaas.conf"
fi


vi $KAFKA_HOME/bin/kafka-console-producer.sh

if [ "x$KAFKA_OPTS" = "x" ]; then
    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/kafka_client_jaas.conf"
fi


# 在启动 producer 和 consumer 时,分别添加jvm参数
#vi $KAFKA_HOME/bin/kafka-console-producer.sh
#if [ "x$KAFKA_OPTS" = "x" ]; then
#    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/producer_jaas.conf"
#fi

#vi $KAFKA_HOME/bin/kafka-console-consumer.sh
#if [ "x$KAFKA_OPTS" = "x" ]; then
#    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/consumer_jaas.conf"
#fi


# ==================================================================node1
scp -r $KAFKA_HOME/config/kafka_server_jaas.conf node2:$KAFKA_HOME/config/
scp -r $KAFKA_HOME/config/kafka_client_jaas.conf node2:$KAFKA_HOME/config/
scp -r $KAFKA_HOME/config/producer.properties node2:$KAFKA_HOME/config/
scp -r $KAFKA_HOME/config/consumer.properties node2:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/bin/kafka-server-start.sh node2:$KAFKA_HOME/bin/
scp -r $KAFKA_HOME/bin/kafka-console-consumer.sh node2:$KAFKA_HOME/bin/
scp -r $KAFKA_HOME/bin/kafka-console-producer.sh node2:$KAFKA_HOME/bin/

scp -r $KAFKA_HOME/config/kafka_server_jaas.conf node3:$KAFKA_HOME/config/
scp -r $KAFKA_HOME/config/kafka_client_jaas.conf node3:$KAFKA_HOME/config/
scp -r $KAFKA_HOME/config/producer.properties node3:$KAFKA_HOME/config/
scp -r $KAFKA_HOME/config/consumer.properties node3:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/bin/kafka-server-start.sh node3:$KAFKA_HOME/bin/
scp -r $KAFKA_HOME/bin/kafka-console-consumer.sh node3:$KAFKA_HOME/bin/
scp -r $KAFKA_HOME/bin/kafka-console-producer.sh node3:$KAFKA_HOME/bin/



# 启动
# ==================================================================node1 node2 node3
zkServer.sh start

zkServer.sh status

kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties


# ==================================================================node1
# kafka-topics.sh --delete --zookeeper node1:2181/kafka --topic sasltopic1
# 创建topic
kafka-topics.sh --create --zookeeper node1:2181/kafka --replication-factor 1 --partitions 1 --topic sasltopic1

# 查看topic列表
kafka-topics.sh --list --zookeeper node1:2181/kafka

# 查询topic状态
kafka-topics.sh --describe --zookeeper node1:2181/kafka --topic sasltopic1

# 生产者授权
kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181/kafka --add --allow-principal User:kafka --operation Write --topic sasltopic1

# 生产者
kafka-console-producer.sh --broker-list node1:9092 --topic sasltopic1 --producer.config $KAFKA_HOME/config/producer.properties
> This is a message
> This is another message

# 消费者 主题Read授权
kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181/kafka --add --allow-principal User:kafka --operation Read --topic sasltopic1

# 消费者 组Read授权
kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181/kafka --add --allow-principal User:kafka --operation Read --group sasltopic1group1

# 消费者
kafka-console-consumer.sh --bootstrap-server node2:9092 --topic sasltopic1 --from-beginning --consumer.config $KAFKA_HOME/config/consumer.properties


# 停止
kafka-server-stop.sh
zkServer.sh stop



# 启动
# ==================================================================node1 node2 node3
zkServer.sh start

zkServer.sh status

kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties


# ==================================================================node1
# kafka-topics.sh --delete --zookeeper node1:2181/kafka --topic sasltopic2
# 创建topic
kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181/kafka --replication-factor 3 --partitions 3 --topic sasltopic2

# 查看topic列表
kafka-topics.sh --list --zookeeper node1:2181/kafka

# 查询topic状态
kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic sasltopic2

# 生产者授权
kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Write --topic sasltopic2

# 生产者
kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic sasltopic2 --producer.config $KAFKA_HOME/config/producer.properties
> This is a message
> This is another message

# 消费者 主题Read授权
kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Read --topic sasltopic2

# 消费者 组Read授权
kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Read --group sasltopic2group2

# 消费者
kafka-console-consumer.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --topic sasltopic2 --from-beginning --consumer.config $KAFKA_HOME/config/consumer.properties


# 停止
kafka-server-stop.sh
zkServer.sh stop


shutdown -h now
# 快照 kafka_saslplain

 

 

 

# ==================================================================多节点zookeeper下认证 SASL认证配置

# ==================================================================node1
vi $ZOOKEEPER_HOME/conf/zoo.cfg

authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000


# ==================================================================node2
vi $ZOOKEEPER_HOME/conf/zoo.cfg

authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000


# ==================================================================node3
vi $ZOOKEEPER_HOME/conf/zoo.cfg

authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000


# ==================================================================node1
vi $KAFKA_HOME/config/kafka_zoo_jaas.conf

ZKServer {
 org.apache.kafka.common.security.plain.PlainLoginModule required
 username="kafka"
 password="Kafka*123456"
 user_kafka="Kafka*123456";
};


vi $KAFKA_HOME/bin/zookeeper-server-start.sh

if [ "x$KAFKA_OPTS" = "x" ]; then
  export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/kafka_zoo_jaas.conf -Dzookeeper.sasl.serverconfig=ZKServer"
fi



scp -r $KAFKA_HOME/config/kafka_zoo_jaas.conf node2:$KAFKA_HOME/config/
scp -r $KAFKA_HOME/config/kafka_zoo_jaas.conf node3:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/bin/zookeeper-server-start.sh node2:$KAFKA_HOME/bin/
scp -r $KAFKA_HOME/bin/zookeeper-server-start.sh node3:$KAFKA_HOME/bin/


# 启动
# ==================================================================node1 node2 node3
zkServer.sh start

zkServer.sh status

kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties


# ==================================================================node1
# kafka-topics.sh --delete --zookeeper node1:2181/kafka --topic sasltopic3
# 创建topic
kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181/kafka --replication-factor 3 --partitions 3 --topic sasltopic3

# 查看topic列表
kafka-topics.sh --list --zookeeper node1:2181/kafka

# 查询topic状态
kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic sasltopic3

# 生产者授权
kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Write --topic sasltopic3

# 生产者
kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic sasltopic3 --producer.config $KAFKA_HOME/config/producer.properties
> This is a message
> This is another message

# 消费者 主题Read授权
kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Read --topic sasltopic3

# 消费者 组Read授权
kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Read --group sasltopic3group3

# 消费者
kafka-console-consumer.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --topic sasltopic3 --from-beginning --consumer.config $KAFKA_HOME/config/consumer.properties


# 停止
kafka-server-stop.sh
zkServer.sh stop


shutdown -h now
# 快照 kafka_morezookeeper_saslplain

 

转载于:https://www.cnblogs.com/zcf5522/p/9757619.html

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值