linux消息队列配置调整,Hadoop Kafka消息队列服务安装与配置

禁用SElinux设置

[root@ip-172-31-47-5 ~]# setenforce 0

[root@ip-172-31-47-5 ~]# sudo sed -i 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

上传并安装JAVA运行环境

[root@ip-172-31-47-5 ~]# yum -y install lrzsz

[root@ip-172-31-47-5 ~]# rz

rz waiting to receive.

Starting zmodem transfer. Press Ctrl+C to cancel.

Transferring jdk-8u241-linux-x64.rpm...

100% 174745 KB 604 KB/sec 00:04:49 0 Errors

[root@ip-172-31-47-5 ~]#

[root@ip-172-31-47-5 ~]# yum -y install jdk-8u241-linux-x64.rpm

[root@ip-172-31-47-5 ~]# java -version

java version "1.8.0_241"

Java(TM) SE Runtime Environment (build 1.8.0_241-b07)

Java HotSpot(TM) 64-Bit Server VM (build 25.241-b07, mixed mode)

[root@ip-172-31-47-5 ~]#

下载Kafka二进制包并解压缩

https://downloads.apache.org/

[root@ip-172-31-47-5 ~]# curl -O https://downloads.apache.org/kafka/2.4.0/kafka_2.11-2.4.0.tgz

% Total % Received % Xferd Average Speed Time Time Time Current

Dload Upload Total Spent Left Speed

100 66.8M 100 66.8M 0 0 5877k 0 0:00:11 0:00:11 --:--:-- 7302k

[root@ip-172-31-47-5 ~]#

[root@ip-172-31-47-5 ~]# tar xzf kafka_2.11-2.4.0.tgz

[root@ip-172-31-47-5 ~]# mv kafka_2.11-2.4.0 /usr/local/

[root@ip-172-31-47-5 ~]#

启动zookeeper服务

[root@ip-172-31-47-5 ~]# cd /usr/local/kafka_2.11-2.4.0/

[root@ip-172-31-47-5 kafka_2.11-2.4.0]# ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties

[root@ip-172-31-47-5 kafka_2.11-2.4.0]#

修改kafka服务堆内存配置

[root@ip-172-31-47-5 kafka_2.11-2.4.0]# vi bin/kafka-server-start.sh

默认

export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G

修改为

export KAFKA_HEAP_OPTS="-Xmx8G -Xms8G

启动服务

[root@ip-172-31-47-5 kafka_2.11-2.4.0]# ./bin/kafka-server-start.sh config/server.properties

成功启动服务后的终端屏幕输出最后一行

[2020-03-06 07:19:10,271] INFO [KafkaServer id=0] started (kafka.server.KafkaServer)

查看端口监听

[root@ip-172-31-47-5 ~]# netstat -lntp

Active Internet connections (only servers)

Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name

tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 1/systemd

tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 3712/sshd

tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 3368/master

tcp6 0 0 :::111 :::* LISTEN 1/systemd

tcp6 0 0 :::22 :::* LISTEN 3712/sshd

tcp6 0 0 :::45848 :::* LISTEN 13243/java

tcp6 0 0 ::1:25 :::* LISTEN 3368/master

tcp6 0 0 :::37435 :::* LISTEN 13406/java

tcp6 0 0 :::9092 :::* LISTEN 13406/java

tcp6 0 0 :::2181 :::* LISTEN 13243/java

[root@ip-172-31-47-5 ~]#

创建测试主题

[centos@ip-172-31-47-5 ~]$ cd /usr/local/kafka_2.11-2.4.0/

[centos@ip-172-31-47-5 kafka_2.11-2.4.0]$ ./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test

Created topic test.

[centos@ip-172-31-47-5 kafka_2.11-2.4.0]$

查看主题

[centos@ip-172-31-47-5 kafka_2.11-2.4.0]$ ./bin/kafka-topics.sh --list --zookeeper localhost:2181

test

[centos@ip-172-31-47-5 kafka_2.11-2.4.0]$

使用测试主题生产消息(生产者)

[centos@ip-172-31-47-5 kafka_2.11-2.4.0]$ echo "Welcome to Kafka!" | ./bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test > /dev/null

[centos@ip-172-31-47-5 kafka_2.11-2.4.0]$

使用消息(消费者)(Ctrl+C退出)

[root@ip-172-31-47-5 kafka_2.11-2.4.0]# ./bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning

Welcome to Kafka!

^C

Processed a total of 1 messages

[root@ip-172-31-47-5 kafka_2.11-2.4.0]#