相关文档
- 介质路径:https://kafka.apache.org/downloads.html
依赖服务
系统优化
hostname hadoop1
vi /etc/selinux/config
SELINUX=disabled
getenforce
systemctl stop firewalld && systemctl disable firewalld && systemctl status firewalld
yum install chrony -y
vi /etc/chonry.conf
server hadoop1 iburst
systemctl enable chronyd --now
chronyc sources -v
210 Number of sources = 1
.-- Source mode '^' = server, '=' = peer, '#' = local clock.
/ .- Source state '*' = current synced, '+' = combined , '-' = not combined,
| / '?' = unreachable, 'x' = time may be in error, '~' = time too variable.
|| .- xxxx [ yyyy ] +/- zzzz
|| Reachability register (octal) -. | xxxx = adjusted offset,
|| Log2(Polling interval) --. | | yyyy = measured offset,
|| \ | | zzzz = estimated error.
|| | |
MS Name/IP address Stratum Poll Reach LastRx Last sample
====================================================================================================================
^* hadoop1 4 6 377 12 -28us[ -45us] +/- 75ms
ssh-keygen -t rsa
cat id_rsa.pub >> hadoop1:/root/.ssh/authorized_keys
cat id_rsa.pub >> hadoop2:/root/.ssh/authorized_keys
cat id_rsa.pub >> hadoop3:/root/.ssh/authorized_keys
cat /root/.ssh/authorized_keys
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDwuKw9LdfDO3Ln+ViNtQEqZtH/RvoFymKkexBXRUK/2XcczKHPv967KHH71L/5vPOQPUXZLZg3TPERlRTIW9MvCh0LmceGAiQHrxczx56RnYh8nESknd2jbHBToGwqgoB8xsB2IQuhze0CqvRs7A0nrbyBvnUpg/DvePTOSSgii4z9kishBCbrCPamQm20drXVDK3gQ9Q+/YJLKa3+mxzI67xfk/jby0A0DD9XKL7fflRgMK0GXEtYsJ04tKc5Bo+w6Zc8gHyryFrKD4wpeoPakqmrdzaTVYI1x5WvrAPrQplxAP8iNfBqRJSHvlDBXVeXgSxz2I4HBshsStkKp root@redis1
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDkspWeTwWoWyr6biMnefOYT4kh+7gPAboHAWe7p67IR9pfu+Rkk/vxLFDbi7X6Td9AhIXEZH6fY5BhihBzhRO/VtjE24QqnXdOLDHV1i0rSEYh6GOAbnVl/93lKidQF/2wvnQET31m1iwls3ul6aWw8/pOcxWy6kB+6MRiOExhu+0erE3jBFLcl+e0IJLKp/nLjCof/qWh3hLGVyhgMn/WmGhf7OyUbedXFqAwwS83/M60jSL1nB1lnIOoHrNSdnrN/GJVXmmwJjJAG4g4hbAg2zNind2rz6p4mq5k7iBbDUFghFwKKYsGeV0Onm7SKErFlHCJNFSOgfVNpaUYJ root@redis2
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+DGKAYw3tbdmv2GDsz3HEOdoKk8JVCEvDFczap2g3DoaqwEBkRag2l9IQ3RZL/WtpKe0f2vZzcm5t3d7e6YhyfEXRn1fjOmynTcykB13xAVrlRfJ6Sayur0OiPzWBktpNj8qaTKjwH+lyHGBwa5duqKiVEglEH2mX5grcOa/mH2Mo+IWsCYeCldKjfdBy2drlAim1fYvJwvtg0uDe8sfDUdDonG4phNOVaWB2u79SxKlGnGewGNuOrifIzkbc0mH9kNgrlw/xdSIqaFA738Yn/4n/kSe3BgceJ0wBowLzorgW2ogyGOdQp6MzBRlg/hxn4EDLJisrC9mSCMOOl root@redis3
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvWawSJqu4/Adnu6TjvV8rVDAqTU2CGNaSBOTDjcytHHOaY8UiwUMKvXUJugBmRkyhtWhQPHrVSmOH6+qMnHk5XQcWBmce8qCQqDoz49WwyZH95ciY/ynKR9dzAJwXN5fvJEoKxBhSJLk27SDsgRUX05IAjTN5Wx05GCNC36CRGHr6bwsC5iK+nv1ZllkRPyqoICJcvVVoJFDe+svNwLJS8bEpTUS/3C6w1RdfEgGVK0/NLnmANz6VIu5LAZqOpwFcB8Zed3wgnoHUfDCSXLEUQbcgRxDvba7lcvOqbiNh4Tr6WctSHw0UD9PSK6AXdS0jAAyjZ1J5kbWaI+vmZ root@hadoop1
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwCqgQWDgw7sSqNer1oONzsFhoCWBmLqdWOQCcC7RYhD6kiVzdAEP7qZwWKRwoe/E++xP0+slgxsIsXGVoObGrlT3n+g/2xsgTCaBT/6sGV7k28UOozh76GlyfJjzavbwWE9Q2yR2mkb3/ILGE6CUNCkqqLuYEDTG4DxNupGhsGSYChAcjclzYFrMxDARiOJ8cahDjVlmGzFWxNhzJ36pFC1Rdyeu4CrtZ8tkuqQagGZqB63bVmvTiOM2fY8Wp8TNv0Zz2XmFmv7IUhpDXlPZdFCviwLYLLoJ9LTG32rO/jY0U78LFdDpsYdebthztNakKMZEhCqVIR+k1VMPtp root@hadoop2
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmj5qT64jSc3LCR2EBKB+12C1XxhFlc44X8zdf3mL8gbepG+ndMgBV4eombLg7QjZshCsjhk9d8esofAlrPk5tX/nWWHg3p5jYTh5/6V+iU7VDpWmMVN/87dsjBbmM9P6jTNiwqk4rdSXDKXkmrVygGHnEj95eP35Nq1JKg+GS7RjWWB0+loGQ4eYKO1nj2nYNOlNBi28CKh1uMWf42bDtcfKP3Z4gEOtPBD5rVPiU2Tq6jgtAs/VvaYGv5FHO4MB0lBE1ik8zp/4trfGU5hie/1PzCRAIvsqPEBSzeUs9nhHODj6vZYwgQupK9Qv5jEbQgh6pCGEfFZlfsC03 root@hadoop3
cat > /etc/profile << EOF
# Oracle JDK 1.8.0_333
export JAVA_HOME=/data/service/jdk/jdk1.8.0_333
export CLASSPATH=$:CLASSPATH:$JAVA_HOME/lib/
export PATH=$PATH:$JAVA_HOME/bin
EOF
source /etc/profile
java -version
java version "1.8.0_333"
Java(TM) SE Runtime Environment (build 1.8.0_333-b02)
Java HotSpot(TM) 64-Bit Server VM (build 25.333-b02, mixed mode)
cat > /etc/hosts << EOF
# redis-nodes
10.10.10.21 redis1
10.10.10.22 redis2
10.10.10.23 redis3
# hadoop-nodes
10.10.10.131 hadoop1
10.10.10.132 hadoop2
10.10.10.133 hadoop3
EOF
swapoff -a
vi /etc/fstab
echo "vm.swappiness = 0" >> /etc/sysctl.conf
sysctl -p
echo never > /sys/kernel/mm/transparent_hugepage/enabled && echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.local && echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.local
vi /etc/security/limits.d/20-nproc.conf
* - nofile 655350
* - nproc 655350
查看 ZooKeeper 集群状态
$ZK_HOME/bin/zkCli.sh -server hadoop1:2181,hadoop2:2181,hadoop3:2181
[zk: hadoop1:2181,hadoop2:2181,hadoop3:2181(CONNECTED) 0] ls /
[admin, brokers, cluster, config, consumers, controller, controller_epoch, hadoop-ha, hbase, isr_change_notification, latest_producer_id_block, log_dir_event_notification, rmstore, spark, yarn-leader-election, zookeeper]
创建路径
mkdir -p /data/service/kafka/{kafka-data,kafka-logs}
配置 /etc/profile
export KAFKA_HOME=/data/service/kafka/kafka_2.12-2.5.0
export PATH=$PATH:$KAFKA_HOME/bin:$KAFKA_HOME/sbin:$KAFKA_HOME/lib
配置 $KAFKA_HOME/config/server.properties
- 只需在某一个节点上部署即可
- 配置测试成功后,推送到所有其余节点
broker.id=0
listeners=PLAINTEXT://10.10.10.131:9092
log.dirs=/data/service/kafka/kafka-data
delete.topic.enable=true
unclean.leader.election.enable=true
zookeeper.connect=localhost:hadoop1:2181,hadoop2:2181,hadoop3:2181
export KAFKA_OPTS="-XX:+UseG1GC -Xmx1g -Xms1g"
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.retention.check.interval.ms=300000
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
配置 $KAFKA_HOME/bin/kafka-run-class.sh
if [ "x$LOG_DIR" = "x" ]; then
LOG_DIR="/data/service/kafka/kafka-logs"
fi
- 调整 Kafka JVM 路径
- 若不希望使用节点的 JDK 的版本或无法进行节点 JDK 的更新,则可以通过调整如下部分的内容,写死 JDK 路径,起到指定 JDK 版本的效果
if [ -z "$JAVA_HOME" ]; then
JAVA="java"
else
JAVA="$JAVA_HOME/bin/java"
fi
分发配置
scp -r /data/service/kafka root@hadoop2:/data/service/
scp -r /data/service/kafka root@hadoop3:/data/service/
调整 $KAFKA_HOME/config/server.properties
broker.id=1
listeners=PLAINTEXT://10.10.10.132:9092
broker.id=2
listeners=PLAINTEXT://10.10.10.133:9092
启动 Kafka
$KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties
$KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties
$KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties
验证 Kafka
$KAFKA_HOME/bin/kafka-topics.sh --bootstrap-server hadoop1:9092 --list
$KAFKA_HOME/bin/kafka-topics.sh --create --topic test --bootstrap-server hadoop1:9092 --partitions 3 --replication-factor 2
$KAFKA_HOME/bin/kafka-topics.sh --bootstrap-server hadoop1:9092 --describe --topic test
$KAFKA_HOME/bin/kafka-console-producer.sh --topic test --bootstrap-server hadoop1:9092
> a = 1
$KAFKA_HOME/bin/kafka-console-consumer.sh --topic test --bootstrap-server hadoop1:9092 --from-beginning