kafka常用命令

#!/bin/sh

#启动zookeeper

/home/huchao/kafka/kafka_2.12-1.1.0/bin/zookeeper-server-start.sh -daemon  /home/huchao/kafka/kafka_2.12-1.1.0/config/zookeeper-001.properties

sleep 3 #等3秒后执行

#启动kafka

/home/huchao/kafka/kafka_2.12-1.1.0/bin/kafka-server-start.sh  -daemon  /home/huchao/kafka/kafka_2.12-1.1.0/config/server-001.properties
----------------------------------------------------------------------------
启动zookeeper

/bin/zookeeper-server-start.sh  -daemon /config/zookeeper.properties 

守护进程 -daemon

2>&1 < /dev/null &

启动kafka

bin/kafka-server-start.sh  config/server.properties 

停止kafka
bin/kafka-server-stop.sh

停止zookeeper
bin/zookeeper-server-stop.sh

查看zookeeper端口
netstat -lnp|grep 2181
netstat -tnl|grep 2181
netstat -tnl|grep 9092

杀死该进程
kill - 9  ****

查看进程
ps -ef|grep kafka

查看broker状态
./kafka-broker-api-versions.sh --bootstrap-server 10.2.5.6:9092
./kafka-broker-api-versions.sh --bootstrap-server 10.2.5.6:9092,10.3.34:9092,10.2.5.15:9092
---------------------------------------------------------------------------
kafka主题负载均衡
每当代理停止或崩溃领导，该代理的分区传输到其他副本。这意味着默认情况下，当代理重新启动时，它将只是所有其分区的跟随者，这意味着它不会用于客户端读取和写入。
为了避免这种不平衡，Kafka有一个优选副本的概念。如果分区的副本的列表为1,5,9，则节点1优选为节点5或9的引导者，因为它在副本列表中较早。您可以通过运行以下命令让Kafka集群尝试恢复恢复的副本的领导：

   bin / kafka-preferred-replica-election.sh --zookeeper  10.2.5.6
由于运行此命令可能很繁琐，您还可以配置Kafka通过设置以下配置自动执行此操作：
 
auto.leader.rebalance.enable = true

./kafka-preferred-replica-election.sh --zookeeper 10.2.5.201
-------------------------------------------------------------------------

查看zookeeper状态
echo stat | nc 10.3.34.34 2181

创建topic
./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
./bin/kafka-topics.sh --zookeeper 10.2.5.6:2181,10.2.5.15:2181,10.101.36.3:2181  --topic MAC_BigData --partitions 1 --replication-factor 2 --create

展示topic
bin/kafka-topics.sh --list --zookeeper localhost:2181

查看指定topic信息
bin/kafka-topics.sh --zookeeper localhost:2181 --describe --topic test_topic
 

控制台向topic生产数据
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test1

bin/kafka-console-producer.sh --broker-list 10.2.5.200:9092,10.2.5.201:9092,10.2.5.202:9092 --topic test1
 

控制台消费topic的数据
bin/kafka-console-consumer.sh  --bootstrap-server localhost:9092 --topic test --from-beginning
 

查看topic某分区偏移量最大（小）值
bin/kafka-run-class.sh kafka.tools.GetOffsetShell --topic hive-mdatabase-hostsltable  --time -1 --broker-list node86:9092 --partitions 0
注： time为-1时表示最大值，time为-2时表示最小值

查看当前正在使用的消费者组
./kafka-consumer-groups.sh --bootstrap-server 10.2.5.6:9092,10.3.34:9092,10.2.5.15:9092 --list

./bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group my-group

增加topic分区数
为topic t_cdr 增加10个分区

bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic t_cdr --partitions 10
 

删除topic，慎用，只会删除zookeeper中的元数据，消息文件须手动删除
bin/kafka-run-class.sh kafka.admin.DeleteTopicCommand --zookeeper localhost:2181 --topic test

删除Topic，broker的delete.topic.enable=true才可以删除，否则删除topic的请求会被忽略.删除topic会丢弃topic里所有数据
bin/kafka-topics.sh --zookeeper zookeeper:2181 --delete --topic topicAgroup
 

查看topic消费进度
这个会显示出consumer group的offset情况， 必须参数为--group， 不指定--topic，默认为所有topic

Displays the: Consumer Group, Topic, Partitions, Offset, logSize, Lag, Owner for the specified set of Topics and Consumer Group

bin/kafka-run-class.sh kafka.tools.ConsumerOffsetChecker

描述topic
bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic my-replicated-topic

生产者：
bin/kafka-console-producer.sh --broker-list 10.2.5.6:9092,10.3.34.34:9092,10.2.5.15:9092 --topic test_topic
bin/kafka-console-producer.sh --broker-list 10.2.5.200:9092 --topic test1

消费者：
bin/kafka-console-consumer.sh --zookeeper 10.2.5.6:2181,10.3.34.34:2181,10.2.5.15:2181 --topic test_topic --from-beginnin

下线某个broker：
比如下线broker0
bin/kafka-run-class.sh kafka.admin.ShutdownBroker --zookeeper 127.0.0.1:2181 --broker 0 --num.retries 3 --retry.interval.ms 60

shutdown broker
--------------------------------------------------------------------------------------------------
修改副本：
./kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file increase-replication-factor.json --execute

{
  "version": 1,
  "partitions": [
    {
      "topic": "BMSToCMS",
      "partition": 0,
      "replicas": [ 0, 1, 2 ]
    }
   ]
}

kafka-reassign-partitions.sh --zookeeper localhost:2181 --topics-to-move-json-file topic.conf.multi --broker-list 0,1,2 --generate > ressgintopic.conf

查看主题：
./kafka-topics.sh --zookeeper localhost:2181 --topic CMSToPGW --describe

==========================安全=======================
执行

bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file increase-replication-factor.json --execute

验证

bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file increase-replication-factor.json --verify

export KAFKA_SASL_OPTS='-Djava.security.auth.login.config=/home/grgbanking/kafka/kafka_2.12-1.1.0/config/kafka_client_jaas.conf'

export KAFKA_SASL_OPTS='-Djava.security.auth.login.config=/home/grgbanking/kafka/kafka_2.12-1.1.0/config/kafka_server_jaas.conf'
# Launch mode
if [ "x$DAEMON_MODE" = "xtrue" ]; then
  nohup $JAVA $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS  $KAFKA_SASL_OPTS  $KAFKA_JMX_OPTS $KAFKA_LOG4J_OPTS -cp $CLASSPATH $KAFKA_OPTS "$@" > "$CONSOLE_OUTPUT_FILE" 2>&1 < /dev/null &
else
  exec $JAVA $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS  $KAFKA_SASL_OPTS $KAFKA_JMX_OPTS $KAFKA_LOG4J_OPTS -cp $CLASSPATH $KAFKA_OPTS "$@"
fi

####./bin/kafka-acls.sh --add --authorizer-properties zookeeper.connect=10.2.5.200:2181 --allow-principal User:alice --allow-host * --operation Read --operation Write --topic test1

./bin/kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=10.2.5.200:2181,10.2.5.201:2181,10.2.5.202:2181 --add --allow-principal User:alice --operation All --topic test1

./kafka-acls.sh --authorizer-properties zookeeper.connect=10.2.5.200:2181,10.2.5.201:2181,10.2.5.202:2181 --list --topic test1

-------------------------------Java环境变量-------------------------------------
===================Java open jdk====================
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.161-2.b14.el7.x86_64

export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
===============================================================

export JRE_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.65-3.b17.el7.x86_64/jre
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.65-3.b17.el7.x86_64
export PATH=$JAVA_HOME/bin:$PATH

===================java jdk========================
export JRE_HOME=/usr/java/jdk1.8.0_181/jre
export JAVA_HOME=/usr/java/jdk1.8.0_181
export PATH=$JAVA_HOME/bin:$PATH
--------------------------------------------------------------------------------
export?PATH=$PATH:$JAVA_HOME/bin

-------------------------------------------------

if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
    export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G"
    export JMX_PORT="9999"
fi
------------------------------------------------
* * * * * /usr/local/bin/process_monitor.sh "/home/grgbanking/kafka/kafka_2.12-1.1.0/bin/zookeeper-server-start.sh  -daemon  /home/grgbanking/kafka/kafka_2.12-1.1.0/config/zookeeper-003.properties" "/home/grgbanking/kafka/kafka_2.12-1.1.0/bin/zookeeper-server-start.sh  -daemon  /home/grgbanking/kafka/kafka_2.12-1.1.0/config/zookeeper-003.properties"

* * * * * /usr/local/bin/process_monitor.sh "/usr/local/java kafkaServer" "/home/grgbanking/kafka/kafka_2.12-1.1.0/bin/kafka-server-start.sh -daemon /home/grgbanking/kafka/kafka_2.12-1.1.0/config/server-003.properties"
-------------------------------------------------