importcom.com.wj.kafkaAPI.conf.KafkaConsumerConfig;importkafka.admin.AdminClient;importkafka.admin.AdminUtils;importkafka.admin.RackAwareMode;importkafka.cluster.Broker;importkafka.cluster.Cluster;importkafka.consumer.ConsumerConfig;importkafka.consumer.KafkaStream;importkafka.coordinator.group.GroupOverview;importkafka.javaapi.consumer.ConsumerConnector;importkafka.javaapi.consumer.ZookeeperConsumerConnector;importkafka.server.ConfigType;importkafka.utils.ZkUtils;importorg.apache.kafka.common.TopicPartition;importorg.apache.kafka.common.security.JaasUtils;importorg.apache.zookeeper.KeeperException;importorg.apache.zookeeper.ZooKeeper;importorg.springframework.beans.factory.annotation.Autowired;importorg.springframework.boot.autoconfigure.condition.ConditionalOnExpression;importorg.springframework.stereotype.Component;importscala.collection.JavaConversions;importscala.collection.Map;importscala.collection.Set;importjava.io.IOException;import java.util.*;public classConsoleApi {
@AutowiredprivateKafkaConsumerConfig consumerConfig;private ZkUtils zkUtils = null;private ZooKeeper zooKeeper = null;/*** -获取集群信息(与getAllBrokersInCluster()只是返回类型不一致)*/
publicCluster getCluster(){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());returnzkUtils.getCluster();
}public void getLeaderAndIsrForPartition(String topicName,intpatition){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
System.out.println("打印:"+zkUtils.getLeaderAndIsrForPartition(topicName,patition));
}public booleancreateConsumer(String groupId,String topic) {try{
Properties properties= newProperties();
properties.put("zookeeper.connect", consumerConfig.getZookeeperConn());//声明zk
properties.put("group.id", groupId);
ConsumerConnector consumer= new ZookeeperConsumerConnector(new ConsumerConfig(properties),true);
java.util.Map topicCountMap = new HashMap();if (topic!=null &&!"".equals(topic)){
topicCountMap.put(topic,1); //一次从主题中获取一个数据
}else{
topicCountMap.put("topic", 1); //一次从主题中获取一个数据
}
java.util.Map>> messageStreams =consumer.createMessageStreams(topicCountMap);return true;
}catch(RuntimeException e){return false;
}
}public booleancreateConsumer(String groupId) {return createConsumer(groupId,null);
}public booleandeleteUselessConsumer(String group) {return deleteUselessConsumer("-1", group);
}/*** -删除topic路径
*@return
*/
publicString deleteTopicsPath(){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());returnzkUtils.DeleteTopicsPath();
}/*** -根据brokerId获取broker的信息
*@parambrokerId*/
public Broker getBrokerInfo(intbrokerId){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());returnzkUtils.getBrokerInfo(brokerId).get();
}/*** -获取消费者的路径
*@return
*/
publicString ConsumersPath(){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());returnzkUtils.ConsumersPath();
}/*** 删除多个topic
*@paramtopicNames
*@return
*/
public String[] deleteTopics(finalString...topicNames) {if(topicNames==null || topicNames.length==0) return new String[0];
java.util.Set deleted = new LinkedHashSet();for(String topicName: topicNames) {if(topicName!=null || !topicName.trim().isEmpty()) {
deleteTopic(topicName);
deleted.add(topicName.trim());
}
}return deleted.toArray(newString[deleted.size()]);
}/*** 获取所有的TopicList
*@return
*/
public ListgetTopicList() {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
List allTopicList =JavaConversions.seqAsJavaList(zkUtils.getAllTopics());returnallTopicList;
}/*** ~获取某个分组下的所有消费者
*@paramgroupName
*@return
*/
public ListgetConsumersInGroup(String groupName) {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
List allTopicList =JavaConversions.seqAsJavaList(zkUtils.getConsumersInGroup(groupName));returnallTopicList;
}/*** 判断某个topic是否存在
*@paramtopicName
*@return
*/
public booleantopicExists(String topicName) {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());boolean exists =AdminUtils.topicExists(zkUtils,topicName);returnexists;
}/*** ~
*@paramgroupName
*@return
*/
public booleanisConsumerGroupActive(String groupName) {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());boolean exists =AdminUtils.isConsumerGroupActive(zkUtils,groupName);returnexists;
}/*** 获取所有消费者组
*@return
*/
public ListgetConsumerGroups() {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
List set =JavaConversions.seqAsJavaList(zkUtils.getConsumerGroups()) ;returnset;
}/*** 根据消费者的名称获取topic
*@paramgroupName
*@return
*/
public ListgetTopicsByConsumerGroup(String groupName) {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
List set2 =JavaConversions.seqAsJavaList(zkUtils.getTopicsByConsumerGroup(groupName)) ;returnset2;
}/*** 获取排序的BrokerList
*@return
*/
public ListgetSortedBrokerList() {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
List set2 =JavaConversions.seqAsJavaList(zkUtils.getSortedBrokerList()) ;returnset2;
}public ListgetAllBrokersInCluster() {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
List set2 =JavaConversions.seqAsJavaList(zkUtils.getAllBrokersInCluster()) ;returnset2;
}/*** 获取消费某个topic发送消息的消费组
*@paramtopicName
*@return
*/
public SetgetAllConsumerGroupsForTopic(String topicName) {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
Set stringSeq =zkUtils.getAllConsumerGroupsForTopic(topicName);returnstringSeq;
}/*** 获取删除主题的路径
*@paramtopicName
*@return
*/
publicString getDeleteTopicPath(String topicName) {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
String stringSeq=zkUtils.getDeleteTopicPath(topicName);returnstringSeq;
}/*** 获取topic路径
*@paramtopicName
*@return
*/
publicString getTopicPath(String topicName) {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
String stringSeq=zkUtils.getTopicPath(topicName);returnstringSeq;
}public booleancreateTopic(String topicName) {
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());try{// AdminUtils.createTopic(zkUtils, topicName, 1, 1, newProperties(),RackAwareMode.Enforced$.MODULE$);return true;
}catch(RuntimeException e){
}return false;
}/*** 删除topic信息(前提是server.properties中要配置delete.topic.enable=true)
*@paramtopicName*/
public voiddeleteTopic(String topicName){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());//删除topic 't1'
AdminUtils.deleteTopic(zkUtils, topicName);
System.out.println("删除成功!");
}/*** 删除topic的某个分区
*@parambrokerId
*@paramtopicName*/
public void deletePartition(intbrokerId,String topicName){//删除topic 't1'
zkUtils = ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
zkUtils.deletePartition(brokerId,topicName);
System.out.println("删除成功!");
}/*** 改变topic的配置
*@paramtopicName*/
public voidupdateTopic(String topicName){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());
Properties props= AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), "test");//增加topic级别属性
props.put("min.cleanable.dirty.ratio", "0.3");//删除topic级别属性
props.remove("max.message.bytes");
props.put("retention.ms","1000");//修改topic 'test'的属性
AdminUtils.changeTopicConfig(zkUtils, "test", props);
System.out.println("修改成功");
zkUtils.close();
}/*** 获取所有topic的配置信息*/
public MapfetchAllTopicConfigs(){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());returnAdminUtils.fetchAllTopicConfigs(zkUtils);
}/*** 获取所有topic或者client的信息()type为:ConfigType.Topic()/ConfigType.Client()*/
public MapfetchAllEntityConfigs(String type){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());returnAdminUtils.fetchAllEntityConfigs(zkUtils, type);
}/*** 获取指定topic的配置信息
*@paramtopicName*/
publicProperties fetchEntityConfig(String topicName){
zkUtils= ZkUtils.apply(consumerConfig.getZookeeperConn(), 30000, 30000, JaasUtils.isZkSecurityEnabled());returnAdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topicName);
}private booleandeleteUselessConsumer(String topic, String group) {//if (topic.endsWith("-1")) {
StringBuilder sb = new StringBuilder().append("/consumers/")
.append(group);returnrecursivelyDeleteData(sb.toString());
}private booleanrecursivelyDeleteData(String path) {
List childList =getChildrenList(path);if (childList == null) {return false;
}else if(childList.isEmpty()) {
deleteData(path);
}else{for(String childName : childList) {
String childPath= path + "/" +childName;
List grandChildList =getChildrenList(childPath);if (grandChildList == null) {return false;
}else if(grandChildList.isEmpty()) {
deleteData(childPath);
}else{
recursivelyDeleteData(childPath);
}
}
deleteData(path);
}return true;
}private booleandeleteData(String path) {try{
zooKeeper.delete(path,-1);
}catch(InterruptedException e) {//log.error("delete error,InterruptedException:" + path, e);
return false;
}catch(KeeperException e) {//log.error("delete error,KeeperException:" + path, e);
return false;
}return true;
}private ListgetChildrenList(String path) {try{
zooKeeper= new ZooKeeper(consumerConfig.getZookeeperConn(), 6000, null);return zooKeeper.getChildren(path, false, null);
}catch(KeeperException e) {return null;
}catch(InterruptedException e) {return null;
}catch(IOException e) {
e.printStackTrace();
}return new ArrayList(Collections.singleton(path));
}/*** get all subscribing consumer group names for a given topic
*@parambrokerListUrl localhost:9092 for instance
*@paramtopic topic name
*@return
*/
public static java.util.SetgetAllGroupsForTopic(String brokerListUrl, String topic) {
AdminClient client=AdminClient.createSimplePlaintext(brokerListUrl);try{
List allGroups =scala.collection.JavaConversions.seqAsJavaList(client.listAllGroupsFlattened().toSeq());
java.util.Set groups = new HashSet();for(GroupOverview overview: allGroups) {
String groupID=overview.groupId();
java.util.Map offsets =JavaConversions.mapAsJavaMap(client.listGroupOffsets(groupID));
groups.add(groupID);
}returngroups;
}finally{
client.close();
}
}
}