kafka consumer group的删除和offset恢复

运维kafka时，会有时需要删除无效或已下线的consumer group，若设置的offset保存位置是zookeepr，则数据都在zookeeper中，可自行删除，方式就是删除zookeeper相应节点。

kafka的offset若想按照生产时间进行恢复，事实严重不准，具体可参见：关于kafka中的timestamp与offset的对应关系 

恢复步骤：

1. 获取该consumer group消费topic的各partition该时间的offset

2. 在zookeeper中修改这些offset

进行操作期间需要暂停该group对topic的消费，恢复offset后再重启，否则修改不生效。

上代码！

ResetOffsetOperator.resetOffset(topic, group, null, whichTime);
ZookeeperOperator zookeeperOperator = new ZookeeperOperator();
zookeeperOperator.deleteUselessConsumer("test1");

public class ZookeeperOperator implements Watcher {

  private ZooKeeper zooKeeper = null;

  private static final Logger log = LoggerFactory
      .getLogger(ZookeeperOperator.class);

  public ZookeeperOperator() throws IOException {
    this(KafkaConf.ZOOKEEPERHOST);
  }

  public ZookeeperOperator(String zookeeperHost) throws IOException {
    zooKeeper = new ZooKeeper(zookeeperHost, 5000, ZookeeperOperator.this);
  }

  public void close() {
    try {
      zooKeeper.close();
    } catch (InterruptedException e) {
      log.error("Failed to close zookeeper:", e);
    }
  }

  /**
   * 读取指定节点下孩子节点数目
   *
   * @param path 节点path
   * @return
   */
  private int readChildren(String path) {
    try {
      return zooKeeper.getChildren(path, false, null).size();
    } catch (KeeperException e) {
      log.error("Read error,KeeperException:" + path, e);
      return 0;
    } catch (InterruptedException e) {
      log.error("Read error,InterruptedException:" + path, e);
      return 0;
    }
  }

  private List<String> getChildrenList(String path) {
    try {
      return zooKeeper.getChildren(path, false, null);
    } catch (KeeperException e) {
      log.error("Read error,KeeperException:" + path, e);
      return null;
    } catch (InterruptedException e) {
      log.error("Read error,InterruptedException:" + path, e);
      return null;
    }
  }

  private boolean setData(String path, String data) {
    try {
      zooKeeper.setData(path, data.getBytes(), -1);
    } catch (KeeperException e) {
      log.error("Set error,KeeperException:" + path + " data:" + data, e);
      return false;
    } catch (InterruptedException e) {
      log.error("Set error,InterruptedException:" + path + " data:" + data, e);
      return false;
    }
    return true;
  }

  private boolean deleteData(String path) {
    try {
      zooKeeper.delete(path, -1);
    } catch (InterruptedException e) {
      log.error("delete error,InterruptedException:" + path, e);
      return false;
    } catch (KeeperException e) {
      log.error("delete error,KeeperException:" + path, e);
      return false;
    }
    return true;
  }

  private boolean recursivelyDeleteData(String path) {
    List<String> childList = getChildrenList(path);
    if (childList == null) {
      return false;
    } else if (childList.isEmpty()) {
      deleteData(path);
    } else {
      for (String childName : childList) {
        String childPath = path + "/" + childName;
        List<String> grandChildList = getChildrenList(childPath);
        if (grandChildList == null) {
          return false;
        } else if (grandChildList.isEmpty()) {
          deleteData(childPath);
        } else {
          recursivelyDeleteData(childPath);
        }
      }
      deleteData(path);
    }
    return true;
  }

  private String getData(String path) {
    try {
      return new String(zooKeeper.getData(path, false, null));
    } catch (KeeperException e) {
      log.error("Read error,KeeperException:" + path, e);
      return "";
    } catch (InterruptedException e) {
      log.error("Read error,InterruptedException:" + path, e);
      return "";
    }
  }

  /**
   * 读取指定节点下孩子节点数目
   *
   * @param topic kafka topic 名称
   * @return
   */
  public int readTopicChildren(String topic) {
    StringBuilder sb = new StringBuilder().append("/brokers/topics/")
        .append(topic).append("/partitions");
    return readChildren(sb.toString());
  }

  public boolean setTopicGroupOffset(String topic, String group,
      String partition, String data) {
    StringBuilder sb = new StringBuilder().append("/consumers/").append(group)
        .append("/offsets/").append(topic).append("/").append(partition);
    return setData(sb.toString(), data);
  }

  public String getTopicGroupOffset(String topic, String group,
      String partition) {
    StringBuilder sb = new StringBuilder().append("/consumers/").append(group)
        .append("/offsets/").append(topic).append("/").append(partition);
    System.out.println(sb.toString());
    return getData(sb.toString());
  }

  public boolean deleteUselessConsumer(String topic, String group) {
    if (topic.endsWith("-1")) {
      StringBuilder sb = new StringBuilder().append("/consumers/")
          .append(group);
      return recursivelyDeleteData(sb.toString());
    } else {
      StringBuilder sb = new StringBuilder().append("/consumers/").append(group)
          .append("/offsets/").append(topic);
      return recursivelyDeleteData(sb.toString());
    }
  }

  public boolean deleteUselessLikeConsumer(String topic, String group) {
    String path = "/consumers";
    List<String> childList = getChildrenList(path);
    int success = 0;
    int count = 0;
    for (String child : childList) {
      if (child.startsWith(group)) {
        count++;
        if (deleteUselessConsumer(topic, child)) {
          success++;
        }
      }
    }
    if (success == count) {
      return true;
    } else {
      return false;
    }
  }

  public boolean deleteUselessLikeConsumer(String group) {
    return deleteUselessLikeConsumer("-1", group);
  }

  public boolean deleteUselessConsumer(String group) {
    return deleteUselessConsumer("-1", group);
  }

  @Override
  public void process(WatchedEvent event) {
    log.info("Receive Event：" + event.getState());
  }
}

public class ResetOffsetOperator {
  public static boolean resetOffset(final String topic, String group,
      Properties properties, long whichTime) throws IOException {
    if (StringUtils.isBlank(topic) || StringUtils.isBlank(group)) {
      System.err.println("topic or group can not be null or empty!");
      System.exit(2);
    }
    if (properties == null) {
      properties = new Properties();
      properties.setProperty("zookeeper.connect", KafkaConf.ZOOKEEPERHOST);
      properties.setProperty("group.id", group);
      // zookeeper最大超时时间，就是心跳的间隔，若是没有反映，那么认为已经死了，不易过大
      properties.setProperty("zookeeper.session.timeout.ms", "10000");
      // ZooKeeper集群中leader和follower之间的同步时间
      properties.setProperty("zookeeper.sync.time.ms", "2000");
      // 自动提交offset到zookeeper的时间间隔
      properties.setProperty("auto.commit.interval.ms", "3000");
      // 当zookeeper中没有初始的offset时候的处理方式 。smallest ：重置为最小值 largest:重置为最大值 anything else：抛出异常
      properties.setProperty("auto.offset.reset", "largest");
    }

    ZookeeperOperator zookeeper = new ZookeeperOperator(
        properties.getProperty("zookeeper.connect"));

    GroupOperator groupOperator = new GroupOperator(topic, group, whichTime,
        zookeeper);
    return groupOperator.retryResetGroupOffset();
  }
}

public class GroupOperator {
  private static final Logger LOG = LoggerFactory
      .getLogger(GroupOperator.class);

  private List<String> m_replicaBrokers;
  private ZookeeperOperator zookeeper;
  private String topic;
  private List<String> seeds;
  private int port;
  private long whichTime;
  private int partitionNum;
  private String group;
  private int retryNum = 5;

  /**
   * 初始化
   *
   * @param topic
   * @param whichTime timestamp(13位)/-1(latest)/-2(earliest)
   * @throws java.io.IOException
   */
  public GroupOperator(String topic, String group, long whichTime,  ZookeeperOperator zookeeper)
      throws IOException {
    this.topic = topic;
    this.group = group;
    this.whichTime = whichTime;
    m_replicaBrokers = new ArrayList<String>();
    seeds = KafkaConf.getBrokerHost();
    port = Integer.parseInt(KafkaConf.BROKERPORT);
    this.zookeeper = zookeeper;
    partitionNum = zookeeper.readTopicChildren(topic);
  }

  /**
   * 将zookeeper中该group对应该topic下的所有分区的offset恢复为所希望的时间
   */
  public boolean resetGroupOffset() {
    List<Long> offsetList = new ArrayList<Long>();
    for (int partition = 0; partition < partitionNum; partition++) {
      long offset = getOffset(partition);
      if (offset == -1) {
        LOG.error("Failed to get offset of " + group + " with partition:"
            + partition);
        return false;
      } else {
        offsetList.add(offset);
      }
    }
    for (int partition = 0; partition < partitionNum; partition++) {
      boolean isSuccess = zookeeper
          .setTopicGroupOffset(topic, group, String.valueOf(partition),
              String.valueOf(offsetList.get(partition)));
      if (!isSuccess) {
        LOG.error("Failed to reset offset of topic:" + topic + " group:" + group
            + " partition" + partition + " value:" + offsetList.get(partition));
        return false;
      }
    }
    return true;
  }

  public boolean retryResetGroupOffset() {
    for (int retry = 0; retry < retryNum; retry++) {
      if (resetGroupOffset()) {
        return true;
      }
    }
    return false;
  }

  /**
   * 获取该partition要恢复时间的offset
   *
   * @param partition
   * @return
   */
  public long getOffset(int partition) {
    // find the meta data about the topic and partition we are interested in
    PartitionMetadata metadata = findLeader(partition);
    if (metadata == null) {
      LOG.error("Can't find metadata for Topic and Partition. Exiting");
      return -1;
    }
    if (metadata.leader() == null) {
      LOG.error("Can't find Leader for Topic and Partition. Exiting");
      return -1;
    }
    String leadBroker = metadata.leader().host();
    String clientName = "Client_" + topic + "_" + partition;

    SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000,
        64 * 1024, clientName);
    long readOffset = getAssignedOffset(consumer, partition, clientName);
    if (consumer != null) {
      consumer.close();
    }
    return readOffset;
  }

  /**
   * Finding the Lead Broker for a Topic and Partition
   *
   * @param a_partition 分区id，从0开始
   * @return
   */
  private PartitionMetadata findLeader(int a_partition) {
    PartitionMetadata returnMetaData = null;
    loop:
    for (String seed : seeds) {
      SimpleConsumer consumer = null;
      try {
        consumer = new SimpleConsumer(seed, port, 100000, 64 * 1024,
            "leaderLookup");
        List<String> topics = Collections.singletonList(topic);
        TopicMetadataRequest req = new TopicMetadataRequest(topics);
        kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);

        List<TopicMetadata> metaData = resp.topicsMetadata();
        for (TopicMetadata item : metaData) {
          for (PartitionMetadata part : item.partitionsMetadata()) {
            if (part.partitionId() == a_partition) {
              returnMetaData = part;
              break loop;
            }
          }
        }
      } catch (Exception e) {
        LOG.error("Error communicating with Broker [" + seed
            + "] to find Leader for [" + topic + ", " + a_partition
            + "] Reason: " + e);
      } finally {
        if (consumer != null)
          consumer.close();
      }
    }
    if (returnMetaData != null) {
      m_replicaBrokers.clear();
      for (kafka.cluster.Broker replica : returnMetaData.replicas()) {
        m_replicaBrokers.add(replica.host());
      }
    }
    return returnMetaData;
  }

  public long getAssignedOffset(SimpleConsumer consumer, int partition,
      String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic,
        partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo
        .put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
        requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
      System.out.println(
          "Error fetching data Offset Data the Broker. Reason: " + response
              .errorCode(topic, partition));
      return -1;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
  }

}