用go 语言实现kafka 高水位

用go 语言实现kafka 高水位

网上找了好久，都没有人写。我自己封装了一下，挺好用的。
用到的包有
“github.com/Shopify/sarama”

func getHW(topic string, partitions []int32) (watermark map[int32]int64) {
	// 连接kafka
	conf := sarama.NewConfig()
	conf.Version = sarama.V2_6_0_0
	// err 这里就不处理了
	client, _:= sarama.NewClient(kafkaAddr, conf)
	defer client.Close()
	leadersMap := make(map[*sarama.Broker][]int32)

	for _, partition := range partitions {
		leader, err := client.Leader(topic, partition)
		if err != nil {
			return nil
		}
		leadersMap[leader] = append(leadersMap[leader], partition)
	}
	wg := sync.WaitGroup{}
	wg.Add(len(leadersMap))
	resultMap := make(chan map[int32]int64, len(leadersMap))

	for leader, partitions := range leadersMap {
		req := &sarama.OffsetRequest{Version: int16(1)}
		for _, partition := range partitions {
			req.AddBlock(topic, partition, int64(-1), int32(0))
		}

		go func(leader *sarama.Broker, req *sarama.OffsetRequest) {
			resp, err := leader.GetAvailableOffsets(req)
			if err != nil {
				return
			}
			waterMarksLeader := make(map[int32]int64)
			for partition, block := range resp.Blocks[topic] {
				waterMarksLeader[partition] = block.Offset
			}
			resultMap <- waterMarksLeader
			wg.Done()
		}(leader, req)
	}
	wg.Wait()
	close(resultMap)
	waterMarks := make(map[int32]int64)
	for resultMap := range resultMap {
		for partition, offset := range resultMap {
			waterMarks[partition] = offset
		}
	}
	return waterMarks
}