用go 语言实现kafka 高水位
网上找了好久,都没有人写。我自己封装了一下,挺好用的。
用到的包有
“github.com/Shopify/sarama”
func getHW(topic string, partitions []int32) (watermark map[int32]int64) {
// 连接kafka
conf := sarama.NewConfig()
conf.Version = sarama.V2_6_0_0
// err 这里就不处理了
client, _:= sarama.NewClient(kafkaAddr, conf)
defer client.Close()
leadersMap := make(map[*sarama.Broker][]int32)
for _, partition := range partitions {
leader, err := client.Leader(topic, partition)
if err != nil {
return nil
}
leadersMap[leader] = append(leadersMap[leader], partition)
}
wg := sync.WaitGroup{}
wg.Add(len(leadersMap))
resultMap := make(chan map[int32]int64, len(leadersMap))
for leader, partitions := range leadersMap {
req := &sarama.OffsetRequest{Version: int16(1)}
for _, partition := range partitions {
req.AddBlock(topic, partition, int64(-1), int32(0))
}
go func(leader *sarama.Broker, req *sarama.OffsetRequest) {
resp, err := leader.GetAvailableOffsets(req)
if err != nil {
return
}
waterMarksLeader := make(map[int32]int64)
for partition, block := range resp.Blocks[topic] {
waterMarksLeader[partition] = block.Offset
}
resultMap <- waterMarksLeader
wg.Done()
}(leader, req)
}
wg.Wait()
close(resultMap)
waterMarks := make(map[int32]int64)
for resultMap := range resultMap {
for partition, offset := range resultMap {
waterMarks[partition] = offset
}
}
return waterMarks
}