Golang 使用 Sarama 消费 Kafka 集群消息 (1)

---

创建ConsumerGroup消费者组

消费者组允许多个消费者并行处理相同主题的消息，每个消费者负责处理不同的分区。创建时需要先实例化Sarama配置对象。

根据连接到的Kafka集群创建Sarama配置对象，需要注意Kafka集群版本以及消费者组的偏移量。

import (
	"github.com/IBM/sarama"
)

func NewSaramaConfig(version sarama.KafkaVersion, oldest bool) *sarama.Config {
	/**
	 * Construct a new Sarama configuration. 构造Sarama配置
	 */
	config := sarama.NewConfig()
	config.Version = version

	if oldest {
		config.Consumer.Offsets.Initial = sarama.OffsetOldest
	}

	return config
}

func main() {
    // 从最旧的便宜开始消费
    oldest := true

    version := sarama.V3_6_0_0
	// 指定kafka集群版本配置，如"0.10.2.1"
	// version, err := sarama.ParseKafkaVersion("0.10.2.1")
	if err != nil {
		log.Panicf("Error parsing Kafka version: %v", err)
	}

	config := NewSaramaConfig(version, oldest)

    // 指定集群brokers列表
    brokers := "host1:9092,host2:9092,host3:9092"
    // 指定group
    group := "sarama_consumergroup"

    client, err := sarama.NewConsumerGroup(strings.Split(brokers, ","), group, config)
	if err != nil {
		log.Panicf("Error creating consumer group client: %v", err)
	}

消费消息

消费消息时需要实现sarama.ConsumerGroupHandler接口，在ConsumeClaim函数中写消息的处理逻辑。

sarama.ConsumerGroupHandler接口实现

import (
	"github.com/IBM/sarama"
)

// 实现sarama.ConsumerGroupHandler的结构体
// 需要包含Setup、Cleanup、ConsumeClaim三个方法
type Consumer struct {
	ready chan bool
}

func (consumer *Consumer) Setup(sarama.ConsumerGroupSession) error {
	close(consumer.ready)
	return nil
}

func (consumer *Consumer) Cleanup(sarama.ConsumerGroupSession) error {
	return nil
}

// ConsumeClaim方法中包含消费消息的处理逻辑
func (consumer *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
	for {
		select {
		case message, ok := <-claim.Messages():
			if !ok {
				log.Printf("message channel was closed")
				return nil
			}
			log.Printf("从 topic = %s 收到消息，长度 length = %d\n", message.Topic, len(message.Value))
            // 自定义的处理函数
            // message.Value是[]byte类型
			ProcessMsg(message.Value)

			session.MarkMessage(message, "")
		case <-session.Context().Done():
			return nil
		}
	}
}

consumer := Consumer{
	ready: make(chan bool),
}

消费过程

消费过程在goroutine中运行，死循环调用client.Consume方法，把实例化的consumer对象作为参数传入

    topics := "mytopic"
	ctx, cancel := context.WithCancel(context.Background())

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			if err := client.Consume(ctx, strings.Split(topics, ","), &consumer); err != nil {
				if errors.Is(err, sarama.ErrOutOfBrokers) {
					// 处理 Kafka 连接超时错误
					log.Println("连接到 Kafka 超时，正在重试...")
					time.Sleep(5 * time.Second) // 等待5秒后重试
                    consumer.ready = make(chan bool)
					continue
				}
				if errors.Is(err, sarama.ErrClosedConsumerGroup) {
					return
				}
				log.Panicf("Error from consumer: %v", err)
			}
			if ctx.Err() != nil {
				return
			}
			consumer.ready = make(chan bool)
		}
	}()

	<-consumer.ready // Await till the consumer has been set up

	keepRunning := true
	for keepRunning {
		select {
		case <-ctx.Done():
			log.Println("terminating: context cancelled")
			keepRunning = false
	}
	cancel()
	wg.Wait()
	if err = client.Close(); err != nil {
		log.Panicf("Error closing client: %v", err)
	}