golang整合kafka之生产者和消费者案例

docker 本地环境搭建

docker-compose.yaml内容如下，创建完成后，使用docker-compose up -d 命令启动。

version: '3.8'
services:
  zookeeper:
    image: wurstmeister/zookeeper
    container_name: zookeeper
    restart: always
    ports:
      - "2181:2181"
  kafka:
    image: wurstmeister/kafka
    container_name: kafka
    restart: always
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 192.168.18.135
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"

生产者和消费者案例

• 生产者

import (
	"fmt"
	"github.com/IBM/sarama"
	"log"
	"sync"
	"testing"
	"time"
)

func TestProduceSyncMsg(t *testing.T) {
	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll          // 发送完数据之后需要leader和follow 都确认
	config.Producer.Partitioner = sarama.NewRandomPartitioner // 新选出一个分区
	config.Producer.Return.Successes = true                   // 成功交付的消息在success channel 中返回
	config.Producer.Return.Errors = true
	config.Producer.Retry.Max = 3
	cli, err := sarama.NewSyncProducer([]string{"127.0.0.1:9092"}, config)
	if err != nil {
		panic(err)
	}
	defer cli.Close()
	partition, offset, err := cli.SendMessage(&sarama.ProducerMessage{
		Topic: "first_topic",
		Value: sarama.StringEncoder("this is a test log"),
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("pid:%v offset:%v\n", partition, offset)
}

• 消费者

func TestConsumerMsg(t *testing.T) {
	consumer, err := sarama.NewConsumer([]string{"127.0.0.1:9092"}, nil)
	if err != nil {
		panic(err)
	}
	partitions, err := consumer.Partitions("first_topic")
	if err != nil {
		panic(err)
	}
	stop := make(chan struct{})
	wg := sync.WaitGroup{}
	for _, partition := range partitions {
		pConsumer, err := consumer.ConsumePartition("first_topic", partition, sarama.OffsetNewest)
		if err != nil {
			log.Fatalf("Failed to create partition consumer for partition %d: %s", partition, err)
		}
		wg.Add(1)
		go func(pc sarama.PartitionConsumer) {
			defer pc.Close()
			defer wg.Done()
			for {
				select {
				case message := <-pc.Messages():
					value := string(message.Value)
					log.Printf("Partitionid: %d; offset:%d, value: %s\n", message.Partition, message.Offset, value)
				case <-stop:
					return
				}
			}
		}(pConsumer)
	}

	time.Sleep(5 * time.Second)
	close(stop)
	wg.Wait()
}