go操作kafka

1、下载包sarama

Go语言中连接kafka使用第三方库:github.com/Shopify/sarama。

go get github.com/Shopify/sarama@v1.19

1.2 注意事项
sarama v1.20之后的版本加入了zstd压缩算法，需要用到cgo，在Windows平台编译时会提示类似如下错误：

# github.com/DataDog/zstd
exec: "gcc":executable file not found in %PATH%

所以在Windows平台请使用v1.19版本的sarama。

2、实践

kafka/producer.go

package main

import (
	"kafka_test/kafka"
	"os/signal"
	"runtime"
	"syscall"
)

func main() {

	// 初始化生产生
	kafka.InitProducer("127.0.0.1:9092")

	// 关闭
	defer kafka.Close()

	// 发送测试消息
	kafka.Send("Test","This is Test Msg")
	kafka.Send("Test","Hello Guoke")

	signal.Ignore(syscall.SIGHUP)
	runtime.Goexit()
}

kafka/consumer.go

package kafka

import (
	"fmt"
	"strings"
	"github.com/Shopify/sarama"
)

var consumer sarama.Consumer

// 消费者回调函数
type ConsumerCallback func(data []byte)

// 初始化消费者
func InitConsumer(hosts string) {
	config := sarama.NewConfig()
	client, err := sarama.NewClient(strings.Split(hosts, ","), config)
	if err != nil {
		fmt.Println("unable to create kafka client: ", err)
	}

	consumer, err = sarama.NewConsumerFromClient(client)
	if err != nil {
		fmt.Println(err)
	}
}

// 消费者循环
func LoopConsumer(topic string, callback ConsumerCallback) {
	partitionConsumer, err := consumer.ConsumePartition(topic, 0, sarama.OffsetNewest)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer partitionConsumer.Close()

	for {
		msg := <-partitionConsumer.Messages()
		if callback != nil {
			callback(msg.Value)
		}
	}
}

func ConsumerClose() {
	if consumer!= nil {
		consumer.Close()
	}
}

consumer.go

package main

import (
	"fmt"
	"kafka_test/kafka"
	"os/signal"
	"runtime"
	"syscall"
)

func main() {
	// 初始化消费者
	kafka.InitConsumer("127.0.0.1:9092")

	// 监听
	go func() {
		kafka.LoopConsumer("Test", TopicCallBack)
	}()

	signal.Ignore(syscall.SIGHUP)
	runtime.Goexit()
}

func TopicCallBack(data []byte) {
	fmt.Println("kafka", "Test:"+string(data))
}

producer.go

package main

import (
	"kafka_test/kafka"
	"os/signal"
	"runtime"
	"syscall"
)

func main() {

	// 初始化生产生
	kafka.InitProducer("127.0.0.1:9092")

	// 关闭
	defer kafka.Close()

	// 发送测试消息
	kafka.Send("Test","This is Test Msg")
	kafka.Send("Test","Hello Guoke")

	signal.Ignore(syscall.SIGHUP)
	runtime.Goexit()
}