golang kafka

kafka 连接

kafka Producer（生产者，放数据）

package main

import (
    "time"
    "fmt"
    "github.com/Shopify/sarama"
)

func main() {
    config := sarama.NewConfig()
    config.Producer.RequiredAcks = sarama.WaitForAll
    config.Producer.Partitioner = sarama.NewRandomPartitioner
    config.Producer.Return.Successes = true

    client, err := sarama.NewSyncProducer([]string{"192.168.1.108:9092"}, config)
    if err != nil {
        fmt.Println("producer close, err:", err)
        return
    }

    defer client.Close()

    var count int
    for {
        count++
        msg := &sarama.ProducerMessage{}
        msg.Topic = "zgl"
        line := fmt.Sprintf("[%d]this is a good test, my message is goo.", count)
        msg.Value = sarama.StringEncoder(line)



        pid, offset, err := client.SendMessage(msg)
        if err != nil {
            fmt.Println("send message failed, err:", err)
            return
        }

        fmt.Printf("pid:%v offset:%v\n", pid, offset)
        time.Sleep(time.Millisecond)

    }
}

kafka Consumer (消费者，取数据)

package main

import (
    "fmt"
    "strings"
    "sync"
    "time"

    "github.com/Shopify/sarama"
)

var (
    wg sync.WaitGroup
)

func main() {

    consumer, err := sarama.NewConsumer(strings.Split("192.168.14.4:9092", ","), nil)

    if err != nil {
        fmt.Println("failed, err:", err)
        return
    }

    partitionList, err := consumer.Partitions("nginx_log")
    if err != nil {
        fmt.Println("failed get partitions for nginx_log topic.")
        return
    }

    fmt.Println(partitionList)

    for partition := range partitionList {
        pc, err := consumer.ConsumePartition("nginx_log", int32(partition), sarama.OffsetNewest)
        if err != nil {
            fmt.Println("failed to start consumer")
            return
        }
        defer pc.AsyncClose()
        go func(sarama.PartitionConsumer) {

            for msg := range pc.Messages() {
                fmt.Printf("Partition:%d, Offset:%d, Key:%s, Value:%s\n", msg.Partition, msg.Offset, msg.Key, msg.Value)
                fmt.Println()
            }
        }(pc)
    }
    time.Sleep(time.Hour)
    consumer.Close()
}