librdkafka 生产者 消费者 使用

// install
git clone https://github.com/edenhill/librdkafka.git
cd librdkafka
./configure --prefix /usr
make
sudo make install
export PKG_CONFIG_PATH=/usr/lib/pkgconfig

//mabye
ln -s /usr/lib/librdkafka.so.1 /usr/lib64/librdkafka.so.1

// producer
func InitKafkaProducer() error {
	p, err := kafka.NewProducer(&kafka.ConfigMap{"bootstrap.servers": MainConfig.KafkaBroken})
	if err != nil {
		return err
	}
	producerMessage = make(chan JobST, 1000)

	log.Info("InitKafkaProducer start!")

	// Delivery report handler for produced messages
	go func() {
		for e := range p.Events() {
			switch ev := e.(type) {
			case *kafka.Message:
				if ev.TopicPartition.Error != nil {
					log.Debugf("InitKafkaProducer producer message:%s failed: %v", string(ev.Value), ev.TopicPartition)
				} else {
					log.Debugf("InitKafkaProducer producer message:%s to %v", string(ev.Value), ev.TopicPartition)
				}
			}
		}
	}()

	go func() {
		for job := range producerMessage {
			p.Produce(&kafka.Message{
				TopicPartition: kafka.TopicPartition{Topic: &job.topic, Partition: kafka.PartitionAny},
				Value:          []byte(job.message),
			}, nil)
		}
	}()
	return nil
}

func KafkaProducer(topic string, message string) error {
	var job JobST
	job.message = message
	job.topic = topic
	producerMessage <- job
	return nil
}

// consume 
/*
[[kafka_consume]]
	consume_from="kafka_consume01"
	zkpoints=""
	topic = "cc"
	group = "cc11"
	initoffset= -1
	process_timeout = 10
	commit_interval = 3
*/

func KafkaConsumeCommit() error {
	consume_from := "kafka_consume01"
	client, err := rpc.GetKafkaConsumeClient(consume_from)
	if err != nil || client == nil {
		log.Error("KafkaConsumeCommit kafkaConsume:", consume_from, ", get kafka consume client err:", err.Error())
		return errors.New("get kafka consume:" + consume_from + ", client err:" + err.Error())
	}
	log.Info("KafkaConsumeCommit start!")
	go func() {
		for {
			select {
			case message := <-commitMessage:
				if err := client.CommitUpto(message); err != nil {
					log.Error("KafkaConsumeCommit kafkaConsume:", consume_from, ", CommitUpto kafka message:", string(message.Value), " err:", err)
				} else {
					log.Debug("KafkaConsumeCommit kafkaConsume:", consume_from, ", CommitUpto kafka message :", string(message.Value))
				}
			}
		}
	}()
	return nil
}

func ReadKafka() {
	kafkaConsumeFrom := "kafka_consume01"
	client, err := rpc.GetKafkaConsumeClient(kafkaConsumeFrom)
	if err != nil || client == nil {
		log.Error("ReadKafka get kafka consume client err:", err.Error())
		return
	}

	log.Info("ReadKafka start!")
	for {
		taskLimit <- 1
		message := <-client.GetMessage()
		log.Debugf("ReadKafka GetMessage kafka message:%s", string(message.Value))
		go videoQuality(message)
	}
}