librdkafka的使用和介绍

librdkafka的使用介绍

librdkafka是kafka的c语言接口

 

下面简单的介绍一下其接口

    1、rd_kafka_conf_set设置全局配置

    2、rd_kafka_topic_conf_set设置topic配置

    3、rd_kafka_brokers_add设置broker地址，启动向broker发送消息的线程

    4、rd_kafka_new启动kafka主线程

    5、rd_kafka_topic_new建topic

   6、rd_kafka_produce使用本函数发送消息

    7、rd_kafka_poll调用回调函数

根据其github上的example我简单的封装一下producer：https://github.com/edenhill/librdkafka

编译静态：把上面的地中的代码下载到你的linux系统中

接下来就是传统的三部安装方法：

./configure

make

make install

编译好的静态和动态在src以及src-cpp目录下，我这里使用的是src目录下librdkafka.a  rdkafka.h  你也可以使用动态库 都是一样的 注意头文件的路径-I跟上头文件路径，我这里把静态库和头文件都是放在当前目录下  所以写 -I./

编译方法：g++ kafka_producer.cpp -o testkafka -I./ -lcurl  librdkafka.a -lpthread -lrt

写成c++的接口

#include <ctype.h>
#include <signal.h>
#include <string.h>
#include <unistd.h>
#include <stdlib.h>
#include <syslog.h>
#include <sys/time.h>
#include <errno.h>

#include "rdkafka.h"

const int PRODUCER_INIT_FAILED = -1;
const int PRODUCER_INIT_SUCCESS = 0;
const int PUSH_DATA_FAILED = -1;
const int PUSH_DATA_SUCCESS = 0;


static void logger(const rd_kafka_t *rk, int level,const char *fac, const char *buf) 
{
    struct timeval tv;
    gettimeofday(&tv, NULL);
    fprintf(stderr, "%u.%03u RDKAFKA-%i-%s: %s: %s\n",
        (int)tv.tv_sec, (int)(tv.tv_usec / 1000),
        level, fac, rk ? rd_kafka_name(rk) : NULL, buf);
}


class ProducerKafka
{
public:
	ProducerKafka(){};
	~ProducerKafka(){}

	int init_kafka(int partition, char *brokers, char *topic);
	int push_data_to_kafka(const char* buf, const int buf_len);
    void destroy();

private:
	int partition_;	
	
	//rd
	rd_kafka_t* handler_;
	rd_kafka_conf_t *conf_;
	
	//topic
	rd_kafka_topic_t *topic_;
	rd_kafka_topic_conf_t *topic_conf_;
};

int ProducerKafka::init_kafka(int partition, char *brokers, char *topic)
{
	char tmp[16]={0};
	char errstr[512]={0};	

	partition_ = partition;	

	/* Kafka configuration */
    conf_ = rd_kafka_conf_new();
	
	//set logger :register log function
	rd_kafka_conf_set_log_cb(conf_, logger);	
	
	/* Quick termination */
    snprintf(tmp, sizeof(tmp), "%i", SIGIO);
    rd_kafka_conf_set(conf_, "internal.termination.signal", tmp, NULL, 0);

	/*topic configuration*/
	topic_conf_ = rd_kafka_topic_conf_new();

	if (!(handler_  = rd_kafka_new(RD_KAFKA_PRODUCER, conf_, errstr, sizeof(errstr)))) 
	{
		fprintf(stderr, "*****Failed to create new producer: %s*******\n",errstr);
		return PRODUCER_INIT_FAILED;
	}

	rd_kafka_set_log_level(handler_, LOG_DEBUG);

	/* Add brokers */
    if (rd_kafka_brokers_add(handler_, brokers) == 0)
    {
        fprintf(stderr, "****** No valid brokers specified********\n");
 		return PRODUCER_INIT_FAILED;       
    }	
	

	/* Create topic */
    topic_ = rd_kafka_topic_new(handler_, topic, topic_conf_);
	
	return PRODUCER_INIT_SUCCESS;
}

void ProducerKafka::destroy()
{
	/* Destroy topic */
    rd_kafka_topic_destroy(topic_);

    /* Destroy the handle */
    rd_kafka_destroy(handler_);
}

int ProducerKafka::push_data_to_kafka(const char* buffer, const int buf_len)
{
	int ret;
	char errstr[512]={0};
	
	if(NULL == buffer)
		return 0;

	ret = rd_kafka_produce(topic_, partition_, RD_KAFKA_MSG_F_COPY, 
							(void*)buffer, (size_t)buf_len, NULL, 0, NULL);

	if(ret == -1)
	{
		fprintf(stderr,"****Failed to produce to topic %s partition %i: %s*****\n",
			rd_kafka_topic_name(topic_), partition_,
			rd_kafka_err2str(rd_kafka_errno2err(errno)));
	
		rd_kafka_poll(handler_, 0);
		return PUSH_DATA_FAILED;
	}
	
	fprintf(stderr, "***Sent %d bytes to topic:%s partition:%i*****\n",
		buf_len, rd_kafka_topic_name(topic_), partition_);

	rd_kafka_poll(handler_, 0);

	return PUSH_DATA_SUCCESS;
}

int main()
{
	char test_data[100];
	strcpy(test_data, "helloworld");

	ProducerKafka* producer = new ProducerKafka;
	if (PRODUCER_INIT_SUCCESS == producer->init_kafka(0, "192.168.1.108:9092", "chenxun"))
	{
		printf("producer init success\n");
	}
	else
	{
		printf("producer init failed\n");
		return 0;
	}
	
	while (fgets(test_data, sizeof(test_data), stdin)) {
		size_t len = strlen(test_data);
		if (test_data[len - 1] == '\n')
			test_data[--len] = '\0';
		if (strcmp(test_data, "end") == 0)
			break;
		if (PUSH_DATA_SUCCESS == producer->push_data_to_kafka(test_data, strlen(test_data)))
			printf("push data success %s\n", test_data);
		else
			printf("push data failed %s\n", test_data);
	}

	producer->destroy();
	
	return 0;	
}