librdkafka C++ 实现手动提交offset，一条一条的消费

最近搞了一个东西，需要一条一条的消费kafka中的内容，处理完成之后提交offset。不能自动提交。于是在网上找了很多资料，包括询问AI。结果都不尽人意。

于是在总结了多方网络上的资料和AI及身边人的提醒之后，总结了一个实例如下。

我当时使用的版本是librdkafka1.6.1

重点

需要配置如下四个参数：

 // 设置是否需要自动提交- 这里为false， 这一点非常重要。高阶的kafkaconsumer的设置在 global conf
    // 低阶的在topic conf.
	if(_pConf->set("enable.auto.commit", "false", _errstr) != RdKafka::Conf::CONF_OK)
	{
		printf("set enable.auto.commit failed %s \n",_errstr.c_str());
		return false;
	}
    // 设置是否自动存储offset - 这里为false 这一点非常重要
	if(_pConf->set("enable.auto.offset.store", "false", _errstr) != RdKafka::Conf::CONF_OK)
	{
		printf("set enable.auto.offset.store failed %s\n",_errstr.c_str());
		return false;
	}
    // 设置每次拉取1条消息 - 重要 -这里保证了你是一条一条的拉取
	if(_pConf->set("max.poll.records", "1", _errstr) != RdKafka::Conf::CONF_OK)
    {
        printf("set max.poll.records failed %s\n",_errstr.c_str());
		return false;
    }
    // 设置消费者拉取消息的最大等待时间为30秒 - 这个根据自己的时间来定
  	if(_pConf->set("fetch.max.wait.ms", "30000", _errstr) != RdKafka::Conf::CONF_OK)
    {
        printf("set fetch.max.wait.ms failed %s\n",_errstr.c_str());
		return false;
    }

完整的代码如下

#include "rdkafkacpp.h"
#include <string>
#include <iostream>
#include <vector>
using namespace std;

/** 需要设置的东西我放在了外面**/
string _brokers = "192.168.12.23:9092,192.168.12.24:9092,192.168.12.25:9092"; // kafka brokers
string _groupID = "test_group_1"; // groupID
string _errstr;       // 用来输出错误信息
string _topicStr = "TEST_TOPIC_XYQ";     // topic 可以弄多个
std::vector<std::string> _vtopics;	// kafka消费者需要的topics的容器
RdKafka::Conf * _pConf = NULL;
RdKafka::KafkaConsumer*_consumer = NULL;//rdkafka 消费者 hight level ， 支持groupid

// 当有错误发生时就重启kafka消费者
static bool _gRun = true;
class CEventCb : public RdKafka::EventCb {
public:
	void event_cb(RdKafka::Event &event) {
		switch (event.type())
		{
		case RdKafka::Event::EVENT_ERROR:
			//LOG(ERROR) << "KafkaConsumer Event error " << event.str();
            printf("KafkaConsumer Event error :%s\n",event.str().c_str());
			_gRun = false;
			break;

		case RdKafka::Event::EVENT_STATS:
			//LOG(ERROR) << "KafkaConsumer stats error " << event.str();
            printf("KafkaConsumer Event stats :%s\n",event.str().c_str());
			break;

		case RdKafka::Event::EVENT_LOG:
			//LOG(ERROR) << "KafkaConsumer stats error " << event.fac() << "--" << event.str();
            printf("KafkaConsumer Event LOG  : %s --%s\n",event.fac().c_str() ,event.str().c_str());
			break;

		default:
			break;
		}
	}
};


bool InitKafka()
{
    CEventCb _ex_event_cb;// 事件回调
    // init kafka params
    _pConf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); // kafka GLOBAL
	RdKafka::Conf * _pTConf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC); // kafka TOPIC

    // 配置brokers
    if(_pConf->set("metadata.broker.list", _brokers, _errstr) !=  RdKafka::Conf::CONF_OK){
        printf("set metadata.broker.list failed %s \n",_errstr.c_str());
        return false;
    }
     // 配置回调
	if(_pConf->set("event_cb",&_ex_event_cb,_errstr)!= RdKafka::Conf::CONF_OK){
        printf("set event_cb failed %s \n",_errstr.c_str());
        return false;
    }
    // 配置groupID
    if(_pConf->set("group.id", _groupID, _errstr) != RdKafka::Conf::CONF_OK){
		printf("set group.id failed %s \n",_errstr.c_str());
		return false;
	}
    // 设置是否需要自动提交- 这里为false， 这一点非常重要。高阶的kafkaconsumer的设置在 global conf
    // 低阶的在topic conf.
	if(_pConf->set("enable.auto.commit", "false", _errstr) != RdKafka::Conf::CONF_OK){
		printf("set enable.auto.commit failed %s \n",_errstr.c_str());
		return false;
	}
    // 设置是否自动存储offset - 这里为false 这一点非常重要
	if(_pConf->set("enable.auto.offset.store", "false", _errstr) != RdKafka::Conf::CONF_OK){
		printf("set enable.auto.offset.store failed %s\n",_errstr.c_str());
		return false;
	}
    // 设置每次拉取1条消息 - 重要 -这里保证了你是一条一条的拉取
	if(_pConf->set("max.poll.records", "1", _errstr) != RdKafka::Conf::CONF_OK){
        printf("set max.poll.records failed %s\n",_errstr.c_str());
		return false;
    }
    // 设置消费者拉取消息的最大等待时间为30秒 - 这个根据自己的时间来定
  	if(_pConf->set("fetch.max.wait.ms", "30000", _errstr) != RdKafka::Conf::CONF_OK){
        printf("set fetch.max.wait.ms failed %s\n",_errstr.c_str());
		return false;
    }
    // 是否允许partition 消费到末尾的提示，这个不重要，只是为了让你知道某个partition消费到末尾了
    //conf->set("enable.partition.eof", "true", errstr);

    // 默认topic设置
	if(_pConf->set("default_topic_conf", _pTConf, _errstr) != RdKafka::Conf::CONF_OK){
		printf("set default_topic_conf failed %s \n",_errstr.c_str());
        return false;
	}
    // 用完了就删了
	delete _pTConf;
	_pTConf  = NULL;    
    return true;
}
void msgConsume(RdKafka::Message* message, void* opaque)
{
    if(message == NULL){
        return ;
    }
    int errcode = message->err();
    if (errcode == RdKafka::ERR_NO_ERROR)  //消费数据
    {
        const char* ptr = static_cast<const char*>(message->payload());
		int lenn = static_cast<int>(message->len());
        string str(ptr,lenn);
		if(lenn > 0){
			printf("topic:%s,partition:%d,msg:%s \n",message->topic_name().c_str(), message->partition(),str.c_str());
            // do something // 在这要怎么处理随便了。
        }

    }else if(errcode == RdKafka::ERR__TIMED_OUT){ // 超时了
        printf("consume time out \n");
    }else if(errcode == RdKafka::ERR__PARTITION_EOF){
        // 消费到了某个partion的末尾，不用管，大部分时间我们不关系这个
        printf("consume at partition end .\n");
    }else{
        // 这里是真正消费到什么错误了，然后把_gRun 弄成false .让程序退出了。
        printf("consume failed. ERROR: %s .\n",message->errstr().c_str() );
        _gRun = false;
    }
}
int main(){

    // init kafka params
    bool bInitResult = InitKafka();
    if(!bInitResult){
        printf("Init kafka failed .\n");
        return -1;
    }
    /*
     * Create consumer using accumulated global configuration.
     * 创建consumer, 使用你之前配置好的全局conf
     */
    _consumer = RdKafka::KafkaConsumer::create(_pConf, _errstr);
	if (!_consumer){
        std::cerr << "Failed to create consumer: " << _errstr << std::endl;
		return -1;
	}

    std::cout << "% Created consumer " << _consumer->name() << std::endl;
	//create topic
	/*
   	 * Subscribe to topics
   	 */
	_vtopics.push_back(_topicStr);		// topic1
	RdKafka::ErrorCode err = _consumer->subscribe(_vtopics);
	if (err){
		std::cout << "% Csubscribe topic failed:  " << RdKafka::err2str(err) << std::endl;
		return -1;
	}


    while(_gRun){
        RdKafka::Message *pMsg = _consumer->consume(5000);// 消费 (这里我的超时时间是5秒，根据自己的情况来定)
        // 处理流程
		msgConsume(pMsg,NULL);
        // 提交offset
        _consumer->commitSync(pMsg);
        // 回收内存
        if(pMsg != NULL){
			delete pMsg;
			pMsg = NULL;
		}
    }
    // 再提交一次-当有问题发生时
	err = _consumer->commitSync();
    std::cerr << "sync commit returned " << RdKafka::err2str(err) << std::endl;

    // 关闭消费者
    if(_consumer  != NULL){
		_consumer->close();		// 关闭消费者
        _consumer->poll(1000);
		delete _consumer;
        _consumer = NULL;
	}

    if(_pConf != NULL){
        delete _pConf;
        _pConf = NULL;
    }
    /*
   * Wait for RdKafka to decommission.
   * This is not strictly needed (when check outq_len() above), but
   * allows RdKafka to clean up all its resources before the application
   * exits so that memory profilers such as valgrind wont complain about
   * memory leaks.
   */
	RdKafka::wait_destroyed(5000);

    return 0;
}

// 我自己编译了 librdkafka1.6.1 然后把 下面四个文件拷贝出来放到了 /root/rdkafka 下面
// librdkafka.a  librdkafka++.a  rdkafkacpp.h  rdkafka.h 
//g++ -std=c++11 -I/root/rdkafka -L/root/rdkafka  -L//usr/local/lib testkafkaconsumerManual.cpp -lrdkafka++ -lrdkafka   -lsasl2 -lssl -lcrypto -lrt -lz