rdkafka线程过多_Kafka快速入门（十一）——RdKafka源码分析

Kafka快速入门(十一)——RdKafka源码分析

一、RdKafka C源码分析

1、Kafka OP队列

RdKafka将与Kafka Broke的交互、内部实现的操作都封装成Operator结构，然后放入OP处理队列里统一处理。Kafka OP队列是线程间通信的管道。

RdKafka队列定义在rdkafka_queue.h文件中，队列相关操作封装在rdsysqueue.h文件中。

(1)Kafka OP队列

typedef struct rd_kafka_q_s rd_kafka_q_t;

struct rd_kafka_q_s

{

mtx_t rkq_lock;// 队列操作加锁

cnd_t rkq_cond; // 队列中放入新元素时, 用条件变量唤醒相应等待线程

struct rd_kafka_q_s *rkq_fwdq; // Forwarded/Routed queue

struct rd_kafka_op_tailq rkq_q; // 放入队列的操作所存储的队列

int rkq_qlen; /* Number of entries in queue */

int64_t rkq_qsize; /* Size of all entries in queue */

int rkq_refcnt; // 引用计数

int rkq_flags; // 当前队列的状态

rd_kafka_t *rkq_rk;// 队列关联的Kafka Handler对象

struct rd_kafka_q_io *rkq_qio; //队列中放入新元素时,向fd写入数据唤醒等待线程

rd_kafka_q_serve_cb_t *rkq_serve; // 队列中的操作被执行时所执行的回调函数

void *rkq_opaque;

const char *rkq_name; // queue name

};

// Kafka Operator队列，对外接口

typedef struct rd_kafka_queue_s rd_kafka_queue_t;

struct rd_kafka_queue_s

{

rd_kafka_q_t *rkqu_q;// Kafka OP 队列

rd_kafka_t *rkqu_rk;// 队列关联的Kafka Handler

int rkqu_is_owner;

};

rd_kafka_queue_t *rd_kafka_queue_new (rd_kafka_t *rk)

{

rd_kafka_q_t *rkq;

rd_kafka_queue_t *rkqu;

rkq = rd_kafka_q_new(rk);

rkqu = rd_kafka_queue_new0(rk, rkq);

rd_kafka_q_destroy(rkq);

return rkqu;

}

创建OP队列

rd_kafka_queue_t *rd_kafka_queue_get_main (rd_kafka_t *rk)

{

return rd_kafka_queue_new0(rk, rk->rk_rep);

}

获取RdKafka与应用程序交互使用的OP队列

rd_kafka_queue_t *rd_kafka_queue_get_consumer (rd_kafka_t *rk) {

if (!rk->rk_cgrp)

return NULL;

return rd_kafka_queue_new0(rk, rk->rk_cgrp->rkcg_q);

}

获取消费者的OP队列

rd_kafka_queue_t *rd_kafka_queue_get_partition (rd_kafka_t *rk,

const char *topic,

int32_t partition) {

shptr_rd_kafka_toppar_t *s_rktp;

rd_kafka_toppar_t *rktp;

rd_kafka_queue_t *result;

if (rk->rk_type == RD_KAFKA_PRODUCER)

return NULL;

s_rktp = rd_kafka_toppar_get2(rk, topic,

partition,

0, /* no ua_on_miss */

1 /* create_on_miss */);

if (!s_rktp)

return NULL;

rktp = rd_kafka_toppar_s2i(s_rktp);

result = rd_kafka_queue_new0(rk, rktp->rktp_fetchq);

rd_kafka_toppar_destroy(s_rktp);

return result;

}

获取Topic的分区的OP队列

rd_kafka_op_t *rd_kafka_q_pop_serve (rd_kafka_q_t *rkq, rd_ts_t timeout_us,

int32_t version,

rd_kafka_q_cb_type_t cb_type,

rd_kafka_q_serve_cb_t *callback,

void *opaque);

处理OP队列中的一个OP操作，按version过滤的可处理OP，没有则等待，如果超时，函数退出。

int rd_kafka_q_serve (rd_kafka_q_t *rkq, int timeout_ms, int max_cnt,

rd_kafka_q_cb_type_t cb_type,

rd_kafka_q_serve_cb_t *callback,

void *opaque);

批量处理OP队列的OP

int rd_kafka_q_serve_rkmessages (rd_kafka_q_t *rkq, int timeout_ms,

rd_kafka_message_t **rkmessages,

size_t rkmessages_size);

处理RD_KAFKA_OP_FETCH OP操作

int rd_kafka_q_purge0 (rd_kafka_q_t *rkq, int do_lock);

#define rd_kafka_q_purge(rkq) rd_kafka_q_purge0(rkq, 1/*lock*/)

清除OP队列中的所有OP操作

rd_kafka_queue_t *rd_kafka_queue_get_background (rd_kafka_t *rk);

获取Kafka Handle的后台OP队列

2、Kafka OP操作

RaKafka OP操作封装在rdkafka_op.h文件中。

typedef enum

{

RD_KAFKA_OP_NONE, // 未指定类型

RD_KAFKA_OP_FETCH, // Kafka thread -> Application

RD_KAFKA_OP_ERR, // Kafka thread -> Application

RD_KAFKA_OP_CONSUMER_ERR, // Kafka thread -> Application

RD_KAFKA_OP_DR, // Kafka thread->Application:Produce message delivery report

RD_KAFKA_OP_STATS, // Kafka thread -> Application

RD_KAFKA_OP_OFFSET_COMMIT, // any -> toppar's Broker thread

RD_KAFKA_OP_NODE_UPDATE, // any -> Broker thread: node update

RD_KAFKA_OP_XMIT_BUF, // transmit buffer: any -> broker thread

RD_KAFKA_OP_RECV_BUF, // received response buffer: broker thr -> any

RD_KAFKA_OP_XMIT_RETRY, // retry buffer xmit: any -> broker thread

RD_KAFKA_OP_FETCH_START, // Application -> toppar's handler thread

RD_KAFKA_OP_FETCH_STOP, // Application -> toppar's handler thread

RD_KAFKA_OP_SEEK, // Application -> toppar's handler thread

RD_KAFKA_OP_PAUSE, // Application -> toppar's handler thread

RD_KAFKA_OP_OFFSET_FETCH, // Broker->broker thread: fetch offsets for topic

RD_KAFKA_OP_PARTITION_JOIN, // cgrp op:add toppar to cgrp，broker op:add toppar to broker

RD_KAFKA_OP_PARTITION_LEAVE, // cgrp op:remove toppar from cgrp，broker op:remove toppar from rkb

RD_KAFKA_OP_REBALANCE, // broker thread -> app:group rebalance

RD_KAFKA_OP_TERMINATE, // For generic use

RD_KAFKA_OP_COORD_QUERY, // Query for coordinator

RD_KAFKA_OP_SUBSCRIBE, // New subscription

RD_KAFKA_OP_ASSIGN, // New assignment

RD_KAFKA_OP_GET_SUBSCRIPTION,// Get current subscription Reuses u.subscribe

RD_KAFKA_OP_GET_ASSIGNMENT, // Get current assignment Reuses u.assign

RD_KAFKA_OP_THROTTLE, // Throttle info

RD_KAFKA_OP_NAME, // Request name

RD_KAFKA_OP_OFFSET_RESET, // Offset reset

RD_KAFKA_OP_METADATA, // Metadata response

RD_KAFKA_OP_LOG, // Log

RD_KAFKA_OP_WAKEUP, // Wake-up signaling

RD_KAFKA_OP_CREATETOPICS, // Admin: CreateTopics: u.admin_request

RD_KAFKA_OP_DELETETOPICS, // Admin: DeleteTopics: u.admin_request

RD_KAFKA_OP_CREATEPARTITIONS,// Admin: CreatePartitions: u.admin_request

RD_KAFKA_OP_ALTERCONFIGS, // Admin: AlterConfigs: u.admin_request

RD_KAFKA_OP_DESCRIBECONFIGS, // Admin: DescribeConfigs: u.admin_request

RD_KAFKA_OP_ADMIN_RESULT, // Admin API .._result_t

RD_KAFKA_OP_PURGE, // Purge queues

RD_KAFKA_OP_CONNECT, // Connect (to broker)

RD_KAFKA_OP_OAUTHBEARER_REFRESH, // Refresh OAUTHBEARER token

RD_KAFKA_OP_MOCK, // Mock cluster command

RD_KAFKA_OP_BROKER_MONITOR, // Broker state change

RD_KAFKA_OP_TXN, // Transaction command

RD_KAFKA_OP__END // 操作结束符

} rd_kafka_op_type_t;

rd_kafka_op_type_t枚举类型定义了RaKafka 所有OP操作类型。

typedef enum

{

RD_KAFKA_PRIO_NORMAL = 0, // 正常优先级

RD_KAFKA_PRIO_MEDIUM, // 中级

RD_KAFKA_PRIO_HIGH, // 高级

RD_KAFKA_PRIO_FLASH // 最高优先级：立即

} rd_kafka_prio_t;

rd_kafka_prio_t枚举类型定义了Kafka OP操作的所有优先级。

typedef enum

{

RD_KAFKA_OP_RES_PASS, // Not handled, pass to caller

RD_KAFKA_OP_RES_HANDLED, // Op was handled (through callbacks)

RD_KAFKA_OP_RES_KEEP, // Op已经被回调函数处理，但禁止被op_handle()销毁

RD_KAFKA_OP_RES_YIELD // Callback called yield

} rd_kafka_op_res_t;

rd_kafka_op_res_t枚举类型定义了OP被处理后的返回结果类型，

如果返回RD_KAFKA_OP_RES_YIELD，handler处理函数需要确定是否需要将OP重新入队列还是将OP销毁。

typedef enum

{

RD_KAFKA_Q_CB_INVALID, // 非法，未使用

RD_KAFKA_Q_CB_CALLBACK, // 基于OP触发回调函数

RD_KAFKA_Q_CB_RETURN, // 返回OP而不是触发回调函数

RD_KAFKA_Q_CB_FORCE_RETURN, // 无论是否触发回调函数都返回OP

RD_KAFKA_Q_CB_EVENT // 返回Event OP而不是触发回调函数

} rd_kafka_q_cb_type_t;

rd_kafka_q_cb_type_t枚举类型定义了OP队列中OP操作执行回调函数的所有类型。

OP队列执行回调函数类型定义如下：

typedef rd_kafka_op_res_t

(rd_kafka_q_serve_cb_t) (rd_kafka_t *rk,

struct rd_kafka_q_s *rkq,

struct rd_kafka_op_s *rko,

rd_kafka_q_cb_type_t cb_type, void *opaque);

OP回调函数定义如下：

typedef rd_kafka_op_res_t (rd_kafka_op_cb_t) (rd_kafka_t *rk,

rd_kafka_q_t *rkq,

struct rd_kafka_op_s *rko);

OP执行结果数据结构定义如下：

typedef struct rd_kafka_replyq_s

{

rd_kafka_q_t *q;// OP执行结果存储队列

int32_t version;// 版本

} rd_kafka_replyq_t;

Kafka OP数据结构定义如下：

struct rd_kafka_op_s

{

TAILQ_ENTRY(rd_kafka_op_s) rko_link;// 增加TAILQ字段

rd_kafka_op_type_t rko_type; // OP类型

rd_kafka_event_type_t rko_evtype;// Event类型

int rko_flags; // OP标识

int32_t rko_version;// 版本

rd_kafka_resp_err_t rko_err;//

int32_t rko_len; //

rd_kafka_prio_t rko_prio; // OP优先级

shptr_rd_kafka_toppar_t *rko_rktp;// 关联TopicPartition

rd_kafka_replyq_t rko_replyq;//

rd_kafka_q_serve_cb_t *rko_serve;// OP队列回调函数

void *rko_serve_opaque;// OP队列回调函数参数

rd_kafka_t *rko_rk;// Kafka Handle

rd_kafka_op_cb_t *rko_op_cb; // OP回调函数

union

{

struct

{

rd_kafka_buf_t *rkbuf;

rd_kafka_msg_t rkm;

int evidx;

} fetch;

struct

{

rd_kafka_topic_partition_list_t *partitions;

int do_free; // free .partitions on destroy()

} offset_fetch;

struct

{

rd_kafka_topic_partition_list_t *partitions;

void (*cb) (rd_kafka_t *rk,

rd_kafka_resp_err_t err,

rd_kafka_topic_partition_list_t *offsets,

void *opaque);

void *opaque;

int silent_empty; // Fail silently if there are no offsets to commit.

rd_ts_t ts_timeout;

char *reason;

} offset_commit;

struct

{

rd_kafka_topic_partition_list_t *topics;

} subscribe;

struct

{

rd_kafka_topic_partition_list_t *partitions;

} assign;

struct

{

rd_kafka_topic_partition_list_t *partitions;

} rebalance;

struct

{

char *str;

} name;

struct

{

int64_t offset;

char *errstr;

rd_kafka_msg_t rkm;

int fatal;

} err;

struct

{

int throttle_time;

int32_t nodeid;

char *nodename;

} throttle;

struct

{

char *json;

size_t json_len;

} stats;

struct

{

rd_kafka_buf_t *rkbuf;

} xbuf;

// RD_KAFKA_OP_METADATA

struct

{

rd_kafka_metadata_t *md;

int force; // force request regardless of outstanding metadata requests.

} metadata;

struct

{

shptr_rd_kafka_itopic_t *s_rkt;

rd_kafka_msgq_t msgq;

rd_kafka_msgq_t msgq2;

int do_purge2;

} dr;

struct

{

int32_t nodeid;

char nodename[RD_KAFKA_NODENAME_SIZE];

} node;

struct

{

int64_t offset;

char *reason;

} offset_reset;

struct

{

int64_t offset;

struct rd_kafka_cgrp_s *rkcg;

} fetch_start; // reused for SEEK

struct

{

int pause;

int flag;

} pause;

struct

{

char fac[64];

int level;

char *str;

} log;

struct

{

rd_kafka_AdminOptions_t options;

rd_ts_t abs_timeout; // Absolute timeout

rd_kafka_timer_t tmr; // Timeout timer

struct rd_kafka_enq_once_s *eonce; // 只入队列OP一次，用于触发Broker状态变化的OP请求

rd_list_t args; // Type depends on request, e.g. rd_kafka_NewTopic_t for CreateTopics

rd_kafka_buf_t *reply_buf; // Protocol reply

struct rd_kafka_admin_worker_cbs *cbs;

// Worker state

enum

{

RD_KAFKA_ADMIN_STATE_INIT,

RD_KAFKA_ADMIN_STATE_WAIT_BROKER,

RD_KAFKA_ADMIN_STATE_WAIT_CONTROLLER,

RD_KAFKA_ADMIN_STATE_CONSTRUCT_REQUEST,

RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE,

} state;

int32_t broker_id; // Requested broker id to communicate with.

// Application's reply queue

rd_kafka_replyq_t replyq;

rd_kafka_event_type_t reply_event_type;

} admin_request;</