从ip_queue到nfnetlink_queue(上)

从ip_queue到nfnetlink_queue(上)

本文档的Copyleft归yfydz所有，使用GPL发布，可以自由拷贝，转载，转载时请保持文档的完整性，严禁用于任何商业用途。
msn: yfydz_no1@hotmail.com
来源： http://yfydz.cublog.cn

1. 前言

在2.4内核中出现了ip_queue，用于将数据包从内核空间传递到用户空间，其不足之处是只能有一个应用程序接收内核数据。到了2.6.14以后，新增了nfnetlink_queue，理论上可最大可支持65536个应用程序接口，而且可以兼容ip_queue。

不过从内核到用户空间的通道还是只有一个，实际上netfilter对每个协议族也只有一个队列，这里说的65536个子队列的实现就象 802.1Q实现VLAN一样是在数据包中设置ID号来区分的，不同ID的包都通过相同的接口传输，只是在两端根据ID号进行了分类处理。

2. 用户空间

用户空间的支持库包括两个：libnfnetlink和libnetfilter_queue，后者需要前者支持，其源码可到netfilter网站上下载。

2.1 数据结构

2.1.1
/*linux_nfnetlink.h */
// 基本属性结构
struct nfattr
{
// 长度
u_int16_t nfa_len;
// leix
u_int16_t nfa_type;/* we use 15 bits for the type, and the highest
* bit to indicate whether the payload is nested */
} __attribute__ ((packed));

// nf基本消息结构
struct nfgenmsg {
u_int8_t nfgen_family;/* AF_xxx */
u_int8_t version;/* nfnetlink version */
u_int16_t res_id;/* resource id */
} __attribute__ ((packed));

/* nfnl是netfilter netlink的缩写 */
// nfnl回调结构
struct nfnl_callback
{
int (*call)(struct sock *nl, struct sk_buff *skb,
struct nlmsghdr *nlh, struct nfattr *cda[], int *errp);
u_int16_t attr_count;/* number of nfattr's */
};

// netfilter netlink子系统结构
struct nfnetlink_subsystem
{
const char *name;
__u8 subsys_id;/* nfnetlink subsystem ID */
__u8 cb_count;/* number of callbacks */
struct nfnl_callback *cb; /* callback for individual types */
};

2.1.2
/*libnfnetlink.h */
/* nfnl是netfilter netlink的缩写 */
// netfilter的netlink消息头结构
struct nfnlhdr {
struct nlmsghdr nlh;
struct nfgenmsg nfmsg;
};

// netfilter的netlink回调函数
struct nfnl_callback {
int (*call)(struct nlmsghdr *nlh, struct nfattr *nfa[], void *data);
void *data;
u_int16_t attr_count;
};

2.1.3
/* libnfnetlink.c */
// netfilter netlink子系统的handle结构
struct nfnl_subsys_handle {
struct nfnl_handle *nfnlh;
u_int32_tsubscriptions;
u_int8_tsubsys_id;
u_int8_tcb_count;
struct nfnl_callback *cb;/* array of callbacks */
};

#defineNFNL_MAX_SUBSYS16 /* enough for now */
// netfilter netlink的handle结构
struct nfnl_handle {
intfd; // socket
struct sockaddr_nllocal; // 本地netlink socket信息
struct sockaddr_nlpeer; // 对端的netlink socket信息
u_int32_tsubscriptions;
u_int32_tseq; // 序号
u_int32_tdump;
struct nlmsghdr *last_nlhdr; // 上一个消息
struct nfnl_subsys_handle subsys[NFNL_MAX_SUBSYS+1]; // 各子系统的handle
};

/*linux_nfnetlink_queue.h */
/* nfqnl是netfilter queue netlink的缩写 */

// nfqueue_netlink消息类型
enum nfqnl_msg_types {
NFQNL_MSG_PACKET,/* packet from kernel to userspace */
NFQNL_MSG_VERDICT,/* verdict from userspace to kernel */
NFQNL_MSG_CONFIG,/* connect to a particular queue */

NFQNL_MSG_MAX
};

// nfqueue_netlink消息包头结构
struct nfqnl_msg_packet_hdr {
u_int32_tpacket_id;/* unique ID of packet in queue */
u_int16_thw_protocol;/* hw protocol (network order) */
u_int8_thook;/* netfilter hook */
} __attribute__ ((packed));

// nfqueue_netlink消息包头的硬件地址信息
struct nfqnl_msg_packet_hw {
u_int16_thw_addrlen;
u_int16_t_pad;
u_int8_thw_addr[8];
} __attribute__ ((packed));

// nfqueue_netlink消息数据包时间戳，都是64位的
struct nfqnl_msg_packet_timestamp {
aligned_u64sec;
aligned_u64usec;
} __attribute__ ((packed));

// nfqueue_netlink属性类型
enum nfqnl_attr_type {
NFQA_UNSPEC,
NFQA_PACKET_HDR,
NFQA_VERDICT_HDR,/* nfqnl_msg_verdict_hrd */
NFQA_MARK,/* u_int32_t nfmark */
NFQA_TIMESTAMP,/* nfqnl_msg_packet_timestamp */
NFQA_IFINDEX_INDEV,/* u_int32_t ifindex */
NFQA_IFINDEX_OUTDEV,/* u_int32_t ifindex */
NFQA_IFINDEX_PHYSINDEV,/* u_int32_t ifindex */
NFQA_IFINDEX_PHYSOUTDEV,/* u_int32_t ifindex */
NFQA_HWADDR,/* nfqnl_msg_packet_hw */
NFQA_PAYLOAD,/* opaque data payload */

__NFQA_MAX
};
#define NFQA_MAX (__NFQA_MAX - 1)

// nfqueue_netlink数据包的裁定结果头结构
struct nfqnl_msg_verdict_hdr {
u_int32_t verdict;
u_int32_t id;
} __attribute__ ((packed));

// nfqueue_netlink消息的配置命令类型
enum nfqnl_msg_config_cmds {
NFQNL_CFG_CMD_NONE,
NFQNL_CFG_CMD_BIND, // 和队列绑定
NFQNL_CFG_CMD_UNBIND, // 取消和队列的绑定
NFQNL_CFG_CMD_PF_BIND, // 和协议族绑定
NFQNL_CFG_CMD_PF_UNBIND, // 取消和协议族的绑定
};

// nfqueue_netlink消息的配置命令结构
struct nfqnl_msg_config_cmd {
u_int8_tcommand;/* nfqnl_msg_config_cmds */
u_int8_t_pad;
u_int16_tpf;/* AF_xxx for PF_[UN]BIND */
} __attribute__ ((packed));

// nfqueue_netlink消息的配置模式类型
enum nfqnl_config_mode {
NFQNL_COPY_NONE, // 不拷贝
NFQNL_COPY_META, // 只拷贝头部信息
NFQNL_COPY_PACKET, // 拷贝整个数据包
};

// nfqueue_netlink消息的配置结构
struct nfqnl_msg_config_params {
u_int32_tcopy_range;
u_int8_tcopy_mode;/* enum nfqnl_config_mode */
} __attribute__ ((packed));

// nfqueue_netlink属性类型
enum nfqnl_attr_config {
NFQA_CFG_UNSPEC,
NFQA_CFG_CMD,/* nfqnl_msg_config_cmd */
NFQA_CFG_PARAMS,/* nfqnl_msg_config_params */
__NFQA_CFG_MAX
};
#define NFQA_CFG_MAX (__NFQA_CFG_MAX-1)

2.1.4

/* libnetfilter_queue.c */
// netfilter queue的handle结构
struct nfq_handle
{
struct nfnl_handle *nfnlh; // nf netlink handle
struct nfnl_subsys_handle *nfnlssh; // 子系统的handle
struct nfq_q_handle *qh_list; //
};

// netfilter queue的节点队列结构
struct nfq_q_handle
{
// 下一个节点
struct nfq_q_handle *next;
// 该队列的handle
struct nfq_handle *h;
// id号，每个queue有唯一ID，一共可支持65536个queue
u_int16_t id;

// typedef int nfq_callback(struct nfq_q_handle *gh, struct nfgenmsg *nfmsg,
// struct nfq_data *nfad, void *data);
// nf queue的回调函数
nfq_callback *cb;
// nf queue的回调函数输入数据指针
void *data;
};

struct nfq_data {
struct nfattr **data;
};

2.2 接口函数声明

2.2.1 libnfnetlink

实际上这些函数和宏都是被netfilter_queue的接口函数所包装，一般用户应用程序中不用直接调用这些函数或宏。

/* libnfnetlink.h */

extern int nfnl_fd(struct nfnl_handle *h);

/* get a new library handle */
extern struct nfnl_handle *nfnl_open(void);
extern int nfnl_close(struct nfnl_handle *);

extern struct nfnl_subsys_handle *nfnl_subsys_open(struct nfnl_handle *,
u_int8_t, u_int8_t,
unsigned int);
extern void nfnl_subsys_close(struct nfnl_subsys_handle *);

/* sending of data */
extern int nfnl_send(struct nfnl_handle *, struct nlmsghdr *);
extern int nfnl_sendmsg(const struct nfnl_handle *, const struct msghdr *msg,
unsigned int flags);
extern int nfnl_sendiov(const struct nfnl_handle *nfnlh,
const struct iovec *iov, unsigned int num,
unsigned int flags);
extern void nfnl_fill_hdr(struct nfnl_subsys_handle *, struct nlmsghdr *,
unsigned int, u_int8_t, u_int16_t, u_int16_t,
u_int16_t);
extern int nfnl_talk(struct nfnl_handle *, struct nlmsghdr *, pid_t,
unsigned, struct nlmsghdr *,
int (*)(struct sockaddr_nl *, struct nlmsghdr *, void *),
void *);

/* simple challenge/response */
extern int nfnl_listen(struct nfnl_handle *,
int (*)(struct sockaddr_nl *, struct nlmsghdr *, void *),
void *);

/* receiving */
extern ssize_t nfnl_recv(const struct nfnl_handle *h, unsigned char *buf, size_t len);
extern int nfnl_callback_register(struct nfnl_subsys_handle *,
u_int8_t type, struct nfnl_callback *cb);
extern int nfnl_callback_unregister(struct nfnl_subsys_handle *, u_int8_t type);
extern int nfnl_handle_packet(struct nfnl_handle *, char *buf, int len);

/* parsing */
extern struct nfattr *nfnl_parse_hdr(const struct nfnl_handle *nfnlh,
const struct nlmsghdr *nlh,
struct nfgenmsg **genmsg);
extern int nfnl_check_attributes(const struct nfnl_handle *nfnlh,
const struct nlmsghdr *nlh,
struct nfattr *tb[]);
extern struct nlmsghdr *nfnl_get_msg_first(struct nfnl_handle *h,
const unsigned char *buf,
size_t len);
extern struct nlmsghdr *nfnl_get_msg_next(struct nfnl_handle *h,
const unsigned char *buf,
size_t len);

#define nfnl_attr_present(tb, attr)/
(tb[attr-1])

#define nfnl_get_data(tb, attr, type)/
({type __ret = 0;/
if (tb[attr-1])/
__ret = *(type *)NFA_DATA(tb[attr-1]);/
__ret;/
})

#define nfnl_get_pointer_to_data(tb, attr, type)/
({type *__ret = NULL;/
if (tb[attr-1])/
__ret = NFA_DATA(tb[attr-1]);/
__ret;/
})

/* nfnl attribute handling functions */
extern int nfnl_addattr_l(struct nlmsghdr *, int, int, void *, int);
extern int nfnl_addattr16(struct nlmsghdr *, int, int, u_int16_t);
extern int nfnl_addattr32(struct nlmsghdr *, int, int, u_int32_t);
extern int nfnl_nfa_addattr_l(struct nfattr *, int, int, void *, int);
extern int nfnl_nfa_addattr16(struct nfattr *, int, int, u_int16_t);
extern int nfnl_nfa_addattr32(struct nfattr *, int, int, u_int32_t);
extern int nfnl_parse_attr(struct nfattr **, int, struct nfattr *, int);
#define nfnl_parse_nested(tb, max, nfa) /
nfnl_parse_attr((tb), (max), NFA_DATA((nfa)), NFA_PAYLOAD((nfa)))
#define nfnl_nest(nlh, bufsize, type) /
({struct nfattr *__start = NLMSG_TAIL(nlh);/
nfnl_addattr_l(nlh, bufsize, (NFNL_NFA_NEST | type), NULL, 0); /
__start; })
#define nfnl_nest_end(nlh, tail) /
({(tail)->nfa_len = (void *) NLMSG_TAIL(nlh) - (void *) tail; })

extern void nfnl_build_nfa_iovec(struct iovec *iov, struct nfattr *nfa,
u_int16_t type, u_int32_t len,
unsigned char *val);
extern unsigned int nfnl_rcvbufsiz(struct nfnl_handle *h, unsigned int size);

extern void nfnl_dump_packet(struct nlmsghdr *, int, char *);

2.2.2 libnetfilter_queue

/* libnetfilter_queue.h */
// 打开一个nfqueue的handle,返回NULL表示失败
extern struct nfq_handle *nfq_open(void);

// 打开nf netlink handle对应的nfqueue
extern struct nfq_handle *nfq_open_nfnl(struct nfnl_handle *nfnlh);

// 关闭nfqueue
extern int nfq_close(struct nfq_handle *h);

// 对nfqueue绑定协议族
extern int nfq_bind_pf(struct nfq_handle *h, u_int16_t pf);

// 对nfqueue取消协议族绑定
extern int nfq_unbind_pf(struct nfq_handle *h, u_int16_t pf);

// 建立具体的queue的handle,由num指定queue的序号, 返回NULL失败
extern struct nfq_q_handle *nfq_create_queue(struct nfq_handle *h,
u_int16_t num,
nfq_callback *cb,
void *data);
// 释放队列
extern int nfq_destroy_queue(struct nfq_q_handle *qh);

// 处理队列的数据包
extern int nfq_handle_packet(struct nfq_handle *h, char *buf, int len);

// 设置queue handle的数据拷贝模式
extern int nfq_set_mode(struct nfq_q_handle *qh,
u_int8_t mode, unsigned int len);

// 设置数据包的判定结果, id用于指定具体的包, buf和data_len用于传递修改后的数据
extern int nfq_set_verdict(struct nfq_q_handle *qh,
u_int32_t id,
u_int32_t verdict,
u_int32_t data_len,
unsigned char *buf);

// 设置数据包的mark值
extern int nfq_set_verdict_mark(struct nfq_q_handle *qh,
u_int32_t id,
u_int32_t verdict,
u_int32_t mark,
u_int32_t datalen,
unsigned char *buf);

/* message parsing function */
// 从缓冲区原始数据中返回消息头结构
extern struct nfqnl_msg_packet_hdr *
nfq_get_msg_packet_hdr(struct nfq_data *nfad);

// 获取数据的mark信息
extern u_int32_t nfq_get_nfmark(struct nfq_data *nfad);

extern int nfq_get_timestamp(struct nfq_data *nfad, struct timeval *tv);

/* return 0 if not set */
// 返回数据包进入网卡的索引号
extern u_int32_t nfq_get_indev(struct nfq_data *nfad);
// 返回数据包进入的物理网卡的索引号
extern u_int32_t nfq_get_physindev(struct nfq_data *nfad);
// 返回数据包发出网卡的索引号
extern u_int32_t nfq_get_outdev(struct nfq_data *nfad);
// 返回数据包发出的物理网卡的索引号
extern u_int32_t nfq_get_physoutdev(struct nfq_data *nfad);
// 返回数据包硬件地址
extern struct nfqnl_msg_packet_hw *nfq_get_packet_hw(struct nfq_data *nfad);

/* return -1 if problem, length otherwise */
// 获取数据包中载荷地址
extern int nfq_get_payload(struct nfq_data *nfad, char **data);

2.3 netfilter queue接口函数的实现

/* libnetfilter_queue.c */

// 删除一个queue节点
// 各个nfq_q_handle结构都是其nfq_handle中的qh_list链表中的一个节点
// 所以删除节点就是将其从链表中移出即可，该函数不进行内存释放操作
// 结构可表示如下：

// nfq_handle -> qh_list
// ^ |
// | V
// | nfq_q_handle -> nfq_q_handle -> ...
// | | |
// |______________|_______________|_________________

static void del_qh(struct nfq_q_handle *qh)
{
struct nfq_q_handle *cur_qh, *prev_qh = NULL;

for (cur_qh = qh->h->qh_list; cur_qh; cur_qh = cur_qh->next) {
if (cur_qh == qh) {
if (prev_qh)
prev_qh->next = qh->next;
else
qh->h->qh_list = qh->next;
return;
}
prev_qh = cur_qh;
}
}

// 把一个nfq_q_handle结构添加到链表中
static void add_qh(struct nfq_q_handle *qh)
{
qh->next = qh->h->qh_list;
qh->h->qh_list = qh;
}

// 根据ID号找nfq_q_handle节点
static struct nfq_q_handle *find_qh(struct nfq_handle *h, u_int16_t id)
{
struct nfq_q_handle *qh;

for (qh = h->qh_list; qh; qh = qh->next) {
if (qh->id == id)
return qh;
}
return NULL;
}

/* build a NFQNL_MSG_CONFIG message */
// 向netlink socket发送配置信息，该函数是static的，外部函数不可见
static int
__build_send_cfg_msg(struct nfq_handle *h, u_int8_t command,
u_int16_t queuenum, u_int16_t pf)
{
char buf[NFNL_HEADER_LEN
+NFA_LENGTH(sizeof(struct nfqnl_msg_config_cmd))];
struct nfqnl_msg_config_cmd cmd;
struct nlmsghdr *nmh = (struct nlmsghdr *) buf;

nfnl_fill_hdr(h->nfnlssh, nmh, 0, AF_UNSPEC, queuenum,
NFQNL_MSG_CONFIG, NLM_F_REQUEST|NLM_F_ACK);

cmd.command = command;
cmd.pf = htons(pf);
nfnl_addattr_l(nmh, sizeof(buf), NFQA_CFG_CMD, &cmd, sizeof(cmd));

return nfnl_talk(h->nfnlh, nmh, 0, 0, NULL, NULL, NULL);
}

// 接收netlink数据包，该函数是也static的，外部函数不可见
static int __nfq_rcv_pkt(struct nlmsghdr *nlh, struct nfattr *nfa[],
void *data)
{
struct nfgenmsg *nfmsg = NLMSG_DATA(nlh);
struct nfq_handle *h = data;
u_int16_t queue_num = ntohs(nfmsg->res_id);
// 根据ID找到nfq_q_handle
struct nfq_q_handle *qh = find_qh(h, queue_num);
struct nfq_data nfqa;

if (!qh)
return -ENODEV;

if (!qh->cb)
return -ENODEV;

nfqa.data = nfa;
// 调用nfq_q_handle的回调函数
return qh->cb(qh, nfmsg, &nfqa, qh->data);
}

// 固定的回调结构
static struct nfnl_callback pkt_cb = {
.call= &__nfq_rcv_pkt,
.attr_count= NFQA_MAX,
};

/* public interface */
// 返回nfq_handle的netlink handle
struct nfnl_handle *nfq_nfnlh(struct nfq_handle *h)
{
return h->nfnlh;
}

// 返回nfq_handle的netlink handle的netlink套接字
int nfq_fd(struct nfq_handle *h)
{
return nfnl_fd(nfq_nfnlh(h));
}

struct nfq_handle *nfq_open(void)
{
// 先打开netlink handle
struct nfnl_handle *nfnlh = nfnl_open();
struct nfq_handle *qh;

if (!nfnlh)
return NULL;
// 再调用nfq_open_nfnl()打开nf queue handle
qh = nfq_open_nfnl(nfnlh);
if (!qh)
nfnl_close(nfnlh);

return qh;
}

// 已存在netlink handle时打开nfq_handle
struct nfq_handle *nfq_open_nfnl(struct nfnl_handle *nfnlh)
{
struct nfq_handle *h;
int err;
// 分配内存
h = malloc(sizeof(*h));
if (!h)
return NULL;

memset(h, 0, sizeof(*h));
// 把nfq_handle和netlink handle连接起来
h->nfnlh = nfnlh;

// 打开NFNL_SUBSYS_QUEUE类型的子系统
h->nfnlssh = nfnl_subsys_open(h->nfnlh, NFNL_SUBSYS_QUEUE,
NFQNL_MSG_MAX, 0);
if (!h->nfnlssh) {
/* FIXME: nfq_errno */
goto out_free;
}

// 登记回调处理函数
pkt_cb.data = h;
err = nfnl_callback_register(h->nfnlssh, NFQNL_MSG_PACKET, &pkt_cb);
if (err < 0) {
nfq_errno = err;
goto out_close;
}

return h;
out_close:
nfnl_subsys_close(h->nfnlssh);
out_free:
free(h);
return NULL;
}

int nfq_close(struct nfq_handle *h)
{
int ret;
// 关闭子系统
nfnl_subsys_close(h->nfnlssh);
// 关闭netlink handle
ret = nfnl_close(h->nfnlh);
if (ret == 0)
free(h);
return ret;
}

/* bind nf_queue from a specific protocol family */
// 以下函数均是调用__build_send_cfg_msg()函数向内核发送消息命令
int nfq_bind_pf(struct nfq_handle *h, u_int16_t pf)
{
return __build_send_cfg_msg(h, NFQNL_CFG_CMD_PF_BIND, 0, pf);
}

/* unbind nf_queue from a specific protocol family */
int nfq_unbind_pf(struct nfq_handle *h, u_int16_t pf)
{
return __build_send_cfg_msg(h, NFQNL_CFG_CMD_PF_UNBIND, 0, pf);
}

/* bind this socket to a specific queue number */
// 生成一个号码为num的队列
struct nfq_q_handle *nfq_create_queue(struct nfq_handle *h,
u_int16_t num,
nfq_callback *cb,
void *data)
{
int ret;
struct nfq_q_handle *qh;

if (find_qh(h, num))
return NULL;
// 分配queue节点空间, 设置相应参数
qh = malloc(sizeof(*qh));

memset(qh, 0, sizeof(*qh));
qh->h = h;
qh->id = num;
qh->cb = cb;
qh->data = data;

ret = __build_send_cfg_msg(h, NFQNL_CFG_CMD_BIND, num, 0);
if (ret < 0) {
nfq_errno = ret;
free(qh);
return NULL;
}
// 添加到队列中
add_qh(qh);
return qh;
}

/* unbind this socket from a specific queue number */
// 释放队列
int nfq_destroy_queue(struct nfq_q_handle *qh)
{
int ret = __build_send_cfg_msg(qh->h, NFQNL_CFG_CMD_UNBIND, qh->id, 0);
if (ret == 0) {
del_qh(qh);
free(qh);
}

return ret;
}

int nfq_handle_packet(struct nfq_handle *h, char *buf, int len)
{
// 实际就是netlink处理包
return nfnl_handle_packet(h->nfnlh, buf, len);
}

int nfq_set_mode(struct nfq_q_handle *qh,
u_int8_t mode, u_int32_t range)
{
char buf[NFNL_HEADER_LEN
+NFA_LENGTH(sizeof(struct nfqnl_msg_config_params))];
struct nfqnl_msg_config_params params;
struct nlmsghdr *nmh = (struct nlmsghdr *) buf;

nfnl_fill_hdr(qh->h->nfnlssh, nmh, 0, AF_UNSPEC, qh->id,
NFQNL_MSG_CONFIG, NLM_F_REQUEST|NLM_F_ACK);

params.copy_range = htonl(range);
params.copy_mode = mode;
nfnl_addattr_l(nmh, sizeof(buf), NFQA_CFG_PARAMS, &params,
sizeof(params));

return nfnl_talk(qh->h->nfnlh, nmh, 0, 0, NULL, NULL, NULL);
}

static int __set_verdict(struct nfq_q_handle *qh, u_int32_t id,
u_int32_t verdict, u_int32_t mark, int set_mark,
u_int32_t data_len, unsigned char *data)
{
struct nfqnl_msg_verdict_hdr vh;
char buf[NFNL_HEADER_LEN
+NFA_LENGTH(sizeof(mark))
+NFA_LENGTH(sizeof(vh))];
struct nlmsghdr *nmh = (struct nlmsghdr *) buf;

struct iovec iov[3];
int nvecs;

/* This must be declared here (and not inside the data
* handling block) because the iovec points to this. */
struct nfattr data_attr;

memset(iov, 0, sizeof(iov));
// 设置裁定结果头
vh.verdict = htonl(verdict);
vh.id = htonl(id);

nfnl_fill_hdr(qh->h->nfnlssh, nmh, 0, AF_UNSPEC, qh->id,
NFQNL_MSG_VERDICT, NLM_F_REQUEST);

/* add verdict header */
nfnl_addattr_l(nmh, sizeof(buf), NFQA_VERDICT_HDR, &vh, sizeof(vh));

// 设置数据包的mark值
if (set_mark)
nfnl_addattr32(nmh, sizeof(buf), NFQA_MARK, mark);

iov[0].iov_base = nmh;
iov[0].iov_len = NLMSG_TAIL(nmh) - (void *)nmh;
nvecs = 1;

if (data_len) {
// 如果数据进行修改要传回内核,相应将数据添加到要发送到内核的数据向量中
nfnl_build_nfa_iovec(&iov[1], &data_attr, NFQA_PAYLOAD,
data_len, data);
nvecs += 2;
/* Add the length of the appended data to the message
* header. The size of the attribute is given in the
* nfa_len field and is set in the nfnl_build_nfa_iovec()
* function. */
nmh->nlmsg_len += data_attr.nfa_len;
}
// 向内核发送数据向量
return nfnl_sendiov(qh->h->nfnlh, iov, nvecs, 0);
}

int nfq_set_verdict(struct nfq_q_handle *qh, u_int32_t id,
u_int32_t verdict, u_int32_t data_len,
unsigned char *buf)
{
return __set_verdict(qh, id, verdict, 0, 0, data_len, buf);
}

int nfq_set_verdict_mark(struct nfq_q_handle *qh, u_int32_t id,
u_int32_t verdict, u_int32_t mark,
u_int32_t datalen, unsigned char *buf)
{
return __set_verdict(qh, id, verdict, mark, 1, datalen, buf);
}

/*************************************************************
* Message parsing functions
*************************************************************/
// 以下函数均是调用nfnl_get_pointer_to_data()和nfnl_get_data()函数获取
// 指定数据
struct nfqnl_msg_packet_hdr *nfq_get_msg_packet_hdr(struct nfq_data *nfad)
{
return nfnl_get_pointer_to_data(nfad->data, NFQA_PACKET_HDR,
struct nfqnl_msg_packet_hdr);
}

uint32_t nfq_get_nfmark(struct nfq_data *nfad)
{
return ntohl(nfnl_get_data(nfad->data, NFQA_MARK, u_int32_t));
}

int nfq_get_timestamp(struct nfq_data *nfad, struct timeval *tv)
{
struct nfqnl_msg_packet_timestamp *qpt;
qpt = nfnl_get_pointer_to_data(nfad->data, NFQA_TIMESTAMP,
struct nfqnl_msg_packet_timestamp);
if (!qpt)
return -1;

tv->tv_sec = __be64_to_cpu(qpt->sec);
tv->tv_usec = __be64_to_cpu(qpt->usec);

return 0;
}

/* all nfq_get_*dev() functions return 0 if not set, since linux only allows
* ifindex >= 1, see net/core/dev.c:2600 (in 2.6.13.1) */
u_int32_t nfq_get_indev(struct nfq_data *nfad)
{
return ntohl(nfnl_get_data(nfad->data, NFQA_IFINDEX_INDEV, u_int32_t));
}

u_int32_t nfq_get_physindev(struct nfq_data *nfad)
{
return ntohl(nfnl_get_data(nfad->data, NFQA_IFINDEX_PHYSINDEV, u_int32_t));
}

u_int32_t nfq_get_outdev(struct nfq_data *nfad)
{
return ntohl(nfnl_get_data(nfad->data, NFQA_IFINDEX_OUTDEV, u_int32_t));
}

u_int32_t nfq_get_physoutdev(struct nfq_data *nfad)
{
return ntohl(nfnl_get_data(nfad->data, NFQA_IFINDEX_PHYSOUTDEV, u_int32_t));
}

struct nfqnl_msg_packet_hw *nfq_get_packet_hw(struct nfq_data *nfad)
{
return nfnl_get_pointer_to_data(nfad->data, NFQA_HWADDR,
struct nfqnl_msg_packet_hw);
}

int nfq_get_payload(struct nfq_data *nfad, char **data)
{
*data = nfnl_get_pointer_to_data(nfad->data, NFQA_PAYLOAD, char);
if (*data)
return NFA_PAYLOAD(nfad->data[NFQA_PAYLOAD-1]);

return -1;
}

2.4 程序实例

/* nfqnl_test.c */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <netinet/in.h>
#include <linux/netfilter.h> /* for NF_ACCEPT */

#include <libnetfilter_queue/libnetfilter_queue.h>

/* returns packet id */
// 对数据包的处理函数, 本示例仅用于打印数据包的信息
static u_int32_t print_pkt (struct nfq_data *tb)
{
int id = 0;
struct nfqnl_msg_packet_hdr *ph;
u_int32_t mark,ifi;
int ret;
char *data;

// 提取数据包头信息,包括id,协议和hook点信息
ph = nfq_get_msg_packet_hdr(tb);
if (ph){
id = ntohl(ph->packet_id);
printf("hw_protocol=0x%04x hook=%u id=%u ",
ntohs(ph->hw_protocol), ph->hook, id);
}

// 获取数据包的mark值, 也就是内核skb的nfmark值
mark = nfq_get_nfmark(tb);
if (mark)
printf("mark=%u ", mark);

// 获取数据包的进入网卡的索引号
ifi = nfq_get_indev(tb);
if (ifi)
printf("indev=%u ", ifi);

// 获取数据包的发出网卡的索引号
ifi = nfq_get_outdev(tb);
if (ifi)
printf("outdev=%u ", ifi);

// 获取数据包载荷,data指针指向载荷,从实际的IP头开始
ret = nfq_get_payload(tb, &data);
if (ret >= 0)
printf("payload_len=%d ", ret);

fputc('/n', stdout);

return id;
}

// 回调函数定义, 基本结构是先处理包,然后返回裁定
static int cb(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg,
struct nfq_data *nfa, void *data)
{
// 数据包处理
u_int32_t id = print_pkt(nfa);
printf("entering callback/n");
// 设置裁定
return nfq_set_verdict(qh, id, NF_ACCEPT, 0, NULL);
}

int main(int argc, char **argv)
{
struct nfq_handle *h;
struct nfq_q_handle *qh;
struct nfnl_handle *nh;
int fd;
int rv;
char buf[4096];

printf("opening library handle/n");
// 打开nfq_handle
h = nfq_open();
if (!h) {
fprintf(stderr, "error during nfq_open()/n");
exit(1);
}

printf("unbinding existing nf_queue handler for AF_INET (if any)/n");
// 先解开和AF_INET的绑定
if (nfq_unbind_pf(h, AF_INET) < 0) {
fprintf(stderr, "error during nfq_unbind_pf()/n");
exit(1);
}

printf("binding nfnetlink_queue as nf_queue handler for AF_INET/n");
// 绑定到AF_INET
if (nfq_bind_pf(h, AF_INET) < 0) {
fprintf(stderr, "error during nfq_bind_pf()/n");
exit(1);
}

printf("binding this socket to queue '0'/n");
// 建立nfq_q_handle, 号码是0, 回调函数是cb
// 可建立多个queue,用不同的号码区分即可
qh = nfq_create_queue(h, 0, &cb, NULL);
if (!qh) {
fprintf(stderr, "error during nfq_create_queue()/n");
exit(1);
}

printf("setting copy_packet mode/n");
// 设置数据拷贝模式, 全包拷贝
if (nfq_set_mode(qh, NFQNL_COPY_PACKET, 0xffff) < 0) {
fprintf(stderr, "can't set packet_copy mode/n");
exit(1);
}

nh = nfq_nfnlh(h);
fd = nfnl_fd(nh);

// 从netlink套接字接收数据
while ((rv = recv(fd, buf, sizeof(buf), 0)) && rv >= 0) {
printf("pkt received/n");
// 处理数据,最终会调用到相应的回调函数
nfq_handle_packet(h, buf, rv);
}

printf("unbinding from queue 0/n");
// 释放队列
nfq_destroy_queue(qh);

#ifdef INSANE
/* normally, applications SHOULD NOT issue this command, since
* it detaches other programs/sockets from AF_INET, too ! */
printf("unbinding from AF_INET/n");
nfq_unbind_pf(h, AF_INET);
#endif

printf("closing library handle/n");
// 关闭nfq_handle
nfq_close(h);

exit(0);
}

2.4 包装libipq

可用netlink_queue包装libipq，ipq就相当于是号码为0的一个nfqueue而已：

/* libipq_compat.c */

struct ipq_handle *ipq_create_handle(u_int32_t flags, u_int32_t protocol)
{
int status;
struct ipq_handle *h;

h = (struct ipq_handle *)malloc(sizeof(struct ipq_handle));
if (h == NULL) {
ipq_errno = IPQ_ERR_HANDLE;
return NULL;
}

memset(h, 0, sizeof(struct ipq_handle));
// 打开ipq的nfqueue handle
h->nfqnlh = nfq_open();
if (!h->nfqnlh) {
ipq_errno = IPQ_ERR_SOCKET;
goto err_free;
}

// 绑定到PF_INET或PF_INET6
if (protocol == PF_INET)
status = nfq_bind_pf(h->nfqnlh, PF_INET);
else if (protocol == PF_INET6)
status = nfq_bind_pf(h->nfqnlh, PF_INET6);
else {
ipq_errno = IPQ_ERR_PROTOCOL;
goto err_close;
}

h->family = protocol;
if (status < 0) {
ipq_errno = IPQ_ERR_BIND;
goto err_close;
}

// 按号码0建立queue，无回调函数，数据包由ipq直接读后处理
h->qh = nfq_create_queue(h->nfqnlh, 0, NULL, NULL);
if (!h->qh) {
ipq_errno = IPQ_ERR_BIND;
goto err_close;
}

return h;

err_close:
nfq_close(h->nfqnlh);
err_free:
free(h);
return NULL;
}

/*
* No error condition is checked here at this stage, but it may happen
* if/when reliable messaging is implemented.
*/
int ipq_destroy_handle(struct ipq_handle *h)
{
if (h) {
nfq_close(h->nfqnlh);
free(h);
}
return 0;
}

int ipq_set_mode(const struct ipq_handle *h,
u_int8_t mode, size_t range)
{
return nfq_set_mode(h->qh, mode, range);
}

/*
* timeout is in microseconds (1 second is 1000000 (1 million) microseconds)
*
*/
// ipq_read包装得有点疑问，实际没进行接收操作，需要显式的recv接收数据包
// 现在的ipq_read只是对接收的数据进行解析
ssize_t ipq_read(const struct ipq_handle *h,
unsigned char *buf, size_t len, int timeout)
{
struct nfattr *tb[NFQA_MAX];
struct nlmsghdr *nlh = (struct nlmsghdr *)buf;
struct nfgenmsg *msg = NULL;
struct nfattr *nfa;

//return ipq_netlink_recvfrom(h, buf, len, timeout);

/* This really sucks. We have to copy the whole packet
* in order to build a data structure that is compatible to
* the old ipq interface... */

nfa = nfnl_parse_hdr(nfq_nfnlh(h->nfqnlh), nlh, &msg);
if (!msg || !nfa)
return 0;

if (msg->nfgen_family != h->family)
return 0;

nfnl_parse_attr(tb, NFQA_MAX, nfa, 0xffff);

return 0;
}

int ipq_message_type(const unsigned char *buf)
{
return ((struct nlmsghdr*)buf)->nlmsg_type;
}

int ipq_get_msgerr(const unsigned char *buf)
{
struct nlmsghdr *h = (struct nlmsghdr *)buf;
struct nlmsgerr *err = (struct nlmsgerr*)NLMSG_DATA(h);
return -err->error;
}

ipq_packet_msg_t *ipq_get_packet(const unsigned char *buf)
{
return NLMSG_DATA((struct nlmsghdr *)(buf));
}

int ipq_set_verdict(const struct ipq_handle *h,
ipq_id_t id,
unsigned int verdict,
size_t data_len,
unsigned char *buf)
{
return nfq_set_verdict(h->qh, id, verdict, data_len, buf);
}

/* Not implemented yet */
int ipq_ctl(const struct ipq_handle *h, int request, ...)
{
return 1;
}

char *ipq_errstr(void)
{
return ipq_strerror(ipq_errno);
}

void ipq_perror(const char *s)
{
if (s)
fputs(s, stderr);
else
fputs("ERROR", stderr);
if (ipq_errno)
fprintf(stderr, ": %s", ipq_errstr());
if (errno)
fprintf(stderr, ": %s", strerror(errno));
fputc('/n', stderr);
}