netlink学习小结

概述

内核态与用户态通信有多种方式，例如系统调用，ioctl，procfs/sysfs，uevent等，netlink也是内核态和用户态通信的一种重要方式，而且它提供全双工的工作模式，用户态和内核态都可主动向对方发送信息。

netlink接口简介

数据结构

用户态数据结构：
struct sockaddr_nl
{
  sa_family_t    nl_family;  /* AF_NETLINK   */
  unsigned short nl_pad;     /* zero         */
  __u32          nl_pid;     /* process pid */
  __u32          nl_groups;  /* mcast groups mask */
} nladdr;

struct nlmsghdr
{
  __u32 nlmsg_len;   /* Length of message */
  __u16 nlmsg_type;  /* Message type*/
  __u16 nlmsg_flags; /* Additional flags */
  __u32 nlmsg_seq;   /* Sequence number */
  __u32 nlmsg_pid;   /* Sending process PID */
};

struct msghdr {
    void *msg_name;        /* Address to send to/receive from.  */
    socklen_t msg_namelen;    /* Length of address data.  */

    struct iovec *msg_iov;    /* Vector of data to send/receive into.  */
    size_t msg_iovlen;        /* Number of elements in the vector.  */

    void *msg_control;        /* Ancillary data (eg BSD filedesc passing). */
    size_t msg_controllen;    /* Ancillary data buffer length.
                   !! The type should be socklen_t but the
                   definition of the kernel is incompatible
                   with this.  */

    int msg_flags;        /* Flags on received message.  */
};

不同内核版本，可能会有差异。

用户态

int socket(int domain, int type, int protocol)
The socket domain (address family) is AF_NETLINK, and the type of socket is either SOCK_RAW or SOCK_DGRAM, because netlink is a datagram-oriented service

struct sockaddr_nl nladdr;
When used with bind(), the nl_pid field of the sockaddr_nl can be filled with the calling process' own pid. The nl_pid serves here as the local address of this netlink socket. The application is responsible for picking a unique 32-bit integer to fill in nl_pid: nl_pid = getpid(). It uses the process ID of the application as nl_pid, which is a natural choice if, for the given netlink protocol type, only one netlink socket is needed for the process.
In scenarios where different threads of the same process want to have different netlink sockets opened under the same netlink protocol, Formula 2 can be used to generate the nl_pid: nl_pid = pthread_self() << 16 | getpid(); In this way, different pthreads of the same process each can have their own netlink socket for the same netlink protocol type. In fact, even within a single pthread it's possible to create multiple netlink sockets for the same protocol type. Developers need to be more creative, however, in generating a unique nl_pid, and we don't consider this to be a normal-use case.
bind(fd, (struct sockaddr*)&nladdr, sizeof(nladdr));

struct msghdr msg;
msg.msg_name = (void *)&(nladdr);
msg.msg_namelen = sizeof(nladdr);
struct iovec iov;
iov.iov_base = (void *)nlh;
iov.iov_len = nlh->nlmsg_len;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
sendmsg(fd, &msg, 0);
...
recvmsg(fd, &msg, 0);

内核态

struct sock *netlink_kernel_create(int unit, void (*input)(struct sock *sk, int len));
int netlink_unicast(struct sock *ssk, struct sk_buff *skb, u32 pid, int nonblock);
void netlink_broadcast(struct sock *ssk, struct sk_buff *skb, u32 pid, u32 group, int allocation);
netlink_kernel_release

例程

用户态代码

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <sys/socket.h>
#include <linux/netlink.h>

#define NL_MON_TEST     (1 << 20)
#define NETLINK_TEST    11
#define STR_MAX_LEN     (64)

static char buf[STR_MAX_LEN] = {0};

int main() {
    int sock_fd;
    int sock_op;
    struct iovec iov;
    struct msghdr msg;
    struct nlmsghdr *nlh = NULL;
    struct sockaddr_nl src_addr, dest_addr;

    sock_fd = socket(PF_NETLINK, SOCK_RAW, NETLINK_TEST);
    if (0 >= sock_fd) {
        printf("socket failed, error: %s\n", strerror(errno));
        exit(10);
    }

    memset(&src_addr, 0, sizeof(src_addr));
    src_addr.nl_family = AF_NETLINK;
    src_addr.nl_pid = getpid();  /* self pid */
    src_addr.nl_groups = NL_MON_TEST;
    if (bind(sock_fd, (struct sockaddr *)&src_addr, sizeof(src_addr))) {
        printf("cannot bind netlink socket\n");
        exit(20);
    }

    // In my tests, without following 2 lines, msg from kernel can't be received.
    sock_op = src_addr.nl_groups;
    if (setsockopt(sock_fd, 270/*SOL_NETLINK*/, NETLINK_ADD_MEMBERSHIP, &sock_op, sizeof(sock_op))) {
        printf("setsockopt failed\n");
        exit(30);
    }

    memset(&dest_addr, 0, sizeof(dest_addr));
    dest_addr.nl_family = AF_NETLINK;
    dest_addr.nl_pid = 0;
    dest_addr.nl_groups = NL_MON_TEST;
    //dest_addr.nl_groups = 0;

    nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(STR_MAX_LEN));
    if (NULL == nlh) {
        printf("failed to allocate memory for nlh\n");
        exit(30);
    }
    memset(nlh, 0, NLMSG_SPACE(STR_MAX_LEN));
    nlh->nlmsg_len   = NLMSG_SPACE(STR_MAX_LEN);
    nlh->nlmsg_pid   = getpid();
    /* without following lines, test_show_nl_msg in driver module won't be called */
    nlh->nlmsg_type  = 20;              //it should be not less than NLMSG_MIN_TYPE(it's defined to 0x10 in kernel4.19.90)
    nlh->nlmsg_flags = NLM_F_REQUEST;   //NLM_F_REQUEST is defined to 0x01 in kernel4.19.90. More bits can be ORed.
    //strcpy(NLMSG_DATA(nlh), "Hello from user space!");

    iov.iov_base = (void *)nlh;
    iov.iov_len = NLMSG_SPACE(STR_MAX_LEN);

#if 0
    memset(&msg, 0, sizeof(msg));
#else
    // without following 2 lines, sendmsg report error: No buffer space available.
    // memset can work too. Here we assign values one member by one member.
    msg.msg_flags = 0;
    msg.msg_controllen = 0;
#endif
    msg.msg_name = (void *)&dest_addr;
    msg.msg_namelen = sizeof(dest_addr);
    msg.msg_iov = &iov;
    msg.msg_iovlen = 1;

    for (int i = 0; i < 3; i++) {
        memset(buf, 0, sizeof(buf));
        snprintf(buf, sizeof(buf), "The %dth hello from user space to kernel", i);
        memcpy(NLMSG_DATA(nlh), buf, sizeof(buf));
        if (sendmsg(sock_fd, &msg, 0) < 0) {
            printf("sendmsg failed. Error: %s\n", strerror(errno));
        }
        printf("String sent to kernel: %s\n", buf);
        sleep(2);
    }

    printf("Waiting for message from kernel\n");

    /* Read message from kernel */
    recvmsg(sock_fd, &msg, 0);
    printf("Received message: %s\n", (char *)NLMSG_DATA(nlh));

    if (sock_fd != -1)
        close(sock_fd);

    if (nlh)
        free(nlh);

    return 0;
}

内核代码

#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <net/netlink.h>
#include <linux/rtnetlink.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/time.h>
#include <linux/rtc.h>

#define NL_MON_TEST     (1 << 20)
#define NETLINK_TEST    11      // NETLINK_CONNECTOR
#define STR_MAX_LEN     (64)

static char buf[STR_MAX_LEN] = {0};
static struct sock *test_nl_sock = NULL;
static struct task_struct *task = NULL;
static DECLARE_COMPLETION(test_comp);

static int test_task(void * unused)
{
    while (!kthread_should_stop()) {
        if ((NULL != test_nl_sock) && netlink_has_listeners(test_nl_sock, NL_MON_TEST)) {
            struct nlmsghdr *nlh = NULL;
            struct sk_buff  *skb = NULL;
            int size = sizeof(buf);

            memset(buf, 0, sizeof(buf));
            {
                struct timex txc;
                struct rtc_time tm;
                do_gettimeofday(&(txc.time));
                rtc_time_to_tm(txc.time.tv_sec,&tm);
                snprintf(buf, sizeof(buf), "UTC time :%d-%d-%d %d:%d:%d",tm.tm_year+1900,tm.tm_mon+1, tm.tm_mday,tm.tm_hour,tm.tm_min,tm.tm_sec);
            }

            skb = nlmsg_new(size, GFP_ATOMIC);
            if (skb) {
                nlh = nlmsg_put(skb, 0, 0, NLMSG_DONE, size, 0);
                if (nlh) {
                    memcpy(nlmsg_data(nlh), buf, size);
                    NETLINK_CB(skb).portid = 0;
                    NETLINK_CB(skb).dst_group = NL_MON_TEST;
                    netlink_broadcast(test_nl_sock, skb, 0, NL_MON_TEST, GFP_ATOMIC);
                    printk(KERN_INFO "String sent to userspace: %s\n", buf);
                } else {
                    kfree_skb(skb);
                }
            } else {
                printk(KERN_WARNING "func:%s: Cannot allocate skb\n", __func__);
            }
        } else {
            printk(KERN_WARNING "func:%s: Netlink socket has NO listeners\n", __func__);
        }

        msleep(1000 * 5);
    }
    complete(&test_comp);
    return 0;
}

static int test_show_nl_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
			       struct netlink_ext_ack *extack)
{
    printk(KERN_INFO "String received: %s\n", (char *)(nlmsg_data(nlh)));
    return 0;
}

static void test_netlink_rcv(struct sk_buff *skb)
{
#if 0   // following is used to check data received when test_show_nl_msg is not called for some reason.
    struct nlmsghdr *nlh = NULL;
    char *umsg = NULL;

    if(skb->len >= nlmsg_total_size(0)) {
        nlh = nlmsg_hdr(skb);
        umsg = nlmsg_data(nlh);
        printk(KERN_WARNING "func:%s, kernel recv from user: %s\n", __func__, umsg);
    }
#endif

    netlink_rcv_skb(skb, &test_show_nl_msg);
}

int test_init(void)
{
	struct netlink_kernel_cfg cfg = {
		.groups	= NL_MON_TEST,
        .input	= test_netlink_rcv,
	};

    printk(KERN_INFO "module init\n");

    test_nl_sock = netlink_kernel_create(&init_net, NETLINK_TEST, &cfg);
    if (!test_nl_sock) {
        printk(KERN_WARNING "func:%s, Failed to create netlink socket\n", __func__);
    }

    /*
     *Create one kernel thread to send msg to userspace
     */
	task = kthread_run(test_task, NULL, "csr test task");
	if (IS_ERR(task)) {
        printk(KERN_ERR "func:%s, unable to create kernel thread\n", __func__);
		return 1;
	}

    return 0;
}

void test_clean(void)
{
    kthread_stop(task);

    wait_for_completion(&test_comp);

    if (NULL != test_nl_sock) {
        netlink_kernel_release(test_nl_sock);
    }
    printk(KERN_INFO "module cleanup\n");
}

module_init(test_init);
module_exit(test_clean);
MODULE_LICENSE( "GPL" );

log:

kernel log：

[13632.929870] module init
[13632.930101] func:test_task: Netlink socket has NO listeners
[13638.070838] func:test_task: Netlink socket has NO listeners
[13643.190799] func:test_task: Netlink socket has NO listeners
[13647.181579] String received: The 0th hello from user space to kernel
[13648.311351] String sent to userspace: UTC time :2022-3-23 9:34:41
[13649.181729] String received: The 1th hello from user space to kernel
[13651.181869] String received: The 2th hello from user space to kernel
[13653.430742] func:test_task: Netlink socket has NO listeners
[13658.550693] func:test_task: Netlink socket has NO listeners
[13663.670728] module cleanup

用户态log：

String sent to kernel: The 0th hello from user space to kernel
String sent to kernel: The 1th hello from user space to kernel
String sent to kernel: The 2th hello from user space to kernel
Waiting for message from kernel
Received message: UTC time :2022-3-23 9:34:41

参考

Kernel Korner - Why and How to Use Netlink Socket | Linux Journal

https://man7.org/linux/man-pages/man2/setsockopt.2.html