linux socket can code flow

使用socket can一年多的时间，因为与不同设备的通信会把驱动改成uart can spi can等兼容user space接口，抽时间梳理了一下socket can的code flow 

socket can 接收流程：

read(socket_fd)

[<c0648c64>] (__skb_wait_for_more_packets) from [<c0649d0c>] (__skb_recv_datagram+0x8c/0xa0)
[<c0649d0c>] (__skb_recv_datagram) from [<c0649d54>] (skb_recv_datagram+0x34/0x3c) 
[<c0649d54>] (skb_recv_datagram) from [<c074d3d0>] (raw_recvmsg+0x30/0x154)        
[<c074d3d0>] (raw_recvmsg) from [<c0639244>] (sock_read_iter+0x80/0xa8)        
[<c0639244>] (sock_read_iter) from [<c022ebd8>] (__vfs_read+0xf0/0x118) 
[<c022ebd8>] (__vfs_read) from [<c022ec90>] (vfs_read+0x90/0xfc)   
[<c022ec90>] (vfs_read) from [<c022f0bc>] (SyS_read+0x3c/0x74)       
[<c022f0bc>] (SyS_read) from [<c01070a0>] (ret_fast_syscall+0x0/0x48)    

user space的socket can读接口是sleep wait在网络层，哪里把它唤醒的呢？

我们先确认清楚它到底睡在什么地方，__skb_wait_for_more_packets，中调用prepare_to_wait_exclusive 睡在 sk_sleep(sock)上，实质是sk->sk_wq

int __skb_wait_for_more_packets(struct sock *sk, int *err, long *timeo_p,
				const struct sk_buff *skb)
{
	int error;
	DEFINE_WAIT_FUNC(wait, receiver_wake_function);

	prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
    schedule_timeout();
}

static inline wait_queue_head_t *sk_sleep(struct sock *sk)
{
	BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
	return &rcu_dereference_raw(sk->sk_wq)->wait;
}

sk->sk_wq，每一个sock初始化的都是都有自己的等待队列，can sock的初始can_create --> sock_init_data ，其中还有一处sk->sk_data_ready    =    sock_def_readable，这个函数是code flow的关键，卖个关子继续向下看

static int can_create(struct net *net, struct socket *sock, int protocol,
		      int kern)
{

	cp = can_get_proto(protocol);

	sock->ops = cp->ops;

	sk = sk_alloc(net, PF_CAN, GFP_KERNEL, cp->prot, kern);

	sock_init_data(sock, sk);
	sk->sk_destruct = can_sock_destruct;
	
}

void sock_init_data(struct socket *sock, struct sock *sk)
{
	sk_init_common(sk);

	sk->sk_allocation	=	GFP_KERNEL;
	sk->sk_rcvbuf		=	sysctl_rmem_default;
	sk->sk_sndbuf		=	sysctl_wmem_default;
	sk->sk_state		=	TCP_CLOSE;
    sk->sk_data_ready	=	sock_def_readable;
	sk_set_socket(sk, sock);

	if (sock) {
		sk->sk_type	=	sock->type;
		sk->sk_wq	=	sock->wq;
		sock->sk	=	sk;
		sk->sk_uid	=	SOCK_INODE(sock)->i_uid;
	} else {
		sk->sk_wq	=	NULL;
		sk->sk_uid	=	make_kuid(sock_net(sk)->user_ns, 0);
	}

知道接收线程睡的什么地方后就方便找唤醒的code flow了。

netif_receive_skb 把底层驱动收到的skb传递到网络层，是这个函数把user space唤醒起来收包的？

[<c074a564>] (can_receive) from [<c074a604>] (can_rcv+0x74/0x9c)
[<c074a604>] (can_rcv) from [<c0652948>] (__netif_receive_skb_core+0x8dc/0xae0)
[<c0652948>] (__netif_receive_skb_core) from [<c0657c30>] (netif_receive_skb_internal+0x64/0x128)
[<c0657c30>] (netif_receive_skb_internal) from [<bf000588>] (work_fn_transf+0x2fc/0x3dc [spican])

__netif_receive_skb_core --> can_receive --> can_rcv_filter --> deliver_skb --> can_packet->func --> can_rcv

can_rcv_filter 会讲收到的skb分发到不同的监听设备上去，比如 filter，错误帧，扩展帧等，user space有接口可以查看每个接收设备，可见接收所有报文的socket can有三个都是can0 设备上打开的socket

~ # cat /proc/net/can/
rcvlist_all  rcvlist_eff  rcvlist_err  rcvlist_fil  rcvlist_inv  rcvlist_sff  reset_stats  stats        version

~ # cat /proc/net/can/rcvlist_all 

receive list 'rx_all':
  (any: no entry)
  (can0: no entry)
  device   can_id   can_mask  function  userdata   matches  ident
   can0      000    00000000  c07f703c  c5ae7000   5284958  raw
   can0      000    00000000  c07f703c  c5ad0800   5287093  raw
   can0      000    00000000  c07f703c  c5ad0400   5287094  raw

 

static struct packet_type can_packet __read_mostly = {
     .type = cpu_to_be16(ETH_P_CAN),
     .func = can_rcv,
};

static void can_receive(struct sk_buff *skb, struct net_device *dev)
{

	/* update statistics */
	can_stats->rx_frames++;
	can_stats->rx_frames_delta++;

	/* deliver the packet to sockets listening on all devices */
	matches = can_rcv_filter(net->can.can_rx_alldev_list, skb);

	/* find receive list for this device */
	d = find_dev_rcv_lists(net, dev);
	if (d)
		matches += can_rcv_filter(d, skb);

	/* consume the skbuff allocated by the netdevice driver */
	consume_skb(skb);

}

static int can_rcv_filter(struct dev_rcv_lists *d, struct sk_buff *skb)
{
	struct receiver *r;
	int matches = 0;
	struct can_frame *cf = (struct can_frame *)skb->data;
	canid_t can_id = cf->can_id;


	if (can_id & CAN_ERR_FLAG) {
		/* check for error message frame entries only */
		hlist_for_each_entry_rcu(r, &d->rx[RX_ERR], list) {
			if (can_id & r->mask) {
				deliver(skb, r);
				matches++;
			}
		}
		return matches;
	}

	/* check for unfiltered entries */
	hlist_for_each_entry_rcu(r, &d->rx[RX_ALL], list) {
		deliver(skb, r);
		matches++;
	}

	/* check for can_id/mask entries */
	hlist_for_each_entry_rcu(r, &d->rx[RX_FIL], list) {
		if ((can_id & r->mask) == r->can_id) {
			deliver(skb, r);
			matches++;
		}
	}

	/* check for inverted can_id/mask entries */
	hlist_for_each_entry_rcu(r, &d->rx[RX_INV], list) {
		if ((can_id & r->mask) != r->can_id) {
			deliver(skb, r);
			matches++;
		}
	}

	/* check filterlists for single non-RTR can_ids */
	if (can_id & CAN_RTR_FLAG)
		return matches;

	if (can_id & CAN_EFF_FLAG) {
		hlist_for_each_entry_rcu(r, &d->rx_eff[effhash(can_id)], list) {
			if (r->can_id == can_id) {
				deliver(skb, r);
				matches++;
			}
		}
	} else {
		can_id &= CAN_SFF_MASK;
		hlist_for_each_entry_rcu(r, &d->rx_sff[can_id], list) {
			deliver(skb, r);
			matches++;
		}
	}

	return matches;
}

static inline void deliver(struct sk_buff *skb, struct receiver *r)
{
	r->func(skb, r->data);
	r->matches++;
}

receiver 是netif_receive_skb code flow的最后一棒，不同接收类型的设备的receiver分别是什么是何时注册到dev_rcv_lists中的呢？

default register
raw_bind --> raw_enable_allfilters --> raw_enable_filters --> can_rx_register
setsocketopt 
raw_setsockopt --> raw_enable_allfilters --> raw_enable_filters --> can_rx_register

对应userspace 设置socket 通信步骤中的 bind 与setsockopt函数注册的receiver

由如下code flow可知can_rx_register 注册的接收设备的接收函数是raw_rcv。

static int raw_enable_filters(struct net *net, struct net_device *dev,
			      struct sock *sk, struct can_filter *filter,
			      int count)
{

	for (i = 0; i < count; i++) {
		err = can_rx_register(net, dev, filter[i].can_id,
				      filter[i].can_mask,
				      raw_rcv, sk, "raw", sk);
		if (err) {
			/* clean up successfully registered filters */
			while (--i >= 0)
				can_rx_unregister(net, dev, filter[i].can_id,
						  filter[i].can_mask,
						  raw_rcv, sk);
			break;
		}
	}

	return err;
}


/**
 * can_rx_register - subscribe CAN frames from a specific interface
 * @dev: pointer to netdevice (NULL => subcribe from 'all' CAN devices list)
 * @can_id: CAN identifier (see description)
 * @mask: CAN mask (see description)
 * @func: callback function on filter match
 * @data: returned parameter for callback function
 * @ident: string for calling module identification
 * @sk: socket pointer (might be NULL)
 *
 * Description:
 *  Invokes the callback function with the received sk_buff and the given
 *  parameter 'data' on a matching receive filter. A filter matches, when
 *
 *          <received_can_id> & mask == can_id & mask
 *
 *  The filter can be inverted (CAN_INV_FILTER bit set in can_id) or it can
 *  filter for error message frames (CAN_ERR_FLAG bit set in mask).
 *
 *  The provided pointer to the sk_buff is guaranteed to be valid as long as
 *  the callback function is running. The callback function must *not* free
 *  the given sk_buff while processing it's task. When the given sk_buff is
 *  needed after the end of the callback function it must be cloned inside
 *  the callback function with skb_clone().
 *
 * Return:
 *  0 on success
 *  -ENOMEM on missing cache mem to create subscription entry
 *  -ENODEV unknown device
 */
int can_rx_register(struct net *net, struct net_device *dev, canid_t can_id,
		    canid_t mask, void (*func)(struct sk_buff *, void *),
		    void *data, char *ident, struct sock *sk)
{

	r = kmem_cache_alloc(rcv_cache, GFP_KERNEL);

	d = find_dev_rcv_lists(net, dev);
	if (d) {
		rl = find_rcv_list(&can_id, &mask, d);

		r->can_id  = can_id;
		r->mask    = mask;
		r->matches = 0;
		r->func    = func;
		r->data    = data;
		r->ident   = ident;
		r->sk      = sk;

		hlist_add_head_rcu(&r->list, rl);
		d->entries++;

	return err;
}
EXPORT_SYMBOL(can_rx_register);

raw_rcv函数把skb放到对应的socket skb queue中，然后调用了sk->sk_data_ready 即 sock_def_readable，该函数里面就会wakeup睡在sk->sk_wq等待队列上的线程。破案！！！

static void raw_rcv(struct sk_buff *oskb, void *data)
{

	/* clone the given skb to be able to enqueue it into the rcv queue */
	skb = skb_clone(oskb, GFP_ATOMIC);
	if (!skb)
		return;

	/*
	 *  Put the datagram to the queue so that raw_recvmsg() can
	 *  get it from there.  We need to pass the interface index to
	 *  raw_recvmsg().  We pass a whole struct sockaddr_can in skb->cb
	 *  containing the interface index.
	 */

	sock_skb_cb_check_size(sizeof(struct sockaddr_can));
	addr = (struct sockaddr_can *)skb->cb;
	memset(addr, 0, sizeof(*addr));
	addr->can_family  = AF_CAN;
	addr->can_ifindex = skb->dev->ifindex;

	/* add CAN specific message flags for raw_recvmsg() */
	pflags = raw_flags(skb);
	*pflags = 0;
	if (oskb->sk)
		*pflags |= MSG_DONTROUTE;
	if (oskb->sk == sk)
		*pflags |= MSG_CONFIRM;

	if (sock_queue_rcv_skb(sk, skb) < 0)
		kfree_skb(skb);
}

int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
{

	spin_lock_irqsave(&list->lock, flags);
	sock_skb_set_dropcount(sk, skb);
	__skb_queue_tail(list, skb);
	spin_unlock_irqrestore(&list->lock, flags);

	if (!sock_flag(sk, SOCK_DEAD))
		sk->sk_data_ready(sk);
	return 0;
}
EXPORT_SYMBOL(__sock_queue_rcv_skb);


static void sock_def_readable(struct sock *sk)
{
	struct socket_wq *wq;

	rcu_read_lock();
	wq = rcu_dereference(sk->sk_wq);
	if (skwq_has_sleeper(wq))
		wake_up_interruptible_sync_poll(&wq->wait, POLLIN | POLLPRI |
						POLLRDNORM | POLLRDBAND);
	sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
	rcu_read_unlock();
}

------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

spican socket can userspace can发送流程

[<c04ed484>] (spi_transfer_one_message) from [<c04edeec>] (__spi_pump_messages+0x6b0/0x6ec)
[<c04edeec>] (__spi_pump_messages) from [<c04ef25c>] (__spi_sync+0x1f4/0x268)
[<c04ef25c>] (__spi_sync) from [<c04ef2f4>] (spi_sync+0x24/0x3c)
[<c04ef2f4>] (spi_sync) from [<bf000470>] (can_spi_transfer.constprop.1+0x1e4/0x22c [spican])
[<bf000470>] (can_spi_transfer.constprop.1 [spican]) from [<bf0008e8>] (canspi_write+0x54/0x90 [spican])
[<bf0008e8>] (canspi_write [spican]) from [<bf000968>] (st_can_start_xmit+0x44/0x6dc [spican])
[<bf000968>] (st_can_start_xmit [spican]) from [<c0656a5c>] (dev_hard_start_xmit+0x108/0x268)
[<c0656a5c>] (dev_hard_start_xmit) from [<c067b19c>] (sch_direct_xmit+0x98/0x17c)
[<c067b19c>] (sch_direct_xmit) from [<c0656f60>] (__dev_queue_xmit+0x2e4/0x588)
[<c0656f60>] (__dev_queue_xmit) from [<c0749b24>] (can_send+0x160/0x1e8)
[<c0749b24>] (can_send) from [<c074d634>] (raw_sendmsg+0x140/0x1b4)
[<c074d634>] (raw_sendmsg) from [<c063a630>] (sock_sendmsg+0x14/0x24)
[<c063a630>] (sock_sendmsg) from [<c063a6c4>] (sock_write_iter+0x84/0xac)
[<c063a6c4>] (sock_write_iter) from [<c022ee20>] (__vfs_write+0xf0/0x11c)
[<c022ee20>] (__vfs_write) from [<c022efc0>] (vfs_write+0xb8/0x144)
[<c022efc0>] (vfs_write) from [<c022f130>] (SyS_write+0x3c/0x74)
[<c022f130>] (SyS_write) from [<c01070a0>] (ret_fast_syscall+0x0/0x48)

socket can 发送数据流向  拥塞控制qdisc，软中断net_tx_sofrirq(ksoftirqd软中断线程) 

源码路径net/can/af_can.c  net/core/*  net/sched/*  net/unix/*  drivers/net/can