linux内核网络模块原理,linux网络模块的发展历程

2010/6/18：还在分析linux/net代码中，觉得在2.6的内核用来分析代码流程的确很麻烦。原理的东西一般都很好理解，有的技术PPT可能几页就能讲的很清楚了。比如GRE，理论上就是圆环套圆环(记得好像是那部电影？？)，可是分析2.6的代码就麻烦的很。还要理解tunnel。

所以突然想到，跟踪linux/net功能的发展历程来分析，应该还不错。

于是，先下载了0.96c的代码，发现这个版本的代码还只支持本地socket(AF_UNIX)，还不算真正意义上的net。

找来找去，搞了一个0.99.15。还算完善些。

这个版本支持INET，也就是IPV4。所以有值得分析的部分。

先看看吧，有了心得在写下来。希望借这个版本能够把路由策略(route.c)和TCP协议(tcp.c)很好的掌握以下。

1. 路由策略

在ip报文的发送和转发过程中，都涉及到路由的问题。在ip_build_header和ip_forward中，都调用了rt_route()。rt_route()这个函数根据传入的目的ip，返回一个rtable结构。这个结构就是我们经常看到的路由表，route这个命令能够查看到。

struct rtable {

struct rtable *rt_next;//路由表项是一个单项的链表，没有hash，完全是最简单的实现

unsigned long rt_dst; //目的地址

unsigned long rt_mask; //目的掩码

unsigned long rt_gateway;//目的网关

unsigned char rt_flags; //表示下一跳是个什么东西

unsigned char rt_metric;//可以理解为cost

short rt_refcnt;//引用计数

unsigned long rt_use;//命中次数？

unsigned short rt_mss, rt_mtu;//

struct device *rt_dev;//对应的设备，从哪个接口出去

};

在看一下rt_route的实现，简直是爽死了。就这么几行就搞定了2.6内核那一坨东西，不过功能肯定是不及：)

struct rtable * rt_route(unsigned long daddr, struct options *opt)

{

struct rtable *rt;

//遍历路由表

for (rt = rt_base; rt != NULL || early_out ; rt = rt->rt_next) {

//如果完全匹配或同一个子网，则选择这条路由。原来子网的用途在这里

if (!((rt->rt_dst ^ daddr) & rt->rt_mask))

break;

/* broadcast addresses can be special cases.. */

//如果你是一个广播地址，并且该路由表项允许广播，也就是这个路由表项制定的物理接口支持广播，表示命中这个表项。但有一个问题就是如果有多个接口，只能从第一个表项指定的接口进行广播了，因为break了。

if ((rt->rt_dev->flags & IFF_BROADCAST) &&

rt->rt_dev->pa_brdaddr == daddr)

break;

}

//如果是loopback，但没有loopback路由？奇怪，可能是不支持loopback接口吧

if (daddr == rt->rt_dev->pa_addr) {

if ((rt = rt_loopback) == NULL)

goto no_route;

}

rt->rt_use++;//命中次数

return rt;

no_route:

return NULL;

}

由此看来，早期内核对网络的支持非常简单，用来分析网络的实现也很容易。在看看路由表项是怎么添加进去的。rt_add()负责向路由表项添加路由表，而调用rt_add的地方有rt_ioctl和icmp.c中，icmp的路由重定向的支持。

也就是说，添加路由表项有两种方式：

a) inet_ioctl/rt_ioctl用来添加删除。

b) icmp协议的重定向报文。

在rt_add中，指定类型(flags)，目的地址，掩码，目的网关地址，对应的物理接口等参数。

如此，只要花费半小时，就分析完了route.c，效率高啊：)

2. TCP协议

linux内核在简单，在TCP协议的实现上也简单不了，所以看一下tcp.c，所以linux的tcp实现参考了BSD Socket，这在文件头中有说明。

先看一下tcp_prot结构

struct proto tcp_prot = {

sock_wmalloc,

sock_rmalloc,

sock_wfree,

sock_rfree,

sock_rspace,

sock_wspace,

tcp_close,

tcp_read,

tcp_write,

tcp_sendto,

tcp_recvfrom,

ip_build_header,

tcp_connect,

tcp_accept,

ip_queue_xmit,

tcp_retransmit,

tcp_write_wakeup,

tcp_read_wakeup,

tcp_rcv,

tcp_select,

tcp_ioctl,

NULL,

tcp_shutdown,

tcp_setsockopt,

tcp_getsockopt,

128,

0,

{NULL,},

"TCP"

};

2010/6/21:稍微考虑了一下，一个完整的C/S通信过程，最好先看Server端的处理流程，在分析Client端的处理。

Server端流程一般都是这样的情况

a)socket

b)bind

c)listen

d)accept //这个过程中一般会fork一个进程处理新建立的链接。

e)send/recv

f)close

好，先看一下socket()都做了哪些事情，sock_register函数注册了inet相关操作，对应函数inet_create()。代码就不贴了，这个函数实现比较简单，就是为sock结构分配内存并初始化。

大概过程是socket(user space)->sys_socketcall(kernel space)->sock_socket()->inet_create(前提是指定inet通信)。

这里在sock_socket中为socket结构分配内存和属于进程的fd，在inet_create中为sock分配内存。

接下来看一下inet_bind

static int

inet_bind(struct socket *sock, struct sockaddr *uaddr,

int addr_len)

{

struct sockaddr_in addr;

struct sock *sk, *sk2;

unsigned short snum;

int err;

sk = (struct sock *) sock->data;

if (sk == NULL) {

printk("Warning: sock->data = NULL: %d\n" ,__LINE__);

return(0);

}

/* check this error. */

if (sk->state != TCP_CLOSE) return(-EIO);

if (sk->num != 0) return(-EINVAL);

err=verify_area(VERIFY_READ, uaddr, addr_len);

if(err)

return err;

memcpy_fromfs(&addr, uaddr, min(sizeof(addr), addr_len));

snum = ntohs(addr.sin_port);

DPRINTF((DBG_INET, "bind sk =%X to port = %d\n", sk, snum));

sk = (struct sock *) sock->data;

/*

* We can't just leave the socket bound wherever it is, it might

* be bound to a privileged port. However, since there seems to

* be a bug here, we will leave it if the port is not privileged.

*/

if (snum == 0) { //如果没有指定bind的端口号，系统为你分配一个

snum = get_new_socknum(sk->prot, 0);

}

if (snum < PROT_SOCK && !suser()) return(-EACCES);

if (addr.sin_addr.s_addr!=0 && chk_addr(addr.sin_addr.s_addr)!=IS_MYADDR)

return(-EADDRNOTAVAIL);    /* Source address MUST be ours! */

if (chk_addr(addr.sin_addr.s_addr) || addr.sin_addr.s_addr == 0)

sk->saddr = addr.sin_addr.s_addr;

DPRINTF((DBG_INET, "sock_array[%d] = %X:\n", snum &(SOCK_ARRAY_SIZE -1),

sk->prot->sock_array[snum &(SOCK_ARRAY_SIZE -1)]));

/* Make sure we are allowed to bind here. */

cli();

outside_loop:

//这里以源端口号做了一个散列，在bind操作，找到符合要求的sk

for(sk2 = sk->prot->sock_array[snum & (SOCK_ARRAY_SIZE -1)];

sk2 != NULL; sk2 = sk2->next) {

#if     1    /* should be below! */

if (sk2->num != snum) continue;

/*    if (sk2->saddr != sk->saddr) continue; */

#endif

if (sk2->dead) {

destroy_sock(sk2);

goto outside_loop;

}

if (!sk->reuse) {

sti();

return(-EADDRINUSE);

}

if (sk2->num != snum) continue;        /* more than one */

if (sk2->saddr != sk->saddr) continue;    /* socket per slot ! -FB */

if (!sk2->reuse) {

sti();

return(-EADDRINUSE);

}

}

sti();

remove_sock(sk);

put_sock(snum, sk);

sk->dummy_th.source = ntohs(sk->num);

sk->daddr = 0;

sk->dummy_th.dest = 0;

return(0);

}

再看listen

static int

inet_listen(struct socket *sock, int backlog)

{

struct sock *sk;

sk = (struct sock *) sock->data;

if (sk == NULL) {

printk("Warning: sock->data = NULL: %d\n" ,__LINE__);

return(0);

}

/* We may need to bind the socket. */

if (sk->num == 0) {

sk->num = get_new_socknum(sk->prot, 0);

if (sk->num == 0) return(-EAGAIN);

put_sock(sk->num, sk);

sk->dummy_th.source = ntohs(sk->num);

}

/* We might as well re use these. */

sk->max_ack_backlog = backlog;//比较感兴趣的就是这里和下面的sk->state, backlog这个值如果不分析socket实现的话，是不会理解这个含义的。先说一下，这个数值的功能是能够同时接受多少个sync报文，也算是为了防止sync攻击的初级防御吧，回想一下tcp的三次握手，先是client端发送sync报文，如果server端在接收这个报文后，还没有创建新的连接，那么最多可以缓存backlog个sync报文，多出的部分直接扔掉

if (sk->state != TCP_LISTEN) {

sk->ack_backlog = 0;

sk->state = TCP_LISTEN;//当设置这个值为listen的时候，对应端口的tcp报文就可以被内核处理了

}

return(0);

}

现在可以看一下tcp_recv

case TCP_LISTEN: //如果是listen状态才处理sync报文

if (th->rst) {//这时收到rst报文，直接丢弃，不处理

kfree_skb(skb, FREE_READ);

release_sock(sk);

return(0);

}

if (th->ack) {//如果收到ack报文，说明可能server端down过，通知对方链接已经被重置

tcp_reset(daddr, saddr, th, sk->prot, opt,dev,sk->ip_tos,sk->ip_ttl);

kfree_skb(skb, FREE_READ);

release_sock(sk);

return(0);

}

if (th->syn) {//这个才是正确的报文

#if 0

if (opt->security != 0 || opt->compartment != 0) {

tcp_reset(daddr, saddr, th, prot, opt,dev);

release_sock(sk);

return(0);

}

#endif

/*

* Now we just put the whole thing including

* the header and saddr, and protocol pointer

* into the buffer. We can't respond until the

* user tells us to accept the connection.

*/

tcp_conn_request(sk, skb, daddr, saddr, opt, dev);//这个函数处理sync报文，下面分析

release_sock(sk);

return(0);

}

kfree_skb(skb, FREE_READ);

release_sock(sk);

return(0);

default://缺省的报文丢弃

if (!tcp_sequence(sk, th, len, opt, saddr,dev)) {

kfree_skb(skb, FREE_READ);

release_sock(sk);

return(0);

}

static void

tcp_conn_request(struct sock *sk, struct sk_buff *skb,

unsigned long daddr, unsigned long saddr,

struct options *opt, struct device *dev)

{

struct sk_buff *buff;

struct tcphdr *t1;

unsigned char *ptr;

struct sock *newsk;

struct tcphdr *th;

int tmp;

DPRINTF((DBG_TCP, "tcp_conn_request(sk = %X, skb = %X, daddr = %X, sadd4= %X, \n"

" opt = %X, dev = %X)\n",

sk, skb, daddr, saddr, opt, dev));

th = skb->h.th;

/* If the socket is dead, don't accept the connection. */

if (!sk->dead) {

sk->data_ready(sk,0);

} else {

DPRINTF((DBG_TCP, "tcp_conn_request on dead socket\n"));

tcp_reset(daddr, saddr, th, sk->prot, opt, dev, sk->ip_tos,sk->ip_ttl);

kfree_skb(skb, FREE_READ);

return;

}

/*

* Make sure we can accept more. This will prevent a

* flurry of syns from eating up all our memory.

*/

if (sk->ack_backlog >= sk->max_ack_backlog) {//这里，达到max，丢弃报文

kfree_skb(skb, FREE_READ);

return;

}

/*

* We need to build a new sock struct.

* It is sort of bad to have a socket without an inode attached

* to it, but the wake_up's will just wake up the listening socket,

* and if the listening socket is destroyed before this is taken

* off of the queue, this will take care of it.

*/

//复制一份新的sock

newsk = (struct sock *) kmalloc(sizeof(struct sock), GFP_ATOMIC);

if (newsk == NULL) {

/* just ignore the syn. It will get retransmitted. */

kfree_skb(skb, FREE_READ);

return;

}

DPRINTF((DBG_TCP, "newsk = %X\n", newsk));

memcpy((void *)newsk,(void *)sk, sizeof(*newsk));

newsk->wback = NULL;

newsk->wfront = NULL;

newsk->rqueue = NULL;

newsk->send_head = NULL;

newsk->send_tail = NULL;

newsk->back_log = NULL;

newsk->rtt = TCP_CONNECT_TIME << 3;

newsk->rto = TCP_CONNECT_TIME;

newsk->mdev = 0;

newsk->max_window = 0;

newsk->cong_window = 1;

newsk->cong_count = 0;

newsk->ssthresh = 0;

newsk->backoff = 0;

newsk->blog = 0;

newsk->intr = 0;

newsk->proc = 0;

newsk->done = 0;

newsk->partial = NULL;

newsk->pair = NULL;

newsk->wmem_alloc = 0;

newsk->rmem_alloc = 0;

newsk->max_unacked = MAX_WINDOW - TCP_WINDOW_DIFF;

newsk->err = 0;

newsk->shutdown = 0;

newsk->ack_backlog = 0;

newsk->acked_seq = skb->h.th->seq+1;

newsk->fin_seq = skb->h.th->seq;

newsk->copied_seq = skb->h.th->seq;

newsk->state = TCP_SYN_RECV;//设置状态，注意这个是newsk，原来的sk状态仍为LISTEN，这样在下一个tcp_rcv的处理中，会做进一步的处理

newsk->timeout = 0;

newsk->send_seq = jiffies * SEQ_TICK - seq_offset;//随机生成序列号算法

newsk->window_seq = newsk->send_seq;

newsk->rcv_ack_seq = newsk->send_seq;

newsk->urg =0;

newsk->retransmits = 0;

newsk->destroy = 0;

newsk->timer.data = (unsigned long)newsk;

newsk->timer.function = &net_timer;//协议用到的timer

newsk->dummy_th.source = skb->h.th->dest;

newsk->dummy_th.dest = skb->h.th->source;

/* Swap these two, they are from our point of view. */

newsk->daddr = saddr;

newsk->saddr = daddr;

put_sock(newsk->num,newsk);

newsk->dummy_th.res1 = 0;

newsk->dummy_th.doff = 6;

newsk->dummy_th.fin = 0;

newsk->dummy_th.syn = 0;

newsk->dummy_th.rst = 0;

newsk->dummy_th.psh = 0;

newsk->dummy_th.ack = 0;

newsk->dummy_th.urg = 0;

newsk->dummy_th.res2 = 0;

newsk->acked_seq = skb->h.th->seq + 1;

newsk->copied_seq = skb->h.th->seq;

/* Grab the ttl and tos values and use them */

newsk->ip_ttl=sk->ip_ttl;

newsk->ip_tos=skb->ip_hdr->tos;

/* use 512 or whatever user asked for */

/* note use of sk->user_mss, since user has no direct access to newsk */

if (sk->user_mss)

newsk->mtu = sk->user_mss;

else {

#ifdef SUBNETSARELOCAL

if ((saddr ^ daddr) & default_mask(saddr))

#else

if ((saddr ^ daddr) & dev->pa_mask)

#endif

newsk->mtu = 576 - HEADER_SIZE;

else

newsk->mtu = MAX_WINDOW;

}

/* but not bigger than device MTU */

newsk->mtu = min(newsk->mtu, dev->mtu - HEADER_SIZE);

/* this will min with what arrived in the packet */

tcp_options(newsk,skb->h.th);

//准备发送sync&ack的报文

buff = newsk->prot->wmalloc(newsk, MAX_SYN_SIZE, 1, GFP_ATOMIC);

if (buff == NULL) {

sk->err = -ENOMEM;

newsk->dead = 1;

release_sock(newsk);

kfree_skb(skb, FREE_READ);

return;

}

buff->mem_addr = buff;

buff->mem_len = MAX_SYN_SIZE;

buff->len = sizeof(struct tcphdr)+4;

buff->sk = newsk;

t1 =(struct tcphdr *) buff->data;

/* Put in the IP header and routing stuff. */

tmp = sk->prot->build_header(buff, newsk->saddr, newsk->daddr, &dev,

IPPROTO_TCP, NULL, MAX_SYN_SIZE,sk->ip_tos,sk->ip_ttl);

/* Something went wrong. */

if (tmp < 0) {

sk->err = tmp;

buff->free=1;

kfree_skb(buff,FREE_WRITE);

newsk->dead = 1;

release_sock(newsk);

skb->sk = sk;

kfree_skb(skb, FREE_READ);

return;

}

buff->len += tmp;

t1 =(struct tcphdr *)((char *)t1 +tmp);

memcpy(t1, skb->h.th, sizeof(*t1));

buff->h.seq = newsk->send_seq;

/* Swap the send and the receive. */

t1->dest = skb->h.th->source;

t1->source = newsk->dummy_th.source;

t1->seq = ntohl(newsk->send_seq++);

t1->ack = 1;//设置ACK标志

newsk->window = tcp_select_window(newsk);/*newsk->prot->rspace(newsk);*/

t1->window = ntohs(newsk->window);

t1->res1 = 0;

t1->res2 = 0;

t1->rst = 0;

t1->urg = 0;

t1->psh = 0;

t1->syn = 1;//设置SYNC标志

t1->ack_seq = ntohl(skb->h.th->seq+1);

t1->doff = sizeof(*t1)/4+1;

ptr =(unsigned char *)(t1+1);

ptr[0] = 2;

ptr[1] = 4;

ptr[2] = ((newsk->mtu) >> 8) & 0xff;

ptr[3] =(newsk->mtu) & 0xff;

tcp_send_check(t1, daddr, saddr, sizeof(*t1)+4, newsk);

newsk->prot->queue_xmit(newsk, dev, buff, 0);//在这里发送SYNC/ACK报文

reset_timer(newsk, TIME_WRITE /* -1 ? FIXME ??? */, TCP_CONNECT_TIME);

skb->sk = newsk;

/* Charge the sock_buff to newsk. */

sk->rmem_alloc -= skb->mem_len;

newsk->rmem_alloc += skb->mem_len;

skb_queue_tail(&sk->rqueue,skb);//同时把接收的这个报文上加入到sock接收队列中

sk->ack_backlog++;

release_sock(newsk);

}

这样，一个tcp握手就完成了，下面开始用户调用accept了

accept调用关系sock_accept->inet_accept->tcp_accept，先看一下tcp_accept

/* This will accept the next outstanding connection. */

static struct sock *

tcp_accept(struct sock *sk, int flags)

{

struct sock *newsk;

struct sk_buff *skb;

DPRINTF((DBG_TCP, "tcp_accept(sk=%X, flags=%X, addr=%s)\n",

sk, flags, in_ntoa(sk->saddr)));

/*

* We need to make sure that this socket is listening,

* and that it has something pending.

*/

if (sk->state != TCP_LISTEN) {

sk->err = EINVAL;

return(NULL);

}

/* avoid the race. */

cli();

sk->inuse = 1;

//等待接收的报文，当有报文时，就传给inet_accept，这个报文就是sync报文了，刚刚在tcp_recv函数中已经分析过了，同时state已经变为TCP_SYN_RECV了,在看inet_accept实现

while((skb = get_firstr(sk)) == NULL) {

if (flags & O_NONBLOCK) {

sti();

release_sock(sk);

sk->err = EAGAIN;

return(NULL);

}

release_sock(sk);

interruptible_sleep_on(sk->sleep);

if (current->signal & ~current->blocked) {

sti();

sk->err = ERESTARTSYS;

return(NULL);

}

sk->inuse = 1;

}

sti();

/* Now all we need to do is return skb->sk. */

newsk = skb->sk;

kfree_skb(skb, FREE_READ);

sk->ack_backlog--;

release_sock(sk);

return(newsk);

}

static int

inet_accept(struct socket *sock, struct socket *newsock, int flags)

{

struct sock *sk1, *sk2;

int err;

sk1 = (struct sock *) sock->data;

if (sk1 == NULL) {

printk("Warning: sock->data = NULL: %d\n" ,__LINE__);

return(0);

}

/*

* We've been passed an extra socket.

* We need to free it up because the tcp module creates

* it's own when it accepts one.

*/

if (newsock->data) kfree_s(newsock->data, sizeof(struct sock));

newsock->data = NULL;

if (sk1->prot->accept == NULL) return(-EOPNOTSUPP);

/* Restore the state if we have been interrupted, and then returned. */

if (sk1->pair != NULL ) {

sk2 = sk1->pair;

sk1->pair = NULL;

} else {

sk2 = sk1->prot->accept(sk1,flags);

if (sk2 == NULL) {

if (sk1->err <= 0)

printk("Warning sock.c:sk1->err <= 0. Returning non-error.\n");

err=sk1->err;

sk1->err=0;

return(-err);

}

}

newsock->data = (void *)sk2;

sk2->sleep = newsock->wait;

newsock->conn = NULL;

if (flags & O_NONBLOCK) return(0);

cli(); /* avoid the race. */

while(sk2->state == TCP_SYN_RECV) {//由于这个条件肯定是成立的，所以进入等待状态

interruptible_sleep_on(sk2->sleep);

if (current->signal & ~current->blocked) {

sti();

sk1->pair = sk2;

sk2->sleep = NULL;

newsock->data = NULL;

return(-ERESTARTSYS);

}

}

sti();

//在tcp_rcv中，如果是TCP_SYN_RECV，在接收到ACK后，状态变为TCP_ESTABLISHED

if (sk2->state != TCP_ESTABLISHED && sk2->err > 0) {

err = -sk2->err;

sk2->err=0;

destroy_sock(sk2);

newsock->data = NULL;

return(err);

}

newsock->state = SS_CONNECTED;//新的链接状态为SS_CONNECTED，这样一个TCP链接就建立完成了，可以进行SEND/RECV了。

return(0);

}