1. ISO参考模型与TCP/IP参考模型
2. 在kernel中如何管理网络包(Network Packets)
2.1 定义Socket Buffers
Socket Buffers 由以下两部分组成:
1) Packet data: 它是在网络上传输的数据,其存储位置对应的PDU (Protocol Data Unit)
2) Management data: 当包在kernel中进行处理时,kernel需要另外一些数据,如pointer, timers等,它们是协议实体间交换信息的ICI(Interface Control Information)
Socket Buffer构成如下图所示:
在Kernel处理过程中,网络数据以Socket Buffer的形式存在。
当app通过socket发送数据时,socket将创建一个对应的socket buffer,并把需要发送的数据(payload)放于其中。当它通过各个协议层时,每一层的包头将被插入到payload的前面,在创建socket buffer时,为包头预留了足够空间。按此方案,payload被copy两次:
1) 从用户空间copy到kernel空间
2) 发送数据到network adapter
在协议层间传递时,其数据变化如下图所示:
Socket Buffer数据结构如下所示:
- struct sk_buff {
- /* These two members must be first. */
- struct sk_buff *next;
- struct sk_buff *prev;
- ktime_t tstamp;
- struct sock *sk;
- struct net_device *dev;
- /*
- * This is the control buffer. It is free to use for every
- * layer. Please put your private variables there. If you
- * want to keep them across layers you have to do a skb_clone()
- * first. This is owned by whoever has the skb queued ATM.
- */
- char cb[48] __aligned(8);
- unsigned long _skb_refdst;
- #ifdef CONFIG_XFRM
- struct sec_path *sp;
- #endif
- unsigned int len,
- data_len;
- __u16 mac_len,
- hdr_len;
- union {
- __wsum csum;
- struct {
- __u16 csum_start;
- __u16 csum_offset;
- };
- };
- __u32 priority;
- kmemcheck_bitfield_begin(flags1);
- __u8 local_df:1,
- cloned:1,
- ip_summed:2,
- nohdr:1,
- nfctinfo:3;
- __u8 pkt_type:3,
- fclone:2,
- ipvs_property:1,
- peeked:1,
- nf_trace:1;
- kmemcheck_bitfield_end(flags1);
- __be16 protocol;
- void (*destructor)(struct sk_buff *skb);
- #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
- struct nf_conntrack *nfct;
- #endif
- #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
- struct sk_buff *nfct_reasm;
- #endif
- #ifdef CONFIG_BRIDGE_NETFILTER
- struct nf_bridge_info *nf_bridge;
- #endif
- int skb_iif;
- #ifdef CONFIG_NET_SCHED
- __u16 tc_index; /* traffic control index */
- #ifdef CONFIG_NET_CLS_ACT
- __u16 tc_verd; /* traffic control verdict */
- #endif
- #endif
- __u32 rxhash;
- __u16 queue_mapping;
- kmemcheck_bitfield_begin(flags2);
- #ifdef CONFIG_IPV6_NDISC_NODETYPE
- __u8 ndisc_nodetype:2;
- #endif
- __u8 ooo_okay:1;
- kmemcheck_bitfield_end(flags2);
- /* 0/13 bit hole */
- #ifdef CONFIG_NET_DMA
- dma_cookie_t dma_cookie;
- #endif
- #ifdef CONFIG_NETWORK_SECMARK
- __u32 secmark;
- #endif
- union {
- __u32 mark;
- __u32 dropcount;
- };
- __u16 vlan_tci;
- sk_buff_data_t transport_header; //传输层头
- sk_buff_data_t network_header; //网络层头
- sk_buff_data_t mac_header; //链路层头
- /* These elements must be at the end, see alloc_skb() for details. */
- sk_buff_data_t tail;
- sk_buff_data_t end;
- unsigned char *head,
- *data;
- unsigned int truesize;
- atomic_t users;
- }
对Socket Buffers的操作分为以下三类:
1) 创建、释放和复制sokcet buffers
2) 操作sk_buff结构中的参数和指针,主要是改变包数据空间的操作
3) 管理socket buffer队列
2.1.1 创建、释放和复制sokcet buffers
其相关函数如下所示:
alloc_skb() include/linux/skbuff.h
dev_alloc_skb() net/core/skbuff.c
skb_copy() net/core/skbuff.c
skb_copy_expand() net/core/skbuff.c
skb_clone() net/core/skbuff.c
kfree_skb() net/core/skbuff.c
dev_kfree_skb() include/linux/skbuff.h
kfree_skbmem() net/core/skbuff.c
2.1.2 操作包数据空间
include/linux/skbuff.h
skb_get()
skb_unshare()
skb_put()
skb_push()
skb_pull()
skb_tailroom()
skb_headroom()
skb_realloc_headroom()
skb_reserve()
skb_trim()
skb_cow()
2.2 定义Socket-Buffer Queues
如果Socket Buffer不是正在被处理,则它被sk_buff_head管理,它通过双向链表进行管理,如下图所示:
- struct sk_buff_head {
- /* These two members must be first. */
- struct sk_buff *next;
- struct sk_buff *prev;
- __u32 qlen;
- spinlock_t lock;
- };
2.2.1 队列操作
include/linux/skbuff.h
skb_queue_head_init()
skb_queue_empty()
skb_queue_len()
2.2.2 队列中socket Buffer操作
include/linux/skbuff.h
skb_queue_head()
skb_queue_tail()
skb_dequeue()
skb_dequeue_tail()
skb_queue_purge()
skb_insert()
skb_append()
skb_unlink()
skb_peek()
skb_peek_tail()
3. Network Devices
在Linux系统中的网络架构,基于软件的协议(software-based protocol)与网络适配器(network adapters)间的接口通过network devices来实现。一个network-device接口需要满足以下要求:
1) 是network adapter的技术抽象
2) 提供统一的接口供协议实体访问
3.1 net_device定义
【网络设备】不同于【字符设备】和【块设备】,其主要区别如下:
1) 网络设备在/dev下不存在对应的设备名,即不可通过read和write进行读写操作
2) 网络设备基于包进行处理,且必须经过复杂协议的处理(如TCP和UDP)
net_device定义如下:
- struct net_device {
- /*
- * This is the first field of the "visible" part of this structure
- * (i.e. as seen by users in the "Space.c" file). It is the name
- * of the interface.
- */
- char name[IFNAMSIZ];
- struct pm_qos_request_list pm_qos_req;
- /* device name hash chain */
- struct hlist_node name_hlist;
- /* snmp alias */
- char *ifalias;
- // 硬件相关的信息
- /*
- * I/O specific fields
- * FIXME: Merge these and struct ifmap into one
- */
- unsigned long mem_end; /* shared mem end */
- unsigned long mem_start; /* shared mem start */
- unsigned long base_addr; /* device I/O address */
- unsigned int irq; /* device IRQ number */
- /*
- * Some hardware also needs these fields, but they are not
- * part of the usual set specified in Space.c.
- */
- unsigned long state;
- struct list_head dev_list;
- struct list_head napi_list;
- struct list_head unreg_list;
- /* currently active device features */
- u32 features;
- /* user-changeable features */
- u32 hw_features;
- /* user-requested features */
- u32 wanted_features;
- /* mask of features inheritable by VLAN devices */
- u32 vlan_features;
- /* Interface index. Unique device identifier */
- int ifindex;
- int iflink;
- struct net_device_stats stats;
- atomic_long_t rx_dropped; /* dropped packets by core network
- * Do not use this in drivers.
- */
- // 管理操作
- /* Management operations */
- const struct net_device_ops *netdev_ops; // 最终调用网络设备驱动方法
- const struct ethtool_ops *ethtool_ops;
- // 硬件头描述
- /* Hardware header description */
- const struct header_ops *header_ops;
- unsigned int flags; /* interface flags (a la BSD) */
- unsigned int priv_flags; /* Like 'flags' but invisible to userspace. */
- unsigned short gflags;
- unsigned short padded; /* How much padding added by alloc_netdev() */
- unsigned char operstate; /* RFC2863 operstate */
- unsigned char link_mode; /* mapping policy to operstate */
- unsigned char if_port; /* Selectable AUI, TP,..*/
- unsigned char dma; /* DMA channel */
- unsigned int mtu; /* interface MTU value */
- unsigned short type; /* interface hardware type */
- unsigned short hard_header_len; /* hardware hdr length */
- /* extra head- and tailroom the hardware may need, but not in all cases
- * can this be guaranteed, especially tailroom. Some cases also use
- * LL_MAX_HEADER instead to allocate the skb.
- */
- unsigned short needed_headroom;
- unsigned short needed_tailroom;
- // 接口地址信息
- /* Interface address info. */
- unsigned char perm_addr[MAX_ADDR_LEN]; /* permanent hw address */
- unsigned char addr_assign_type; /* hw address assignment type */
- unsigned char addr_len; /* hardware address length */
- unsigned short dev_id; /* for shared network cards */
- spinlock_t addr_list_lock;
- struct netdev_hw_addr_list uc; /* Unicast mac addresses */
- struct netdev_hw_addr_list mc; /* Multicast mac addresses */
- int uc_promisc;
- unsigned int promiscuity;
- unsigned int allmulti;
- // 协议相关的指针
- /* Protocol specific pointers */
- void *atalk_ptr; /* AppleTalk link */
- struct in_device __rcu *ip_ptr; /* IPv4 specific data */
- struct dn_dev __rcu *dn_ptr; /* DECnet specific data */
- struct inet6_dev __rcu *ip6_ptr; /* IPv6 specific data */
- void *ec_ptr; /* Econet specific data */
- void *ax25_ptr; /* AX.25 specific data */
- struct wireless_dev *ieee80211_ptr; /* IEEE 802.11 specific data,
- assign before registering */
- // 在接收通道中需要缓存的数据
- /*
- * Cache lines mostly used on receive path (including eth_type_trans())
- */
- unsigned long last_rx; /* Time of last Rx
- * This should not be set in
- * drivers, unless really needed,
- * because network stack (bonding)
- * use it if/when necessary, to
- * avoid dirtying this cache line.
- */
- struct net_device *master; /* Pointer to master device of a group,
- * which this device is member of.
- */
- /* Interface address info used in eth_type_trans() */
- unsigned char *dev_addr; /* hw address, (before bcast
- because most packets are
- unicast) */
- struct netdev_hw_addr_list dev_addrs; /* list of device
- hw addresses */
- unsigned char broadcast[MAX_ADDR_LEN]; /* hw bcast add */
- #ifdef CONFIG_RPS
- struct kset *queues_kset;
- struct netdev_rx_queue *_rx;
- /* Number of RX queues allocated at register_netdev() time */
- unsigned int num_rx_queues;
- /* Number of RX queues currently active in device */
- unsigned int real_num_rx_queues;
- #ifdef CONFIG_RFS_ACCEL
- /* CPU reverse-mapping for RX completion interrupts, indexed
- * by RX queue number. Assigned by driver. This must only be
- * set if the ndo_rx_flow_steer operation is defined. */
- struct cpu_rmap *rx_cpu_rmap;
- #endif
- #endif
- rx_handler_func_t __rcu *rx_handler;
- void __rcu *rx_handler_data;
- struct netdev_queue __rcu *ingress_queue;
- // 在发送通道中需要缓存的数据
- /*
- * Cache lines mostly used on transmit path
- */
- struct netdev_queue *_tx ____cacheline_aligned_in_smp;
- /* Number of TX queues allocated at alloc_netdev_mq() time */
- unsigned int num_tx_queues;
- /* Number of TX queues currently active in device */
- unsigned int real_num_tx_queues;
- /* root qdisc from userspace point of view */
- struct Qdisc *qdisc;
- unsigned long tx_queue_len; /* Max frames per queue allowed */
- spinlock_t tx_global_lock;
- #ifdef CONFIG_XPS
- struct xps_dev_maps __rcu *xps_maps;
- #endif
- /* These may be needed for future network-power-down code. */
- /*
- * trans_start here is expensive for high speed devices on SMP,
- * please use netdev_queue->trans_start instead.
- */
- unsigned long trans_start; /* Time (in jiffies) of last Tx */
- int watchdog_timeo; /* used by dev_watchdog() */
- struct timer_list watchdog_timer;
- /* Number of references to this device */
- int __percpu *pcpu_refcnt;
- /* delayed register/unregister */
- struct list_head todo_list;
- /* device index hash chain */
- struct hlist_node index_hlist;
- struct list_head link_watch_list;
- /* register/unregister state machine */
- enum { NETREG_UNINITIALIZED=0,
- NETREG_REGISTERED, /* completed register_netdevice */
- NETREG_UNREGISTERING, /* called unregister_netdevice */
- NETREG_UNREGISTERED, /* completed unregister todo */
- NETREG_RELEASED, /* called free_netdev */
- NETREG_DUMMY, /* dummy device for NAPI poll */
- } reg_state:8;
- bool dismantle; /* device is going do be freed */
- enum {
- RTNL_LINK_INITIALIZED,
- RTNL_LINK_INITIALIZING,
- } rtnl_link_state:16;
- /* Called from unregister, can be used to call free_netdev */
- void (*destructor)(struct net_device *dev);
- #ifdef CONFIG_NETPOLL
- struct netpoll_info *npinfo;
- #endif
- #ifdef CONFIG_NET_NS
- /* Network namespace this network device is inside */
- struct net *nd_net;
- #endif
- /* mid-layer private */
- union {
- void *ml_priv;
- struct pcpu_lstats __percpu *lstats; /* loopback stats */
- struct pcpu_tstats __percpu *tstats; /* tunnel stats */
- struct pcpu_dstats __percpu *dstats; /* dummy stats */
- };
- /* GARP */
- struct garp_port __rcu *garp_port;
- /* class/net/name entry */
- struct device dev;
- /* space for optional device, statistics, and wireless sysfs groups */
- const struct attribute_group *sysfs_groups[4];
- /* rtnetlink link ops */
- const struct rtnl_link_ops *rtnl_link_ops;
- /* for setting kernel sock attribute on TCP connection setup */
- #define GSO_MAX_SIZE 65536
- unsigned int gso_max_size;
- u8 num_tc;
- struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
- u8 prio_tc_map[TC_BITMASK + 1];
- /* n-tuple filter list attached to this device */
- struct ethtool_rx_ntuple_list ethtool_ntuple_list;
- /* phy device may attach itself for hardware timestamping */
- struct phy_device *phydev;
- /* group the device belongs to */
- int group;
- }
net_device是每个网络设备的基础,它不仅包含network adapter硬件信息(如:interrupt, ports, driver functions等),也包含高层网络协议的配置数据(如:IP address, subnet mask等).
在/sys/class/net下列出来所有网络设备的名字,如我的为:
shell@android:/sys/class/net # ll
lrwxrwxrwx root root 2013-07-05 17:08 ip6tnl0
lrwxrwxrwx root root 2013-07-05 17:08 lo (loopback设备)
lrwxrwxrwx root root 2013-07-05 17:08 sit0
lrwxrwxrwx root root 2000-01-01 08:00 wlan0 (Wifi设备)
3.2 管理net_device
从上面的协议实例看net_device。
3.2.1注册和注销网络设备(net_device)
位于文件:kernel/net/core/dev.c
int register_netdev(struct net_device *dev)
void unregister_netdev(struct net_device *dev)
网络设备(net_device)与一个已经存在的network adapter一一对应。
3.2.2 打开和关闭网络设备(net_device)
位于文件:kernel/net/core/dev.c
int dev_open(struct net_device *dev)
int dev_close(struct net_device *dev)
3.2.3 创建、释放和查找网络设备(net_device)
位于文件:kernel/net/core/dev.c
struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
void (*setup)(struct net_device *),
unsigned int txqs, unsigned int rxqs)
void free_netdev(struct net_device *dev)
int dev_alloc_name(struct net_device *dev, const char *name)
struct net_device *dev_get_by_index(struct net *net, int ifindex)
struct net_device *dev_get_by_name(struct net *net, const char *name)
void dev_load(struct net *net, const char *name)
3.2.4网络设备通知上层协议状态变化
int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
网络状态有如下值:
NETDEV_UP: 激活一个网络设备 (dev_open)
NETDEV_DOWN: 禁止一个网络设备 (dev_close)
NETDEV_CHANGE: 通知网络设备状态变化
NETDEV_REGISTER: 网络设备已经被注册,但是没有打开实例
NETDEV_UNREGISTER: 网络设备已经被删除
NETDEV_CHANGEMTU: 网络设备MTU被修改
NETDEV_CHANGEADDR: 网络设备硬件地址被修改
NETDEV_CHANGENAME:网络设备名字被修改
3.2.5 通过net_device发送数据
int dev_queue_xmit(struct sk_buff *skb) // kernel/net/core/dev.c
它由高层的协议实例调用,以通过一个net_device(skb->dev)发送一个socket buffer.
3.3 net_device_ops
- struct net_device_ops {
- int (*ndo_init)(struct net_device *dev);
- void (*ndo_uninit)(struct net_device *dev);
- int (*ndo_open)(struct net_device *dev);
- int (*ndo_stop)(struct net_device *dev);
- netdev_tx_t (*ndo_start_xmit) (struct sk_buff *skb,
- struct net_device *dev);
- u16 (*ndo_select_queue)(struct net_device *dev,
- struct sk_buff *skb);
- void (*ndo_change_rx_flags)(struct net_device *dev,
- int flags);
- void (*ndo_set_rx_mode)(struct net_device *dev);
- void (*ndo_set_multicast_list)(struct net_device *dev);
- int (*ndo_set_mac_address)(struct net_device *dev,
- void *addr);
- int (*ndo_validate_addr)(struct net_device *dev);
- int (*ndo_do_ioctl)(struct net_device *dev,
- struct ifreq *ifr, int cmd);
- int (*ndo_set_config)(struct net_device *dev,
- struct ifmap *map);
- int (*ndo_change_mtu)(struct net_device *dev,
- int new_mtu);
- int (*ndo_neigh_setup)(struct net_device *dev,
- struct neigh_parms *);
- void (*ndo_tx_timeout) (struct net_device *dev);
- struct rtnl_link_stats64* (*ndo_get_stats64)(struct net_device *dev,
- struct rtnl_link_stats64 *storage);
- struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
- void (*ndo_vlan_rx_register)(struct net_device *dev,
- struct vlan_group *grp);
- void (*ndo_vlan_rx_add_vid)(struct net_device *dev,
- unsigned short vid);
- void (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
- unsigned short vid);
- #ifdef CONFIG_NET_POLL_CONTROLLER
- void (*ndo_poll_controller)(struct net_device *dev);
- int (*ndo_netpoll_setup)(struct net_device *dev,
- struct netpoll_info *info);
- void (*ndo_netpoll_cleanup)(struct net_device *dev);
- #endif
- int (*ndo_set_vf_mac)(struct net_device *dev,
- int queue, u8 *mac);
- int (*ndo_set_vf_vlan)(struct net_device *dev,
- int queue, u16 vlan, u8 qos);
- int (*ndo_set_vf_tx_rate)(struct net_device *dev,
- int vf, int rate);
- int (*ndo_get_vf_config)(struct net_device *dev,
- int vf,
- struct ifla_vf_info *ivf);
- int (*ndo_set_vf_port)(struct net_device *dev,
- int vf,
- struct nlattr *port[]);
- int (*ndo_get_vf_port)(struct net_device *dev,
- int vf, struct sk_buff *skb);
- int (*ndo_setup_tc)(struct net_device *dev, u8 tc);
- #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
- int (*ndo_fcoe_enable)(struct net_device *dev);
- int (*ndo_fcoe_disable)(struct net_device *dev);
- int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
- u16 xid,
- struct scatterlist *sgl,
- unsigned int sgc);
- int (*ndo_fcoe_ddp_done)(struct net_device *dev,
- u16 xid);
- int (*ndo_fcoe_ddp_target)(struct net_device *dev,
- u16 xid,
- struct scatterlist *sgl,
- unsigned int sgc);
- #define NETDEV_FCOE_WWNN 0
- #define NETDEV_FCOE_WWPN 1
- int (*ndo_fcoe_get_wwn)(struct net_device *dev,
- u64 *wwn, int type);
- #endif
- #ifdef CONFIG_RFS_ACCEL
- int (*ndo_rx_flow_steer)(struct net_device *dev,
- const struct sk_buff *skb,
- u16 rxq_index,
- u32 flow_id);
- #endif
- int (*ndo_add_slave)(struct net_device *dev,
- struct net_device *slave_dev);
- int (*ndo_del_slave)(struct net_device *dev,
- struct net_device *slave_dev);
- u32 (*ndo_fix_features)(struct net_device *dev,
- u32 features);
- int (*ndo_set_features)(struct net_device *dev,
- u32 features);
- }
4. Network Drivers
4.1 初始化网络适配器(Network Adapter)
在net_device被激活之前,我们必须找到一个匹配的network adapter。网络驱动(network driver)的初始化函数(init/probe)负责找到一个匹配的network adapter并且使用对应的信息初始化net_device。在驱动的probe函数中,主要完成以下任务:(参考:kernel/drivers/net/pci-skeleton.c)
1) 创建net_device
2) 填充相关的硬件信息
3) 调用register_netdev进行注册
4) 设置net_device->netdev_ops (netdev_ops由驱动实现)
4.2 打开、关闭网络适配器
1) 打开: ifconfig wlan0 up->ioctl->dev_open->net_device.netdev_ops.ndo_open
2) 关闭: ifconfig wlan0 down->ioctl->dev_close->net_device.netdev_ops.ndo_stop
4.3 发送数据
在驱动中实现与ndo_start_xmit对应的函数
4.4 接收数据
其流程如下图所示:
在网络驱动的中断处理函数中,首先调用dev_alloc_skb创建socket buffer,然后把接收到的数据copy到其中,最后调用netif_rx把socket buffer放入队列中,供协议层处理。
int netif_rx(struct sk_buff *skb) // kernel/net/core/dev.c
5. Data-Link Layer
1) 逻辑链路控制(LLC)层由Linux内核实现,网络适配器通过net_device连接到操作系统内核。
2) 数据链路层(Data-Link Layer)由LLC和MAC组成。LLC隐藏了所有不同传输介质的差异,从而以上层协议提供统一的接口;而MAC层则反应了不同传输技术(传输协议,如802.3与802.11不同)的差异。
逻辑链路控制 LLC (Logical Link Control)子层
媒体接入控制 MAC (Medium Access Control)子层
与接入到传输媒体有关的内容都放在 MAC子层,而LLC 子层则与传输媒体无关,不管采用何种协议的局域网对 LLC 子层来说都是透明的。
5.1 接收网络包流程