OVS与datapath交互源码分析

前面章节分析了ovs与datapath通过netlink通信机制，本章节重点分析下ovs与datapath的具体交互流程。

 

1.交互消息分类

datapath在初始时注册6类消息，用于ovs与datapath消息传递。

MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY);
MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY);
MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY);
MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY);
MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY);
MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY);
 

我们以OVS_DATAPATH_FAMILY为例看一下消息的注册内容

static const struct genl_ops dp_datapath_genl_ops[] = {
    { .cmd = OVS_DP_CMD_NEW,
      .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
      .policy = datapath_policy,
      .doit = ovs_dp_cmd_new
    },
    { .cmd = OVS_DP_CMD_DEL,
      .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
      .policy = datapath_policy,
      .doit = ovs_dp_cmd_del
    },
    { .cmd = OVS_DP_CMD_GET,
      .flags = 0,            /* OK for unprivileged users. */
      .policy = datapath_policy,
      .doit = ovs_dp_cmd_get,
      .dumpit = ovs_dp_cmd_dump
    },
    { .cmd = OVS_DP_CMD_SET,
      .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
      .policy = datapath_policy,
      .doit = ovs_dp_cmd_set,
    },
};

static struct genl_family dp_datapath_genl_family __ro_after_init = {
    .hdrsize = sizeof(struct ovs_header),
    .name = OVS_DATAPATH_FAMILY,
    .version = OVS_DATAPATH_VERSION,
    .maxattr = OVS_DP_ATTR_MAX,
    .netnsok = true,
    .parallel_ops = true,
    .ops = dp_datapath_genl_ops,
    .n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
    .mcgrps = &ovs_dp_datapath_multicast_group,
    .n_mcgrps = 1,
    .module = THIS_MODULE,
};
共注册了4类cmd操作NEW/DEL/GET/SET

2 datapath处理消息流程

2.1 OVS_DP_CMD_NEW

首先分析创建bridge桥的cmd处理流程OVS_DP_CMD_NEW

 OVS_DP_CMD_NEW

|---ovs_dp_cmd_new

    |---ovs_flow_tbl_init

    |---new_vport

        |---ovs_vport_add

            |---ovs_vport_lookup //根据internal类型的端口注册的是 ovs_internal_vport_ops

                 |---internal_dev_create

                     |---ovs_vport_alloc 创建vport，注册ops最关键的是send函数internal_dev_recv

                 |---alloc_netdev 创建netdev内核中的dev (vport->dev = dev internal_dev->vport = vport) ，建立一一对应关系

                     |---do_setup 最关键的是注册netdev_ops (internal_dev_netdev_ops),其中有ndo_start_xmit 为internal_dev_xmit

                |---register_netdevice 把vport->dev注册到内核，由内核管理       

            

vport与对应的内核dev的关系

static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct vport_parms parms;
	struct sk_buff *reply;
	struct datapath *dp;
	struct vport *vport;
	struct ovs_net *ovs_net;
	int err, i;

	err = -EINVAL;
	if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])  //必须协带name和pid两个属性
		goto err;

	reply = ovs_dp_cmd_alloc_info();
	if (!reply)
		return -ENOMEM;

	err = -ENOMEM;
	dp = kzalloc(sizeof(*dp), GFP_KERNEL);
	if (dp == NULL)
		goto err_free_reply;

	ovs_dp_set_net(dp, sock_net(skb->sk));

	/* Allocate table. */
	err = ovs_flow_tbl_init(&dp->table); 初始化flow table实例
	if (err)
		goto err_free_dp;

	dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
	if (!dp->stats_percpu) {
		err = -ENOMEM;
		goto err_destroy_table;
	}

	dp->ports = kmalloc_array(DP_VPORT_HASH_BUCKETS,
				  sizeof(struct hlist_head),
				  GFP_KERNEL);
	if (!dp->ports) {
		err = -ENOMEM;
		goto err_destroy_percpu;
	}

	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
		INIT_HLIST_HEAD(&dp->ports[i]);

	err = ovs_meters_init(dp);
	if (err)
		goto err_destroy_ports_array;

	/* Set up our datapath device. */
	parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
	parms.type = OVS_VPORT_TYPE_INTERNAL;  //internal类型的端口
	parms.options = NULL;
	parms.dp = dp; //分配的dp结构
	parms.port_no = OVSP_LOCAL; //端口号为0
	parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];

	ovs_dp_change(dp, a); //记录用户属性user_features

	/* So far only local changes have been made, now need the lock. */
	ovs_lock();

	vport = new_vport(&parms); 创建vport口，后面会重点分析创建vport口的过程
	if (IS_ERR(vport)) {
		err = PTR_ERR(vport);
		if (err == -EBUSY)
			err = -EEXIST;

		if (err == -EEXIST) {
			/* An outdated user space instance that does not understand
			 * the concept of user_features has attempted to create a new
			 * datapath and is likely to reuse it. Drop all user features.
			 */
			if (info->genlhdr->version < OVS_DP_VER_FEATURES)
				ovs_dp_reset_user_features(skb, info);
		}

		goto err_destroy_meters;
	}

	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
				   info->snd_seq, 0, OVS_DP_CMD_NEW);
	BUG_ON(err < 0);

	ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
	list_add_tail_rcu(&dp->list_node, &ovs_net->dps);

	ovs_unlock();

	ovs_notify(&dp_datapath_genl_family, reply, info);
	return 0;

err:
	return err;
}

static struct vport *new_vport(const struct vport_parms *parms)
{
	struct vport *vport;

	vport = ovs_vport_add(parms);
	if (!IS_ERR(vport)) {
		struct datapath *dp = parms->dp;
		struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);

		hlist_add_head_rcu(&vport->dp_hash_node, head); //vport加入到dp桥的hash链表中
	}
	return vport;
}

struct vport *ovs_vport_add(const struct vport_parms *parms)
{
	struct vport_ops *ops;
	struct vport *vport;
  
/*
根据端口类型查找注册的ops，internal类型的端口注册的ops为：
static struct vport_ops ovs_internal_vport_ops = {
	.type		= OVS_VPORT_TYPE_INTERNAL,
	.create		= internal_dev_create,
	.destroy	= internal_dev_destroy,
	.send		= internal_dev_recv,
};*/
	ops = ovs_vport_lookup(parms); 
	if (ops) {
		struct hlist_head *bucket;

		if (!try_module_get(ops->owner))
			return ERR_PTR(-EAFNOSUPPORT);

		vport = ops->create(parms); //internal_dev_create
		if (IS_ERR(vport)) {
			module_put(ops->owner);
			return vport;
		}

		bucket = hash_bucket(ovs_dp_get_net(vport->dp),
				     ovs_vport_name(vport)); //ovs_port_name函数获取的是 vport->dev->name，内核中的dev name
		hlist_add_head_rcu(&vport->hash_node, bucket); //vport加入到net的管理列表
		return vport;
	}

	/* Unlock to attempt module load and return -EAGAIN if load
	 * was successful as we need to restart the port addition
	 * workflow.
	 */
	ovs_unlock();
	request_module("vport-type-%d", parms->type);
	ovs_lock();

	if (!ovs_vport_lookup(parms))
		return ERR_PTR(-EAFNOSUPPORT);
	else
		return ERR_PTR(-EAGAIN);
}

2.2 OVS_VPORT_CMD_NEW

接下来再分个vport的创建流程，前面部分与上面分析的OVS_DP_CMD_NEW的类似，只是在根据端口类型与内核进行关联的时候有所区别。

ovs_vport_cmd_new

|---new_vport

    |---ovs_vport_add 创建成功后，会把vport加入dp链表

         |--- ovs_vport_lookup 根据要创建的port类型查找注册的ops

              |---vport = ops->create(parms);  调用的是 netdev_create 具体的注册位置在vport-netdev.c

              |---ovs_vport_alloc 创建vport接触，注册send函数

              |---ovs_netdev_link

                  |---dev_get_by_name(ovs_dp_get_net(vport->dp), name); 通过name查询dev

                  |---netdev_rx_handler_register 挂在接收函数netdev_frame_hook和参数vport

                  |---dev_set_promiscuity 打开混杂模式

                  |---vport->dev->priv_flags |= IFF_OVS_DATAPATH; 设置dev的flags

netdev_rx_handler_register()
linux 内核实现的一个函数，为网络设备 dev 注册一个handler_frame_hook,rx_handle_data 指向的是handler_frame_hook 内存的区域，这个 handler 以后会被__netif_receive_skb()呼叫，就是说netdev_rx_handler_register(netdev_vport->dev,netdev_frame_hook,vport);在收到packet 后会调用 netdev_frame_hook 函数处理