ovs 删除openflow流表

有如下三种方法可以删除openflow流表:
a. controller/ovs-ofctl主动发命令(OFPFC_DELETE or OFPFC_DELETE_STRICT)删除流表。OFPFC_DELETE_STRICT和OFPFC_DELETE 的区别是前者需要匹配所有的字段才能删除(包括优先级)，并且一次只能删除一条流表，后者要求指定的字段必须是流表的一个子集(不能指定优先级)，可以一次删除多个流表。

//比如添加如下两条流表
ovs-ofctl add-flow br10 "priority=80, in_port=2,ip, nw_dst=1.2.0.0/16, action=1"
ovs-ofctl add-flow br10 "priority=100, in_port=2, ip, nw_dst=1.1.1.0/24, action=1"

//使用OFPFC_DELETE删除流表时，可以只指定in_port或者ip就可以将上面两条流表删除
ovs-ofctl del-flows br10 "in_port=2"
ovs-ofctl del-flows br10 "ip"

//使用OFPFC_DELETE_STRICT(加上选项 --strict)删除流表时，需要指定流表的所有内容，包括优先级
ovs-ofctl --strict del-flows br10 "priority=100, in_port=2, ip, nw_dst=1.1.1.0/24"

b. 流表的超时机制: 添加流表时如果指定idle_timeout或者hard_timeout参数，则流表超时后将被删除。如果不指定这俩参数，则默认不会被超时机制删除。hard_timeout指定的超时时间是从创建流表，或者修改流表开始计时，超时时间到后，不管此流表有没有被使用，都会被删除。idle_timeout指定流表空闲超时时间，从最近流表被使用开始计时，如果指定时间内此流表没有被使用，则被删除。
c. 强制回收机制: 添加流表时，如果当前流表个数大于等于最大流表个数，则判断是否可以强制回收之前添加的流表。可以通过Flow_Table里的overflow_policy参数指定当前流表个数大于等于最大流表个数时的行为，如果为refuse则拒绝添加新流表，如果为evict则强制删除即将超时的流表。强制回收只考虑指定了超时时间的流表。

image.png

添加流表时，如果指定了超时时间，并且流表指定的field包含group指定的field，则会将流表添加到一个分组中。具体为根据group指定的这些field计算hash值，再根据hash值到table->eviction_groups_by_id查找struct eviction_group。然后根据超时时间计算优先级，根据优先级将此流表插入struct eviction_group

struct oftable {
    ...
    /* Eviction groups.
     *
     * When a flow is added that would cause the table to have more than
     * 'max_flows' flows, and 'eviction_fields' is nonnull, these groups are
     * used to decide which rule to evict: the rule is chosen from the eviction
     * group that contains the greatest number of rules.*/
    uint32_t eviction_group_id_basis; //使用random_uint32()产生的随机数
    struct hmap eviction_groups_by_id;//hash表，存放struct eviction_group，struct eviction_group中存放rule
    struct heap eviction_groups_by_size;//根据struct eviction_group中rule个数将struct eviction_group排序存放
    ...
}

针对强制删除流表，可以做如下实验

//设置最大流表个数为3，overflow_policy为强制删除
root@ubuntu:~# ovs-vsctl -- --id=@ft  create  Flow_Table  name=test1 flow_limit=3 overflow_policy=evict -- set Bridge br10 flow_tables=0=@ft

//查看设置
root@ubuntu:~# ovs-vsctl list Flow_table
_uuid               : 44527d3b-baf0-40ae-afc9-22979bea3d68
external_ids        : {}
flow_limit          : 3
groups              : []
name                : "test1"
overflow_policy     : evict
prefixes            : []

//当前以及有如下三条流表
root@ubuntu:~# ovs-ofctl dump-flows br10
 cookie=0x0, duration=1.861s, table=0, n_packets=0, n_bytes=0, priority=100,ip,in_port="veth_l0",nw_dst=1.1.1.0/24 actions=output:"veth_r0"
 cookie=0x0, duration=1.868s, table=0, n_packets=0, n_bytes=0, priority=80,ip,in_port="veth_l0",nw_dst=1.2.0.0/16 actions=output:"veth_r0"
 cookie=0x0, duration=477124.741s, table=0, n_packets=6, n_bytes=364, priority=0 actions=NORMAL

//再添加新流表时，提示流表已经满了，但是前面不是已经设置强制删除了吗，为什么不生效？
//因为当前设置的三个流表都没有指定超时时间，而强制删除不会删除永久性的流表
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=100, in_port=2, ip, nw_dst=2.1.1.0/24, action=1"
OFPT_ERROR (xid=0x4): OFPFMFC_TABLE_FULL
OFPT_FLOW_MOD (xid=0x4):
(***truncated to 64 bytes from 80***)
00000000  01 0e 00 50 00 00 00 04-00 32 20 ee 00 02 00 00 |...P.....2 .....|
00000010  00 00 00 00 00 00 00 00-00 00 00 00 00 00 08 00 |................|
00000020  00 00 00 00 00 00 00 00-02 01 01 00 00 00 00 00 |................|
00000030  00 00 00 00 00 00 00 00-00 00 00 00 00 00 00 64 |...............d|

//删除所有流表，重新添加三条流表，其中两条指定了超时时间
root@ubuntu:~# ovs-ofctl del-flows br10 ""
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=0, action=normal"
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=80, hard_timeout=1000, in_port=2,ip, nw_dst=1.2.0.0/16, action=1"
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=100, hard_timeout=1000, in_port=2, ip, nw_dst=1.1.1.0/24, action=1"
//查看当前流表
root@ubuntu:~# ovs-ofctl dump-flows br10
 cookie=0x0, duration=7.037s, table=0, n_packets=0, n_bytes=0, hard_timeout=1000, priority=100,ip,in_port="veth_l0",nw_dst=1.1.1.0/24 actions=output:"veth_r0"
 cookie=0x0, duration=25.689s, table=0, n_packets=0, n_bytes=0, hard_timeout=1000, priority=80,ip,in_port="veth_l0",nw_dst=1.2.0.0/16 actions=output:"veth_r0"
 cookie=0x0, duration=19.506s, table=0, n_packets=0, n_bytes=0, priority=0 actions=NORMAL
root@ubuntu:~#
//再次添加第四条流表可以成功
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=100, in_port=2, ip, nw_dst=2.1.1.0/24, action=1"
//查看当前流表，发现之前存在时间最长的"duration=25.689s"流表被删除了
root@ubuntu:~# ovs-ofctl dump-flows br10
 cookie=0x0, duration=20.731s, table=0, n_packets=0, n_bytes=0, hard_timeout=1000, priority=100,ip,in_port="veth_l0",nw_dst=1.1.1.0/24 actions=output:"veth_r0"
 cookie=0x0, duration=2.579s, table=0, n_packets=0, n_bytes=0, priority=100,ip,in_port="veth_l0",nw_dst=2.1.1.0/24 actions=output:"veth_r0"
 cookie=0x0, duration=33.200s, table=0, n_packets=0, n_bytes=0, priority=0 actions=NORMAL

源码分析

下面分析下这三种情况下的源码
controller/ovs-ofctl主动删除流表

ofproto_flow_mod_start
    delete_flows_start_loose(ofproto, ofm);
        //先根据指定的字段ofm->criteria查找需要删除的rule，这是用的是loose方式查找
        collect_rules_loose(ofproto, &ofm->criteria, rules);
        delete_flows_start__(ofproto, ofm->version, rules);

    delete_flow_start_strict(ofproto, ofm);
        //先根据指定的字段ofm->criteria查找需要删除的rule，这是用的是strict方式查找，必须精确匹配
        collect_rules_strict(ofproto, &ofm->criteria, rules);
        delete_flows_start__(ofproto, ofm->version, rules);

//将rules从ofproto中删除
static void
delete_flows_start__(struct ofproto *ofproto, ovs_version_t version,
                     const struct rule_collection *rules)
    struct rule *rule;

    RULE_COLLECTION_FOR_EACH (rule, rules) {
        struct oftable *table = &ofproto->tables[rule->table_id];
        //流表个数减一
        table->n_flows--;
        //设置versions->remove_version为version，表示此流表从version开始就被删除了
        cls_rule_make_invisible_in_version(&rule->cr, version);
            struct cls_match *cls_match = get_cls_match_protected(rule);
            cls_match_set_remove_version(cls_match, remove_version);
                versions_set_remove_version(&rule->versions, version);
                    atomic_store_relaxed(&versions->remove_version, version);
        //一个流表会插入多个地方保存，这个只是将流表从ofproto中删除，后面会将流表从分类器中删除
        /* Removes 'rule' from the ofproto data structures.  Caller may have deferred
         * the removal from the classifier. */
        /* Remove rule from ofproto data structures. */
        ofproto_rule_remove__(ofproto, rule);
    }

将流表从分类器中删除

ofproto_flow_mod_finish(ofproto, &ofm, req);
    delete_flows_finish(ofproto, ofm, req);
        delete_flows_finish__(ofproto, &ofm->old_rules, OFPRR_DELETE, req);
            remove_rules_postponed(rules);
                ovsrcu_postpone(remove_rules_rcu, rule_collection_detach(rules));
                    remove_rule_rcu__(rule);
                        cls_rule_visible_in_version(&rule->cr, OVS_VERSION_MAX)
                        //将rule从分类器删除
                        classifier_remove(&table->cls, &rule->cr)

超时机制
添加流表时，如果指定了超时时间，则会将流表挂到链表ofproto->expirable上，在ovs-vswitchd的主循环中，会周期性的调用ofproto-dpif.c中的run函数，此函数会遍历链表ofproto->expirable判断流表是否超时，如果超时则将流表删除。

run(struct ofproto *ofproto_)
    long long now = time_msec();
    ovs_mutex_lock(&ofproto_mutex);
    LIST_FOR_EACH_SAFE (rule, next_rule, expirable, &ofproto->up.expirable) {
        rule_expire(rule_dpif_cast(rule), now);
            uint16_t hard_timeout, idle_timeout;
            int reason = -1;

            hard_timeout = rule->up.hard_timeout;
            idle_timeout = rule->up.idle_timeout;
            
            //now 表示当前时间，如果超过了hard_timeout或者idle_timeout，则将rule删除，
            //并设置对应的reason
            /* Has 'rule' expired? */
            if (hard_timeout) {
                long long int modified;

                ovs_mutex_lock(&rule->up.mutex);
                modified = rule->up.modified;
                ovs_mutex_unlock(&rule->up.mutex);

                if (now > modified + hard_timeout * 1000) {
                    reason = OFPRR_HARD_TIMEOUT;
                }
            }

            if (reason < 0 && idle_timeout) {
                long long int used;

                ovs_mutex_lock(&rule->stats_mutex);
                used = rule->stats.used;
                ovs_mutex_unlock(&rule->stats_mutex);

                if (now > used + idle_timeout * 1000) {
                    reason = OFPRR_IDLE_TIMEOUT;
                }
            }

            if (reason >= 0) {
                COVERAGE_INC(ofproto_dpif_expired);
                ofproto_rule_expire(&rule->up, reason);
                    struct rule_collection rules;

                    rule_collection_init(&rules);
                    rule_collection_add(&rules, rule);
                    delete_flows__(&rules, reason, NULL);
                        //收集完需要删除的rules后，后面的流表和主动删除流表时一样的
                        if (rule_collection_n(rules)) {
                            struct ofproto *ofproto = rule_collection_rules(rules)[0]->ofproto;

                            delete_flows_start__(ofproto, ofproto->tables_version + 1, rules);
                            ofproto_bump_tables_version(ofproto);
                            delete_flows_finish__(ofproto, rules, reason, req);
                            ofmonitor_flush(ofproto->connmgr);
                        }
            }

    }
    ovs_mutex_unlock(&ofproto_mutex);

evict强制删除
添加流表时会将流表插入eviction_group中

ofproto_rule_insert__ -> eviction_group_add_rule(rule)

/* Adds 'rule' to an appropriate eviction group for its oftable's
 * configuration.  Does nothing if 'rule''s oftable doesn't have eviction
 * enabled, or if 'rule' is a permanent rule (one that will never expire on its
 * own).
 *
 * The caller must ensure that 'rule' is not already in an eviction group. */
static void
eviction_group_add_rule(struct rule *rule)
    OVS_REQUIRES(ofproto_mutex)
{
    struct ofproto *ofproto = rule->ofproto;
    struct oftable *table = &ofproto->tables[rule->table_id];
    bool has_timeout;

    /* Timeouts may be modified only when holding 'ofproto_mutex'.  We have it
     * so no additional protection is needed. */
    has_timeout = rule->hard_timeout || rule->idle_timeout;
    //配置了overflow_policy为evict，并且流表指定了超时时间
    if (table->eviction && has_timeout) {
        struct eviction_group *evg;
        //使用函数 eviction_group_hash_rule 计算hash值，查找 struct eviction_group
        evg = eviction_group_find(table, eviction_group_hash_rule(rule));
        //使用函数 rule_eviction_priority 计算优先级，将 rule 按优先级插入 evg 中
        rule->eviction_group = evg;
        heap_insert(&evg->rules, &rule->evg_node,
                    rule_eviction_priority(ofproto, rule));
        //按照 evg 中最新rule个数重新排序 table->eviction_groups_by_size
        eviction_group_resized(table, evg);
    }
}

//根据group配置的field计算hash值
/* Hashes the 'rule''s values for the eviction_fields of 'rule''s table, and
 * returns the hash value. */
static uint32_t
eviction_group_hash_rule(struct rule *rule)
    OVS_REQUIRES(ofproto_mutex)
{
    struct oftable *table = &rule->ofproto->tables[rule->table_id];
    const struct mf_subfield *sf;
    struct flow flow;
    uint32_t hash;

    hash = table->eviction_group_id_basis;
    miniflow_expand(rule->cr.match.flow, &flow);
    for (sf = table->eviction_fields;
         sf < &table->eviction_fields[table->n_eviction_fields];
         sf++)
    {
        if (mf_are_prereqs_ok(sf->field, &flow, NULL)) {
            union mf_value value;

            mf_get_value(sf->field, &flow, &value);
            if (sf->ofs) {
                bitwise_zero(&value, sf->field->n_bytes, 0, sf->ofs);
            }
            if (sf->ofs + sf->n_bits < sf->field->n_bytes * 8) {
                unsigned int start = sf->ofs + sf->n_bits;
                bitwise_zero(&value, sf->field->n_bytes, start,
                             sf->field->n_bytes * 8 - start);
            }
            hash = hash_bytes(&value, sf->field->n_bytes, hash);
        } else {
            hash = hash_int(hash, 0);
        }
    }

    return hash;
}

//根据hash值查找table->eviction_groups_by_id，如果不存在则创建新的struct eviction_group
/* Returns an eviction group within 'table' with the given 'id', creating one
 * if necessary. */
static struct eviction_group *
eviction_group_find(struct oftable *table, uint32_t id)
    OVS_REQUIRES(ofproto_mutex)
{
    struct eviction_group *evg;

    HMAP_FOR_EACH_WITH_HASH (evg, id_node, id, &table->eviction_groups_by_id) {
        return evg;
    }

    evg = xmalloc(sizeof *evg);
    hmap_insert(&table->eviction_groups_by_id, &evg->id_node, id);
    heap_insert(&table->eviction_groups_by_size, &evg->size_node,
                eviction_group_priority(0));
    heap_init(&evg->rules);

    return evg;
}

//根据rule的超时时间计算优先级
/* Returns an eviction priority for 'rule'.  The return value should be
 * interpreted so that higher priorities make a rule a more attractive
 * candidate for eviction. */
static uint64_t
rule_eviction_priority(struct ofproto *ofproto, struct rule *rule)
    OVS_REQUIRES(ofproto_mutex)
{
    /* Calculate absolute time when this flow will expire.  If it will never
     * expire, then return 0 to make it unevictable.  */
    long long int expiration = LLONG_MAX;
    if (rule->hard_timeout) {
        /* 'modified' needs protection even when we hold 'ofproto_mutex'. */
        ovs_mutex_lock(&rule->mutex);
        long long int modified = rule->modified;
        ovs_mutex_unlock(&rule->mutex);

        expiration = modified + rule->hard_timeout * 1000;
    }
    if (rule->idle_timeout) {
        uint64_t packets, bytes;
        long long int used;
        long long int idle_expiration;

        ofproto->ofproto_class->rule_get_stats(rule, &packets, &bytes, &used);
        idle_expiration = used + rule->idle_timeout * 1000;
        expiration = MIN(expiration, idle_expiration);
    }
    if (expiration == LLONG_MAX) {
        return 0;
    }

    /* Calculate the time of expiration as a number of (approximate) seconds
     * after program startup.
     *
     * This should work OK for program runs that last UINT32_MAX seconds or
     * less.  Therefore, please restart OVS at least once every 136 years. */
    uint32_t expiration_ofs = (expiration >> 10) - (time_boot_msec() >> 10);

    /* Combine expiration time with OpenFlow "importance" to form a single
     * priority value.  We want flows with relatively low "importance" to be
     * evicted before even considering expiration time, so put "importance" in
     * the most significant bits and expiration time in the least significant
     * bits.
     *
     * Small 'priority' should be evicted before those with large 'priority'.
     * The caller expects the opposite convention (a large return value being
     * more attractive for eviction) so we invert it before returning. */
    uint64_t priority = ((uint64_t) rule->importance << 32) + expiration_ofs;
    return UINT64_MAX - priority;
}

当再次添加流表时，如果当前流表个数超过最大流表个数，则

add_flow_start
    /* If necessary, evict an existing rule to clear out space. */
    if (table->n_flows >= table->max_flows) {
        //选择一个要强制删除的rule，如果不能强制删除，则添加流表失败并返回OFPERR_OFPFMFC_TABLE_FULL
        if (!choose_rule_to_evict(table, &old_rule)) {
            return OFPERR_OFPFMFC_TABLE_FULL;
        }
        //将rule从 evict_group 中删除
        eviction_group_remove_rule(old_rule);
        /* Marks 'old_rule' as an evicted rule rather than replaced rule.
         */
        old_rule->removed_reason = OFPRR_EVICTION;
    }

    replace_rule_start(ofproto, ofm, old_rule, new_rule);
        //设置versions->remove_version为version，表示此流表从version开始就被删除了
        /* Mark the old rule for removal in the next version. */
        cls_rule_make_invisible_in_version(&old_rule->cr, ofm->version);

        //一个流表会插入多个地方保存，这个只是将流表从ofproto中删除，后面会将流表从分类器中删除
        /* Remove the old rule from data structures. */
        ofproto_rule_remove__(ofproto, old_rule);

//最后再 ofproto_flow_mod_finish 中将old_rule从分类器中删除 
ofproto_flow_mod_finish

/* Table overflow policy. */

/* Chooses and updates 'rulep' with a rule to evict from 'table'.  Sets 'rulep'
 * to NULL if the table is not configured to evict rules or if the table
 * contains no evictable rules.  (Rules with a readlock on their evict rwlock,
 * or with no timeouts are not evictable.) */
static bool
choose_rule_to_evict(struct oftable *table, struct rule **rulep)
    OVS_REQUIRES(ofproto_mutex)
{
    struct eviction_group *evg;

    *rulep = NULL;
    //overflow_policy不为evict，则返回false，说明不能强制删除流表
    if (!table->eviction) {
        return false;
    }

    //下面的两层循环用来找到最适合被删除的rule:
    //a. 从rule个数最多的group中选择，由于table->eviction_groups_by_size已经按size排序了，直接取出来即可
    //b. 从 evict_group 取出优先级最高的rule，由于evg->rules也是排好序的，取第一个即可。
    /* In the common case, the outer and inner loops here will each be entered
     * exactly once:
     *
     *   - The inner loop normally "return"s in its first iteration.  If the
     *     eviction group has any evictable rules, then it always returns in
     *     some iteration.
     *
     *   - The outer loop only iterates more than once if the largest eviction
     *     group has no evictable rules.
     *
     *   - The outer loop can exit only if table's 'max_flows' is all filled up
     *     by unevictable rules. */
    //外层循环取出 rule 个数最多的 group
    HEAP_FOR_EACH (evg, size_node, &table->eviction_groups_by_size) {
        struct rule *rule;

        //取出group中，优先级最大的rule。优先级大意味着超时时间将近。
        HEAP_FOR_EACH (rule, evg_node, &evg->rules) {
            *rulep = rule;
            return true;
        }
    }

    return false;
}

也可参考：ovs 删除openflow流表 - 简书 (jianshu.com)