vpp acl-plugin 实现分析

vpp版本：v23.06

一、CLI

• 添加acl，一个acl里由多条规则，规则之间用逗号隔开

set acl-plugin acl [index <idx>] <permit|deny|permit+reflect> 
               src <PREFIX> dst <PREFIX> [proto X] [sport X[-Y]] [dport X[-Y]] 
               [tcpflags <int> mask <int>] [tag FOO] {use comma separated list 
               for multiple rules}

 eg：

• 删除acl

delete acl-plugin acl index <idx>

eg：delete acl-plugin acl index 13

• acl应用于接口

set acl-plugin interface <interface> <input|output> <acl INDEX> [del] 

eg：set acl-plugin interface dpdk0 input acl 13

• 设置acl 有状态会话超时时间

set acl-plugin session timeout {{udp idle}|tcp {idle|transient}} <seconds>

二、实现分析

关键数据结构

fa_5tuple_t 数据存储，查找等都是围绕这个数据在进行。

typedef union {
  struct {
    union {
      struct {
        /* we put the IPv4 addresses
           after padding so we can still
           use them as (shorter) key together with
           L4 info */
        u32 l3_zero_pad[6];
        ip4_address_t ip4_addr[2];
      };
      ip6_address_t ip6_addr[2];
    };
    fa_session_l4_key_t l4;
    /* This field should align with u64 value in bihash_40_8 and bihash_16_8 keyvalue struct */
    fa_packet_info_t pkt;
  };
  clib_bihash_kv_40_8_t kv_40_8;
  struct {
    u64 padding_for_kv_16_8[3];
    clib_bihash_kv_16_8_t kv_16_8;
  };
} fa_5tuple_t;

1、添加acl

acl_add_list()函数分析

static int
acl_add_list (u32 count, vl_api_acl_rule_t rules[],
	      u32 * acl_list_index, u8 * tag)
{
  acl_main_t *am = &acl_main;
  acl_list_t *a;
  acl_rule_t *r;
  acl_rule_t *acl_new_rules = 0;
  size_t tag_len;
  int i;

  tag_len = clib_strnlen ((const char *) tag, sizeof (a->tag));
  if (tag_len == sizeof (a->tag))
    return VNET_API_ERROR_INVALID_VALUE;

  if (am->trace_acl > 255)
    clib_warning ("API dbg: acl_add_list index %d tag %s", *acl_list_index,
		  tag);

  /* check if what they request is consistent */
  for (i = 0; i < count; i++)
    {
      if (acl_api_invalid_prefix (&rules[i].src_prefix))
	return VNET_API_ERROR_INVALID_SRC_ADDRESS;
      if (acl_api_invalid_prefix (&rules[i].dst_prefix))
	return VNET_API_ERROR_INVALID_DST_ADDRESS;
      if (rules[i].src_prefix.address.af != rules[i].dst_prefix.address.af)
	return VNET_API_ERROR_INVALID_SRC_ADDRESS;
      if (ntohs (rules[i].srcport_or_icmptype_first) >
	  ntohs (rules[i].srcport_or_icmptype_last))
	return VNET_API_ERROR_INVALID_VALUE_2;
      if (ntohs (rules[i].dstport_or_icmpcode_first) >
	  ntohs (rules[i].dstport_or_icmpcode_last))
	return VNET_API_ERROR_INVALID_VALUE_2;
    }

  if (*acl_list_index != ~0)
    {
      /* They supplied some number, let's see if this ACL exists */
      if (pool_is_free_index (am->acls, *acl_list_index))
	{
	  /* tried to replace a non-existent ACL, no point doing anything */
	  clib_warning
	    ("acl-plugin-error: Trying to replace nonexistent ACL %d (tag %s)",
	     *acl_list_index, tag);
	  return VNET_API_ERROR_NO_SUCH_ENTRY;
	}
    }
  if (0 == count)
    {
      clib_warning
	("acl-plugin-warning: supplied no rules for ACL %d (tag %s)",
	 *acl_list_index, tag);
    }

  /* Create and populate the rules */
  if (count > 0)
    vec_validate (acl_new_rules, count - 1);

  for (i = 0; i < count; i++)
    {
      r = vec_elt_at_index (acl_new_rules, i);
      clib_memset (r, 0, sizeof (*r));
      r->is_permit = rules[i].is_permit;
      r->is_ipv6 = rules[i].src_prefix.address.af;
      ip_address_decode (&rules[i].src_prefix.address, &r->src);
      ip_address_decode (&rules[i].dst_prefix.address, &r->dst);
      r->src_prefixlen = rules[i].src_prefix.len;
      r->dst_prefixlen = rules[i].dst_prefix.len;
      r->proto = rules[i].proto;
      r->src_port_or_type_first = ntohs (rules[i].srcport_or_icmptype_first);
      r->src_port_or_type_last = ntohs (rules[i].srcport_or_icmptype_last);
      r->dst_port_or_code_first = ntohs (rules[i].dstport_or_icmpcode_first);
      r->dst_port_or_code_last = ntohs (rules[i].dstport_or_icmpcode_last);
      r->tcp_flags_value = rules[i].tcp_flags_value;
      r->tcp_flags_mask = rules[i].tcp_flags_mask;
    }

  if (~0 == *acl_list_index)
    {
      /* Get ACL index */
      pool_get_aligned (am->acls, a, CLIB_CACHE_LINE_BYTES);
      clib_memset (a, 0, sizeof (*a));
      /* Will return the newly allocated ACL index */
      *acl_list_index = a - am->acls;
    }
  else
    {
      a = am->acls + *acl_list_index;
      /* Get rid of the old rules */
      if (a->rules)
	vec_free (a->rules);
    }
  a->rules = acl_new_rules;
  memcpy (a->tag, tag, tag_len + 1);
  if (am->trace_acl > 255)
    warning_acl_print_acl (am->vlib_main, am, *acl_list_index);
  if (am->reclassify_sessions)
    {
      /* a change in an ACLs if they are applied may mean a new policy epoch */
      policy_notify_acl_change (am, *acl_list_index);
    }
  validate_and_reset_acl_counters (am, *acl_list_index);
  acl_plugin_lookup_context_notify_acl_change (*acl_list_index);
  return 0;
}

acl_add_list 主要工作：

• 检查ip4/6地址正确性；
• 分配新的空间存储传入的rules；
• acl如果不存在就分配acl空间，如果存在就释放原来的rules，赋值新的rules；
• 检查reclassify_sessions（这个变量只对有状态acl有用，也就是创建session的acl），如果不为0，改变acl和接口关联的值，这个值的作用是acl应用于接口后，匹配数据包创建session，会把这个值赋给session，每次数据包来匹配到这个session，都会检查这个值和原始值相同不，不同说明acl被改变了，需要重新匹配创建会话；
• 初始化统计相关的数据；
• 初始化acl lookup相关数据。acl在数据包匹配时有两种方式，一种使用linear_multi_acl_match_5tuple进行匹配，也就是依次遍历这个接口配置的acl，遍历acl中的rules，直到找到第一个匹配项，退出，可以看出，效率相对较低，第二种就是在将acl应用于接口时，创建acl_lookup_hash key value关系，进行hash查找，这个在下面具体分析。

acl_plugin_lookup_context_notify_acl_change ()函数分析

新建直接看hash_acl_add()函数实现

void acl_plugin_lookup_context_notify_acl_change(u32 acl_num)
{
  acl_main_t *am = &acl_main;
  if (acl_plugin_acl_exists(acl_num)) {
    if (hash_acl_exists(am, acl_num)) {
        /* this is a modification, clean up the older entries */
        hash_acl_delete(am, acl_num);
    }
    hash_acl_add(am, acl_num);
  } else {
    /* this is a deletion notification */
    hash_acl_delete(am, acl_num);
  }
}

void hash_acl_add(acl_main_t *am, int acl_index)
{
  DBG("HASH ACL add : %d", acl_index);
  int i;
  acl_rule_t *acl_rules = am->acls[acl_index].rules;
  vec_validate(am->hash_acl_infos, acl_index);
  hash_acl_info_t *ha = vec_elt_at_index(am->hash_acl_infos, acl_index);
  clib_memset(ha, 0, sizeof(*ha));
  ha->hash_acl_exists = 1;

  /* walk the newly added ACL entries and ensure that for each of them there
     is a mask type, increment a reference count for that mask type */

  /* avoid small requests by preallocating the entire vector before running the additions */
  if (vec_len(acl_rules) > 0) {
    vec_validate(ha->rules, vec_len(acl_rules)-1);
    vec_reset_length(ha->rules);
  }

  for(i=0; i < vec_len(acl_rules); i++) {
    hash_ace_info_t ace_info;
    fa_5tuple_t mask;
    clib_memset(&ace_info, 0, sizeof(ace_info));
    ace_info.acl_index = acl_index;
    ace_info.ace_index = i;

    make_mask_and_match_from_rule(&mask, &acl_rules[i], &ace_info);
    mask.pkt.flags_reserved = 0b000;
    ace_info.base_mask_type_index = assign_mask_type_index(am, &mask);
    /* assign the mask type index for matching itself */
    ace_info.match.pkt.mask_type_index_lsb = ace_info.base_mask_type_index;
    DBG("ACE: %d mask_type_index: %d", i, ace_info.base_mask_type_index);
    vec_add1(ha->rules, ace_info);
  }
  /*
   * if an ACL is applied somewhere, fill the corresponding lookup data structures.
   * We need to take care if the ACL is not the last one in the vector of ACLs applied to the interface.
   */
  if (acl_index < vec_len(am->lc_index_vec_by_acl)) {
    u32 *lc_index;
    vec_foreach(lc_index, am->lc_index_vec_by_acl[acl_index]) {
      hash_acl_reapply(am, *lc_index, acl_index);
    }
  }
}

hash_acl_add主要工作：

• 在am->hash_acl_infos中添加acl的hash rules， 这里的hash rules和acl rules一一对应，区别是这里存的最终数据是通过mask &操作后的数据，rule里面存的也是一个fa_5tuple_t结构，比如acl配置的rule是src 192.168.2.2/24，那么hash rule里存在 fa_5tuple_t 的ip4_addr 的数据其实192.168.2.0。fa_5tuple_t的其他数据类似。生成的mask会创建一个模板，hash rule里存的是mask 模板的id，便于节约存储空间。这些hash rules存在于hash_acl_info_t 中，这里生成的hash_acl_info_t 会在将acl应用于接口中使用；

• hash_acl_reapply函数是此acl已经用于接口了才会调用，新建acl不会进入此流程；

2、将acl应用于接口

acl_interface_add_del_inout_acl() 分析

• 将acl index 存入sw_if_index （sw_if_index 是接口的索引id）对应的am->input_acl_vec_by_sw_if_index， input_acl_vec_by_sw_if_index你把它当成是二位数组，通过sw_if_index确定这个二维数组的行，acl index 就依次存入sw_if_index确定的一维数组里。其实这里是使用的可变矢量存储，存储空间是连续的。从这里看出，一个接口是可以配置多个acl的；
• 调用acl_interface_set_inout_acl_list()函数；

acl_interface_set_inout_acl_list()分析

static int
acl_interface_set_inout_acl_list (acl_main_t * am, u32 sw_if_index,
				  u8 is_input, u32 * vec_acl_list_index,
				  int *may_clear_sessions)
{
  u32 *pacln;
  uword *seen_acl_bitmap = 0;
  uword *old_seen_acl_bitmap = 0;
  uword *change_acl_bitmap = 0;
  int acln;
  int rv = 0;


  if (am->trace_acl > 255)
    clib_warning
      ("API dbg: acl_interface_set_inout_acl_list: sw_if_index %d is_input %d acl_vec: [%U]",
       sw_if_index, is_input, format_vec32, vec_acl_list_index, "%d");

  vec_foreach (pacln, vec_acl_list_index)
  {
    if (acl_is_not_defined (am, *pacln))
      {
	/* ACL is not defined. Can not apply */
	clib_warning ("ERROR: ACL %d not defined", *pacln);
	rv = VNET_API_ERROR_NO_SUCH_ENTRY;
	goto done;
      }
    if (clib_bitmap_get (seen_acl_bitmap, *pacln))
      {
	/* ACL being applied twice within the list. error. */
	clib_warning ("ERROR: ACL %d being applied twice", *pacln);
	rv = VNET_API_ERROR_ENTRY_ALREADY_EXISTS;
	goto done;
      }
    seen_acl_bitmap = clib_bitmap_set (seen_acl_bitmap, *pacln, 1);
  }


  u32 **pinout_lc_index_by_sw_if_index =
    is_input ? &am->input_lc_index_by_sw_if_index : &am->
    output_lc_index_by_sw_if_index;

  u32 ***pinout_acl_vec_by_sw_if_index =
    is_input ? &am->input_acl_vec_by_sw_if_index : &am->
    output_acl_vec_by_sw_if_index;

  u32 ***pinout_sw_if_index_vec_by_acl =
    is_input ? &am->input_sw_if_index_vec_by_acl : &am->
    output_sw_if_index_vec_by_acl;

  vec_validate ((*pinout_acl_vec_by_sw_if_index), sw_if_index);

  clib_bitmap_validate (old_seen_acl_bitmap, 1);

  vec_foreach (pacln, (*pinout_acl_vec_by_sw_if_index)[sw_if_index])
  {
    old_seen_acl_bitmap = clib_bitmap_set (old_seen_acl_bitmap, *pacln, 1);
  }
  change_acl_bitmap =
    clib_bitmap_dup_xor (old_seen_acl_bitmap, seen_acl_bitmap);

  if (am->trace_acl > 255)
    clib_warning ("bitmaps: old seen %U new seen %U changed %U",
		  format_bitmap_hex, old_seen_acl_bitmap, format_bitmap_hex,
		  seen_acl_bitmap, format_bitmap_hex, change_acl_bitmap);

/* *INDENT-OFF* */
  clib_bitmap_foreach (acln, change_acl_bitmap)  {
    if (clib_bitmap_get(old_seen_acl_bitmap, acln)) {
      /* ACL is being removed. */
      if (acln < vec_len((*pinout_sw_if_index_vec_by_acl))) {
        int index = vec_search((*pinout_sw_if_index_vec_by_acl)[acln], sw_if_index);
        vec_del1((*pinout_sw_if_index_vec_by_acl)[acln], index);
      }
    } else {
      /* ACL is being added. */
      vec_validate((*pinout_sw_if_index_vec_by_acl), acln);
      vec_add1((*pinout_sw_if_index_vec_by_acl)[acln], sw_if_index);
    }
  }
/* *INDENT-ON* */

  vec_free ((*pinout_acl_vec_by_sw_if_index)[sw_if_index]);
  (*pinout_acl_vec_by_sw_if_index)[sw_if_index] =
    vec_dup (vec_acl_list_index);

  if (am->reclassify_sessions)
    {
      /* re-applying ACLs means a new policy epoch */
      increment_policy_epoch (am, sw_if_index, is_input);
    }
  else
    {
      /* if no commonalities between the ACL# - then we should definitely clear the sessions */
      if (may_clear_sessions && *may_clear_sessions
	  && !clib_bitmap_is_zero (change_acl_bitmap))
	{
	  acl_clear_sessions (am, sw_if_index);
	  *may_clear_sessions = 0;
	}
    }

  /*
   * prepare or delete the lookup context if necessary, and if context exists, set ACL list
   */
  vec_validate_init_empty ((*pinout_lc_index_by_sw_if_index), sw_if_index,
			   ~0);
  if (vec_len (vec_acl_list_index) > 0)
    {
      u32 lc_index = (*pinout_lc_index_by_sw_if_index)[sw_if_index];
      if (~0 == lc_index)
	{
	  lc_index =
	    acl_plugin.get_lookup_context_index (am->interface_acl_user_id,
						 sw_if_index, is_input);
	  (*pinout_lc_index_by_sw_if_index)[sw_if_index] = lc_index;
	}
      acl_plugin.set_acl_vec_for_context (lc_index, vec_acl_list_index);
    }
  else
    {
      if (~0 != (*pinout_lc_index_by_sw_if_index)[sw_if_index])
	{
	  acl_plugin.
	    put_lookup_context_index ((*pinout_lc_index_by_sw_if_index)
				      [sw_if_index]);
	  (*pinout_lc_index_by_sw_if_index)[sw_if_index] = ~0;
	}
    }
  /* ensure ACL processing is enabled/disabled as needed */
  acl_interface_inout_enable_disable (am, sw_if_index, is_input,
				      vec_len (vec_acl_list_index) > 0);

done:
  clib_bitmap_free (change_acl_bitmap);
  clib_bitmap_free (seen_acl_bitmap);
  clib_bitmap_free (old_seen_acl_bitmap);
  return rv;
}

acl_interface_set_inout_acl_list的主要工作：

重要代码段

  /*
   * prepare or delete the lookup context if necessary, and if context exists, set ACL list
   */
  vec_validate_init_empty ((*pinout_lc_index_by_sw_if_index), sw_if_index,
			   ~0);
  if (vec_len (vec_acl_list_index) > 0)
    {
      u32 lc_index = (*pinout_lc_index_by_sw_if_index)[sw_if_index];
      if (~0 == lc_index)
	{
	  lc_index =
	    acl_plugin.get_lookup_context_index (am->interface_acl_user_id,
						 sw_if_index, is_input);
	  (*pinout_lc_index_by_sw_if_index)[sw_if_index] = lc_index;
	}
      acl_plugin.set_acl_vec_for_context (lc_index, vec_acl_list_index);
    }
  else
    {
      if (~0 != (*pinout_lc_index_by_sw_if_index)[sw_if_index])
	{
	  acl_plugin.
	    put_lookup_context_index ((*pinout_lc_index_by_sw_if_index)
				      [sw_if_index]);
	  (*pinout_lc_index_by_sw_if_index)[sw_if_index] = ~0;
	}
    }

• 首先是检查此acl的有效性，是否多次在此接口配置同一acl，bitmap实现的；

• 在pinout_sw_if_index_vec_by_acl中建立acl和sw_if_index关系；

• 判断am->reclassify_sessions，非0 改变sw_if_index对应的p_epoch值，和acl_add_list中总用一样，是0，根据may_clear_sessions的值和这个接口配置的acl是否变化确定是否删除这个接口上的所有session。这里保证了接口上配置acl的有效性。

• 每个接口，方向对应生成一个lc_index，lc_index再和acls建立lookup context关系。主要函数是acl_plugin.get_lookup_context_index()(acl_plugin_get_lookup_context_index)，acl_plugin.set_acl_vec_for_contex()(acl_plugin_set_acl_vec_for_context);将lc_index存入pinout_lc_index_by_sw_if_index ；

• acl_interface_inout_enable_disable函数在接口处理node的适当位置加入acl node；没有这一步，数据包处理是进不到acl 处理节点的。

acl_plugin_get_lookup_context_index主要工作：

• 根据传入interface_acl_user_id（acl_init初始化，认为是唯一值就行了），sw_if_index，is_input在am->acl_lookup_contexts中生成唯一lc_index，此值在后续工作中非常重要；

/*
 * Prepare the sequential vector of ACL#s to lookup within a given context.
 * Any existing list will be overwritten. acl_list is a vector.
 */
static int acl_plugin_set_acl_vec_for_context (u32 lc_index, u32 *acl_list)
{
  int rv = 0;
  uword *seen_acl_bitmap = 0;
  u32 *pacln = 0;
  acl_main_t *am = &acl_main;
  acl_lookup_context_t *acontext;
  if (am->trace_acl) {
    u32 i;
    elog_acl_cond_trace_X1(am, (1), "LOOKUP-CONTEXT: set-acl-list lc_index %d", "i4", lc_index);
    for(i=0; i<vec_len(acl_list); i++) {
      elog_acl_cond_trace_X2(am, (1), "   acl-list[%d]: %d", "i4i4", i, acl_list[i]);
    }
  }  
  if (!acl_lc_index_valid(am, lc_index)) {
    clib_warning("BUG: lc_index %d is not valid", lc_index);
    return -1;
  }
  vec_foreach (pacln, acl_list)
  {
    if (pool_is_free_index (am->acls, *pacln))
      {
        /* ACL is not defined. Can not apply */
        clib_warning ("ERROR: ACL %d not defined", *pacln);
        rv = VNET_API_ERROR_NO_SUCH_ENTRY;
        goto done;
      }
    if (clib_bitmap_get (seen_acl_bitmap, *pacln))
      {
        /* ACL being applied twice within the list. error. */
        clib_warning ("ERROR: ACL %d being applied twice", *pacln);
        rv = VNET_API_ERROR_ENTRY_ALREADY_EXISTS;
        goto done;
      }
    seen_acl_bitmap = clib_bitmap_set (seen_acl_bitmap, *pacln, 1);
  }

  acontext = pool_elt_at_index(am->acl_lookup_contexts, lc_index);
  u32 *old_acl_vector = acontext->acl_indices;
  acontext->acl_indices = vec_dup(acl_list);

  unapply_acl_vec(lc_index, old_acl_vector);
  unlock_acl_vec(lc_index, old_acl_vector);
  lock_acl_vec(lc_index, acontext->acl_indices);
  apply_acl_vec(lc_index, acontext->acl_indices);

  vec_free(old_acl_vector);

done:
  clib_bitmap_free (seen_acl_bitmap);
  return rv;
}

acl_plugin_set_acl_vec_for_context主要工作：

• 首先是检查此acl的有效性，是否多次在此接口配置同一acl，bitmap实现的；

• 通过lc_index确定acontext，将acls保存在acontext->acl_indices中。因为是第一次应用于接口，不用考虑unapply_acl_vec(lc_index, old_acl_vector); unlock_acl_vec(lc_index, old_acl_vector);

• 通过lock_acl_vec()函数将lc_index存储在am->lc_index_vec_by_acl[acl]中，建立acl和acontext之间的关系；

• apply_acl_vec函数调用hash_acl_apply建立正在的lookup context hash表项；

void
hash_acl_apply(acl_main_t *am, u32 lc_index, int acl_index, u32 acl_position)
{
  int i;

  DBG0("HASH ACL apply: lc_index %d acl %d", lc_index, acl_index);
  if (!am->acl_lookup_hash_initialized) {
    BV (clib_bihash_init) (&am->acl_lookup_hash, "ACL plugin rule lookup bihash",
                           am->hash_lookup_hash_buckets, am->hash_lookup_hash_memory);
    am->acl_lookup_hash_initialized = 1;
  }

  vec_validate(am->hash_entry_vec_by_lc_index, lc_index);
  vec_validate(am->hash_acl_infos, acl_index);
  applied_hash_ace_entry_t **applied_hash_aces = get_applied_hash_aces(am, lc_index);

  hash_acl_info_t *ha = vec_elt_at_index(am->hash_acl_infos, acl_index);
  u32 **hash_acl_applied_lc_index = &ha->lc_index_list;

  int base_offset = vec_len(*applied_hash_aces);

  /* Update the bitmap of the mask types with which the lookup
     needs to happen for the ACLs applied to this lc_index */
  applied_hash_acl_info_t **applied_hash_acls = &am->applied_hash_acl_info_by_lc_index;
  vec_validate((*applied_hash_acls), lc_index);
  applied_hash_acl_info_t *pal = vec_elt_at_index((*applied_hash_acls), lc_index);

  /* ensure the list of applied hash acls is initialized and add this acl# to it */
  u32 index = vec_search(pal->applied_acls, acl_index);
  if (index != ~0) {
    clib_warning("BUG: trying to apply twice acl_index %d on lc_index %d, according to lc",
                 acl_index, lc_index);
    ASSERT(0);
    return;
  }
  vec_add1(pal->applied_acls, acl_index);
  u32 index2 = vec_search((*hash_acl_applied_lc_index), lc_index);
  if (index2 != ~0) {
    clib_warning("BUG: trying to apply twice acl_index %d on lc_index %d, according to hash h-acl info",
                 acl_index, lc_index);
    ASSERT(0);
    return;
  }
  vec_add1((*hash_acl_applied_lc_index), lc_index);

  /*
   * if the applied ACL is empty, the current code will cause a
   * different behavior compared to current linear search: an empty ACL will
   * simply fallthrough to the next ACL, or the default deny in the end.
   *
   * This is not a problem, because after vpp-dev discussion,
   * the consensus was it should not be possible to apply the non-existent
   * ACL, so the change adding this code also takes care of that.
   */


  vec_validate(am->hash_applied_mask_info_vec_by_lc_index, lc_index);

  /* since we know (in case of no split) how much we expand, preallocate that space */
  if (vec_len(ha->rules) > 0) {
    int old_vec_len = vec_len(*applied_hash_aces);
    vec_validate((*applied_hash_aces), old_vec_len + vec_len(ha->rules) - 1);
    vec_set_len ((*applied_hash_aces), old_vec_len);
  }

  /* add the rules from the ACL to the hash table for lookup and append to the vector*/
  for(i=0; i < vec_len(ha->rules); i++) {
    /*
     * Expand the applied aces vector to fit a new entry.
     * One by one not to upset split_partition() if it is called.
     */
    vec_resize((*applied_hash_aces), 1);

    int is_ip6 = ha->rules[i].match.pkt.is_ip6;
    u32 new_index = base_offset + i;
    applied_hash_ace_entry_t *pae = vec_elt_at_index((*applied_hash_aces), new_index);
    pae->acl_index = acl_index;
    pae->ace_index = ha->rules[i].ace_index;
    pae->acl_position = acl_position;
    pae->action = ha->rules[i].action;
    pae->hitcount = 0;
    pae->hash_ace_info_index = i;
    /* we might link it in later */
    pae->collision_head_ae_index = ~0;
    pae->colliding_rules = NULL;
    pae->mask_type_index = ~0;
    assign_mask_type_index_to_pae(am, lc_index, is_ip6, pae);
    u32 first_index = activate_applied_ace_hash_entry(am, lc_index, applied_hash_aces, new_index);
    if (am->use_tuple_merge)
      check_collision_count_and_maybe_split(am, lc_index, is_ip6, first_index);
  }
  remake_hash_applied_mask_info_vec(am, applied_hash_aces, lc_index);
}

hash_acl_apply函数主要工作：

• 获取此acl在am->hash_acl_infos对应的hash acl，ha（acl_add_list 中添加的），将lc_index添加到ha->lc_index_list；
• 获取applied_hash_aces，遍历ha的rules，通过activate_applied_ace_hash_entry在am->acl_lookup_hash中添加key value表项；
• 判断am->use_tuple_merge是否进行冲突元素合并，默认冲突最大值是39，applied_hash_aces的colliding_rules大于39就会进行合并；
• 调用remake_hash_applied_mask_info_vec，这个函数根据applied_hash_ace_entry_t里的mask_type_index创建hash_applied_mask_info_t存入am->hash_applied_mask_info_vec_by_lc_index，后面数据包匹配会用到这里的数据（mask值），mask_type_index就是前面说的acl rule 生成的mask；

void
fill_applied_hash_ace_kv(acl_main_t *am,
                            applied_hash_ace_entry_t **applied_hash_aces,
                            u32 lc_index,
                            u32 new_index, clib_bihash_kv_48_8_t *kv)
{
  fa_5tuple_t *kv_key = (fa_5tuple_t *)kv->key;
  hash_acl_lookup_value_t *kv_val = (hash_acl_lookup_value_t *)&kv->value;
  applied_hash_ace_entry_t *pae = vec_elt_at_index((*applied_hash_aces), new_index);
  hash_acl_info_t *ha = vec_elt_at_index(am->hash_acl_infos, pae->acl_index);

  /* apply the mask to ace key */
  hash_ace_info_t *ace_info = vec_elt_at_index(ha->rules, pae->hash_ace_info_index);
  ace_mask_type_entry_t *mte = vec_elt_at_index(am->ace_mask_type_pool, pae->mask_type_index);

  u64 *pmatch = (u64 *) &ace_info->match;
  u64 *pmask = (u64 *)&mte->mask;
  u64 *pkey = (u64 *)kv->key;

  *pkey++ = *pmatch++ & *pmask++;
  *pkey++ = *pmatch++ & *pmask++;
  *pkey++ = *pmatch++ & *pmask++;
  *pkey++ = *pmatch++ & *pmask++;
  *pkey++ = *pmatch++ & *pmask++;
  *pkey++ = *pmatch++ & *pmask++;

  kv_key->pkt.mask_type_index_lsb = pae->mask_type_index;
  kv_key->pkt.lc_index = lc_index;
  kv_val->as_u64 = 0;
  kv_val->applied_entry_index = new_index;
}
static u32
activate_applied_ace_hash_entry(acl_main_t *am,
                            u32 lc_index,
                            applied_hash_ace_entry_t **applied_hash_aces,
                            u32 new_index)
{
  clib_bihash_kv_48_8_t kv;
  ASSERT(new_index != ~0);
  DBG("activate_applied_ace_hash_entry lc_index %d new_index %d", lc_index, new_index);

  fill_applied_hash_ace_kv(am, applied_hash_aces, lc_index, new_index, &kv);

  DBG("APPLY ADD KY: %016llx %016llx %016llx %016llx %016llx %016llx",
			kv.key[0], kv.key[1], kv.key[2],
			kv.key[3], kv.key[4], kv.key[5]);

  clib_bihash_kv_48_8_t result;
  hash_acl_lookup_value_t *result_val = (hash_acl_lookup_value_t *)&result.value;
  int res = BV (clib_bihash_search) (&am->acl_lookup_hash, &kv, &result);
  ASSERT(new_index != ~0);
  ASSERT(new_index < vec_len((*applied_hash_aces)));
  if (res == 0) {
    u32 first_index = result_val->applied_entry_index;
    ASSERT(first_index != ~0);
    ASSERT(first_index < vec_len((*applied_hash_aces)));
    /* There already exists an entry or more. Append at the end. */
    DBG("A key already exists, with applied entry index: %d", first_index);
    add_colliding_rule(am, applied_hash_aces, first_index, new_index);
    return first_index;
  } else {
    /* It's the very first entry */
    hashtable_add_del(am, &kv, 1);
    ASSERT(new_index != ~0);
    add_colliding_rule(am, applied_hash_aces, new_index, new_index);
    return new_index;
  }
}

activate_applied_ace_hash_entry主要工作：

• 调用fill_applied_hash_ace_kv填充key， value，其实用的就是acl_add_list添加的hash rules。这里面数据很多，要多看才能理清楚；
• 首先是BV (clib_bihash_search)进行查找，返回值等于0，说明key value已经存在，通过add_colliding_rule将新的hash ace（applied_hash_aces）相关信息添加到冲突链表中。返回值不为0，说明此key value是第一次添加，调用hashtable_add_del将新key value添加到am->acl_lookup_hash中，并初始化冲突链表，其实就是把自己的信息添加到head_pae->colliding_rules中；

以上操作结束，当数据包到达acl node前所有数据都初始化完成，特别是lookup hash 表项初始化完成。

3、数据包经过acl node分析

以“acl-plugin-in-ip4-fa”节点为例，其他节点逻辑类似。

首先调用acl_fa_node_common_prepare_fn函数将数据包的信息添加到fa_5tuple_t结构体中，并生成一个hash值，这个值在session hash查找中使用。

然后调用acl_fa_inner_node_fn进行数据包匹配。

acl_fa_inner_node_fn主要工作：

• 检查with_stateful_datapath，是否进行session hash查找，session的创建是在匹配数据包后，当第一个数据包进来时，查询不到；
• 进入while中switch 的case x，根据数据包个数的不同进行预取操作，提高处理速度；
• 进入case 1，判断f_sess_id是否有效，有效调用process_established_session进行会话状态处理，并返回action，决定这个数据包后续动作；判断reclassify_sessions，在acl应用于接口时会生成一个policy_epoch值（前面acl_add_list时提到过），如果发生改变，说明接口和acl的关系发生改变，删除此sesion；
• 检查acl_check_needed，此数据包查询到了session，此值为0，没有查询到session此值为1。调用acl_plugin_match_5tuple_inline对数据包需要进行acl 匹配，这时就用到lc_index0。这个值就是在将acl应用于接口产生的。结果匹配绝对数据包的下一步操作；
• 如果匹配且打开了acl统计功能，进行acl 匹配统计；
• action 0是drop，action 1是无状态acl ，action 2 是有状态acl，首先是检查是否能新建session，session总数是有限制的，然后回收超时的session，调用acl_fa_add_session添加session，process_established_session对session状态进行更新；

acl_plugin_match_5tuple_inline主要工作：

• 匹配传入fa_5tuple，lc_index0，以及其他需要返回的指针；
• 判断am->use_hash_acl_matching，没有打开hash match只能使用linear_multi_acl_match_5tuple进行匹配，这种匹配很简单，遍历此接口配置了哪些acl，调用single_acl_match_5tuple依次匹配acl的rules，匹配到第一个就退出；打开了hash macth要判断是不是分片非首包（非0表示是），普通报文和分片首包使用hash_multi_acl_match_5tuple进行匹配，分片非首包还是使用linear_multi_acl_match_5tuple进行匹配；

hash_multi_acl_match_5tuple主要工作：

• 通过lc_index在am->hash_entry_vec_by_lc_index找到applied_hash_aces（在acl应用于接口时创建），applied_hash_aces这里面存的是所有应用于接口的所有acl的rules的hash rules，按照acl配置顺序存入。
• 调用multi_acl_match_get_applied_ace_index，传入数据包的fa_5tuple_t进行匹配；返回匹配的index，这个index 是配置在这个接口的所有acl的所有rules按顺序存入的index，通过这个index可以查询到具体acl index和rule index；
• 最终返回匹配的acl index，rule index，以及这个acl在所有配置到这个接口的acl的顺序位置，action；

multi_acl_match_get_applied_ace_index主要工作：

• 通过fa_5tuple_t里存的lc_index找到applied_hash_aces，hash_applied_mask_info_vec；
• 遍历hash_applied_mask_info_vec，其实就是遍历配置到这个接口所有的rules生成的mask（相同mask已经被合并，参见remake_hash_applied_mask_info_vec），数据包fa_5tuple_t与mask进行&操作生成的key在am->acl_lookup_hash进行查找，找到返回pae（applied_hash_ace_entry_t）的index，applied_hash_aces存的是配置到此接口的所有acl的rules的hash rule，通过pae 的index查找pae，如果有冲突链表，还要进行线性匹配；最后确定匹配applied_entry_index；因为所有acl的rules是按照顺序存入的，index越小的说明配置在前，这种逻辑就实现了acl的优先级功能；接口先配置的acl比后配置acl优先级高，acl内部rules排在前面的比后面的优先级高；

acl_fa_add_session主要工作：

• 根据数据包fa_5tuple_t生成key，创建session，并把session的index存入 value中，在am->fa_ip4_sessions_hash中创建双向hash key value；

acl session 在acl_fa_try_recycle_session回收外，还在acl-plugin-fa-cleaner-process节点中定时进行清理。