vpp启动load so，注册node，函数init

本专栏知识点是通过零声教育的线上课学习，进行梳理总结写下文章，对c/c++linux课程感兴趣的读者，可以点击链接https://xxetb.xetslk.com/s/12PH1r C/C++后台高级服务器课程介绍 详细查看课程的服务

vpp的初始化流程

在vpp中所有编译出来的plugin，都是以so动态库的形式存在的。

vpp在启动的时候

vpp启动的时候，vlib_plugin_early_init函数被调用，初始化了vpp plugins的so文件路径

/home/king/share/dpdk/dpdk-frame/vpp/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins路径就是vpp在make build之后编译生成so库文件的路径

vlib_plugin_early_init函数的实现

int
vlib_plugin_early_init (vlib_main_t * vm)
{
  plugin_main_t *pm = &vlib_plugin_main;

  if (pm->plugin_path == 0)
    pm->plugin_path = format (0, "%s%c", vlib_plugin_path, 0);
  //需要加载的so库具体在哪个路径下面
  clib_warning ("plugin path %s", pm->plugin_path);

  pm->plugin_by_name_hash = hash_create_string (0, sizeof (uword));
  pm->vlib_main = vm;

  return vlib_load_new_plugins (pm, 1 /* from_early_init */ );//加载so
}

调用流程：main->vlib_unix_main->vlib_plugin_early_init

vlib_plugin_early_init

vlib_plugin_early_init函数的作用就是加载/home/king/share/dpdk/dpdk-frame/vpp/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins文件夹下编译生成的所有的so库

int
vlib_load_new_plugins (plugin_main_t * pm, int from_early_init)
{
  DIR *dp;
  struct dirent *entry;
  struct stat statb;
  uword *p;
  plugin_info_t *pi;
  u8 **plugin_path;
  u32 *load_fail_indices = 0;
  int i;

  plugin_path = split_plugin_path (pm);

  for (i = 0; i < vec_len (plugin_path); i++)
    {
      dp = opendir ((char *) plugin_path[i]);

      if (dp == 0)
	continue;

      while ((entry = readdir (dp)))
	{
	  u8 *plugin_name;
	  u8 *filename;

	  if (pm->plugin_name_filter)
	    {
	      int j;
	      for (j = 0; j < vec_len (pm->plugin_name_filter); j++)
		if (entry->d_name[j] != pm->plugin_name_filter[j])
		  goto next;
	    }

	  filename = format (0, "%s/%s%c", plugin_path[i], entry->d_name, 0);

	  /* Only accept .so */
	  char *ext = strrchr ((const char *) filename, '.');
	  /* unreadable */
	  if (!ext || (strcmp (ext, ".so") != 0) ||
	      stat ((char *) filename, &statb) < 0)
	    {
	    ignore:
	      vec_free (filename);
	      continue;
	    }

	  /* a dir or other things which aren't plugins */
	  if (!S_ISREG (statb.st_mode))
	    goto ignore;

	  plugin_name = format (0, "%s%c", entry->d_name, 0);
	  /* Have we seen this plugin already? */
	  p = hash_get_mem (pm->plugin_by_name_hash, plugin_name);
	  if (p == 0)
	    {
	      /* No, add it to the plugin vector */
	      vec_add2 (pm->plugin_info, pi, 1);
	      pi->name = plugin_name;
	      pi->filename = filename;
	      pi->file_info = statb;
	      hash_set_mem (pm->plugin_by_name_hash, plugin_name,
			    pi - pm->plugin_info);
	    }
	next:
	  ;
	}
      closedir (dp);
      vec_free (plugin_path[i]);
    }
  vec_free (plugin_path);


  /*
   * Sort the plugins by name. This is important.
   * API traces contain absolute message numbers.
   * Loading plugins in directory (vs. alphabetical) order
   * makes trace replay incredibly fragile.
   */
  vec_sort_with_function (pm->plugin_info, plugin_name_sort_cmp);

  /*
   * Attempt to load the plugins
   */
  for (i = 0; i < vec_len (pm->plugin_info); i++)
    {
      pi = vec_elt_at_index (pm->plugin_info, i);

      if (load_one_plugin (pm, pi, from_early_init))
	{
	  /* Make a note of any which fail to load */
	  vec_add1 (load_fail_indices, i);
	  hash_unset_mem (pm->plugin_by_name_hash, pi->name);
	  vec_free (pi->name);
	  vec_free (pi->filename);
	}
    }

  /* Remove plugin info vector elements corresponding to load failures */
  if (vec_len (load_fail_indices) > 0)
    {
      for (i = vec_len (load_fail_indices) - 1; i >= 0; i--)
	vec_delete (pm->plugin_info, 1, load_fail_indices[i]);
      vec_free (load_fail_indices);
    }

  /* Recreate the plugin name hash */
  for (i = 0; i < vec_len (pm->plugin_info); i++)
    {
      pi = vec_elt_at_index (pm->plugin_info, i);
      hash_unset_mem (pm->plugin_by_name_hash, pi->name);
      hash_set_mem (pm->plugin_by_name_hash, pi->name, pi - pm->plugin_info);
    }

  return 0;
}

其中vlib_plugin_early_init又调用了load_one_plugin

load_one_plugin

load_one_plugin函数的作用是加载一个so库

static int
load_one_plugin (plugin_main_t * pm, plugin_info_t * pi, int from_early_init)
{
  void *handle;
  clib_error_t *error;
  elf_main_t em = { 0 };
  elf_section_t *section;
  u8 *data;
  char *version_required;
  vlib_plugin_registration_t *reg;
  plugin_config_t *pc = 0;
  uword *p;

  if (elf_read_file (&em, (char *) pi->filename))
    return -1;

  error = elf_get_section_by_name (&em, ".vlib_plugin_registration",
				   &section);
  if (error)
    {
      clib_warning ("Not a plugin: %s\n", (char *) pi->name);
      return -1;
    }

  data = elf_get_section_contents (&em, section->index, 1);
  reg = (vlib_plugin_registration_t *) data;

  if (vec_len (data) != sizeof (*reg))
    {
      clib_warning ("vlib_plugin_registration size mismatch in plugin %s\n",
		    (char *) pi->name);
      goto error;
    }

  p = hash_get_mem (pm->config_index_by_name, pi->name);
  if (p)
    {
      pc = vec_elt_at_index (pm->configs, p[0]);
      if (pc->is_disabled)
	{
	  clib_warning ("Plugin disabled: %s", pi->name);
	  goto error;
	}
      if (reg->default_disabled && pc->is_enabled == 0)
	{
	  clib_warning ("Plugin disabled (default): %s", pi->name);
	  goto error;
	}
    }
  else if (reg->default_disabled)
    {
      clib_warning ("Plugin disabled (default): %s", pi->name);
      goto error;
    }

  version_required = str_array_to_vec ((char *) &reg->version_required,
				       sizeof (reg->version_required));

  if ((strlen (version_required) > 0) &&
      (strncmp (vlib_plugin_app_version, version_required,
		strlen (version_required))))
    {
      clib_warning ("Plugin %s version mismatch: %s != %s",
		    pi->name, vlib_plugin_app_version, reg->version_required);
      if (!(pc && pc->skip_version_check == 1))
	{
	  vec_free (version_required);
	  goto error;
	}
    }

  vec_free (version_required);
  vec_free (data);
  elf_main_free (&em);

  handle = dlopen ((char *) pi->filename, RTLD_LAZY);//打开某个具体的so库

  if (handle == 0)
    {
      clib_warning ("%s", dlerror ());
      clib_warning ("Failed to load plugin '%s'", pi->name);
      os_exit (1);
    }

  pi->handle = handle;
  //取出so库文件对应的标签，对应每个模块的初始化注册时候的VLIB_PLUGIN_REGISTER
  reg = dlsym (pi->handle, "vlib_plugin_registration");

  if (reg == 0)
    {
      /* This should never happen unless somebody chagnes registration macro */
      clib_warning ("Missing plugin registration in plugin '%s'", pi->name);
      os_exit (1);
    }

  pi->reg = reg;
  pi->version = str_array_to_vec ((char *) &reg->version,
				  sizeof (reg->version));

  if (reg->early_init)
    {
      clib_error_t *(*ei) (vlib_main_t *);
      void *h;

      //h就是VLIB_INIT_FUNCTION初始化的init函数
      h = dlsym (pi->handle, reg->early_init);
      if (h)
	{
	  ei = h;
	  error = (*ei) (pm->vlib_main);
	  if (error)
	    {
	      clib_error_report (error);
	      os_exit (1);
	    }
	}
      else
	clib_warning ("Plugin %s: early init function %s set but not found",
		      (char *) pi->name, reg->early_init);
    }

  if (reg->description)
    clib_warning ("Loaded plugin: %s (%s)", pi->name, reg->description);
  else
    clib_warning ("Loaded plugin: %s", pi->name);

  return 0;
error:
  vec_free (data);
  elf_main_free (&em);
  return -1;
}

dlsym(pi->handle, "vlib_plugin_registration")，dlsym函数主要是搜寻每个plugin注册时候对应的标签，搜索成功即plugin注册成功

#define VLIB_PLUGIN_REGISTER() \
  vlib_plugin_registration_t vlib_plugin_registration \
  __attribute__((__section__(".vlib_plugin_registration")))

VLIB_PLUGIN_REGISTER() = {

	.version = "1.0",
	.description = "sample of flowtable",

};

dlsym(pi->handle, reg->early_init)返回值就是vpp plugin在初始化的时候VLIB_INIT_FUNCTION初始化的init函数

typedef struct _vlib_init_function_list_elt
{
//_vlib_init_function_list_elt_t其实是一个单链表
  struct _vlib_init_function_list_elt *next_init_function;
  vlib_init_function_t *f;//每个function都会单独的运行
} _vlib_init_function_list_elt_t;
//每个plugin的初始化函数在程序一开始执行的时候就会运行，遍历链表并且运行
/* Declaration is global (e.g. not static) so that init functions can
   be called from other modules to resolve init function depend. */

#define VLIB_DECLARE_INIT_FUNCTION(x, tag)                      \
vlib_init_function_t * _VLIB_INIT_FUNCTION_SYMBOL (x, tag) = x; \
static void __vlib_add_##tag##_function_##x (void)              \
    __attribute__((__constructor__)) ;                          \
static void __vlib_add_##tag##_function_##x (void)              \
{                                                               \
 vlib_main_t * vm = vlib_get_main();                            \
 static _vlib_init_function_list_elt_t _vlib_init_function;     \
 _vlib_init_function.next_init_function                         \
    = vm->tag##_function_registrations;                         \
  vm->tag##_function_registrations = &_vlib_init_function;      \
 _vlib_init_function.f = &x;                                    \
}

#define VLIB_INIT_FUNCTION(x) VLIB_DECLARE_INIT_FUNCTION(x,init)

//flowtable插件加载到vpp框架中的时候，会调用flowtable_init这个函数
static clib_error_t *flowtable_init(vlib_main_t *vm) {

	clib_error_t *error = 0;

	flowtable_main_t *fm = &flowtable_main;
	fm->vnet_main = vnet_get_main();
	fm->vlib_main = vm;

	flow_info_t *flow;
	pool_get_aligned(fm->flows, flow, CLIB_CACHE_LINE_BYTES);
	pool_put(fm->flows, flow);

	BV(clib_bihash_init)(&fm->flows_ht, "flow hash table", FM_NUM_BUCKETS, FM_MEMORY_SIZE);

	return error;
}
//每一个node的初始化函数
VLIB_INIT_FUNCTION(flowtable_init);

VLIB_INIT_FUNCTION的调用流程

vlib_unix_main实现的功能

在vpp初始化的过程中，通过调用vlib_unix_main实现了两个功能：

1. 加载so
2. 启动协程（通过调用clib_calljmp函数）

加载so的过程前面已经总结过，这里总结下启动协程的过程。

协程跳转的实现方式

协程的跳转总共有三种实现方式：

1. setjmp/longjmp
2. ucontext
3. 汇编asm

这里vpp是采用setjmp/longjmp的实现方式。

VLIB_INIT_FUNCTION的实现

通过上面的代码可以看到，协程启动了一个thread0，thread0的实现如下：

thread0线程又调用了vlib_main，在这个vlib_main函数中就实现了vpp的所有plugins的初始化和注册。我们这里先看初始化流程的实现。通过调用vlib_main->vlib_call_all_init_functions->vlib_call_init_exit_functions就实现了vpp的所有模块的初始化：

由上图可知，通过调用i->f(vm)就实现了对vpp plugins中所有node的init_function的初始化

VLIB_REGISTER_NODE的调用流程

将vpp的静态node添加到了vpp运行时的动态node中去

代码定义

#define VLIB_REGISTER_NODE(x,...)                                       \
    __VA_ARGS__ vlib_node_registration_t x;                             \
static void __vlib_add_node_registration_##x (void)                     \
    __attribute__((__constructor__)) ;                                  \
static void __vlib_add_node_registration_##x (void)                     \
{                                                                       \
    vlib_main_t * vm = vlib_get_main();                                 \
    x.next_registration = vm->node_main.node_registrations;             \
    vm->node_main.node_registrations = &x;                              \
}                                                                       \
__VA_ARGS__ vlib_node_registration_t x

vlib_register_all_static_nodes函数

vlib_main函数中调用了vlib_register_all_static_nodes去注册vpp plugin中所有的node节点。

void
vlib_register_all_static_nodes (vlib_main_t * vm)
{
    vlib_node_registration_t *r;

    static char *null_node_error_strings[] = {
        "blackholed packets",
    };

    static vlib_node_registration_t null_node_reg = {
        .function = null_node_fn,
        .vector_size = sizeof (u32),
        .name = "null-node",
        .n_errors = 1,
        .error_strings = null_node_error_strings,
    };

    /* make sure that node index 0 is not used by
     real node */
    register_node (vm, &null_node_reg);

    r = vm->node_main.node_registrations;
    while (r)
    {
        register_node (vm, r);
        r = r->next_registration;
    }
}

vlib_register_all_static_nodes函数的核心就是调用了register_node函数。register_node函数的核心作用就是将register_node中的每一个节点都赋值到进程的运行时的参数中去（为每一个进程中的node list做准备）：

VLIB_CLI_COMMAND宏定义

代码定义

#define VLIB_CLI_COMMAND(x,...)                                         \
    __VA_ARGS__ vlib_cli_command_t x;                                   \
static void __vlib_cli_command_registration_##x (void)                  \
    __attribute__((__constructor__)) ;                                  \
static void __vlib_cli_command_registration_##x (void)                  \
{                                                                       \
    vlib_main_t * vm = vlib_get_main();                                 \
    vlib_cli_main_t *cm = &vm->cli_main;                                \
    x.next_cli_command = cm->cli_command_registrations;                 \
    cm->cli_command_registrations = &x;                                 \
}                                                                       \
__VA_ARGS__ vlib_cli_command_t x

cli_command_registrations同理也是一个单链表，command的初始化实际上类似于前面的VLIB_INIT_FUNCTION的初始化流程。

VNET_FEATURE_INIT宏定义

将一类的、具有相同特色的node放在一起，这里主要表述的是一种逻辑的关系，比如说都需要对数据的IP头进行处理的数据。这里需要注意的是并不是每个node都需要，这里重点强调下只是部分的node需要这种逻辑关系。

代码定义

#define VNET_FEATURE_INIT(x,...)				\
  __VA_ARGS__ vnet_feature_registration_t vnet_feat_##x;	\
static void __vnet_add_feature_registration_##x (void)		\
  __attribute__((__constructor__)) ;				\
static void __vnet_add_feature_registration_##x (void)		\
{								\
  vnet_feature_main_t * fm = &feature_main;			\
  vnet_feat_##x.next = fm->next_feature;			\
  fm->next_feature = & vnet_feat_##x;				\
}								\
__VA_ARGS__ vnet_feature_registration_t vnet_feat_##x

这里需要注意的是一个FEATURE中可能具有多个相同名字的node。

static clib_error_t *
vnet_feature_init (vlib_main_t * vm)
{
  vnet_feature_main_t *fm = &feature_main;
  vnet_feature_registration_t *freg;
  vnet_feature_arc_registration_t *areg;
  u32 arc_index = 0;

  fm->arc_index_by_name = hash_create_string (0, sizeof (uword));
  areg = fm->next_arc;

  /* process feature arc registrations */
  while (areg)
    {
      char *s;
      int i = 0;
      areg->feature_arc_index = arc_index;
      if (areg->arc_index_ptr)
	*areg->arc_index_ptr = arc_index;
      hash_set_mem (fm->arc_index_by_name, areg->arc_name,
		    pointer_to_uword (areg));

      /* process start nodes */
      while ((s = areg->start_nodes[i]))
	{
	  i++;
	}
      areg->n_start_nodes = i;

      /* next */
      areg = areg->next;
      arc_index++;
    }

  vec_validate (fm->next_feature_by_arc, arc_index - 1);
  vec_validate (fm->feature_nodes, arc_index - 1);
  vec_validate (fm->feature_config_mains, arc_index - 1);
  vec_validate (fm->next_feature_by_name, arc_index - 1);
  vec_validate (fm->sw_if_index_has_features, arc_index - 1);
  vec_validate (fm->feature_count_by_sw_if_index, arc_index - 1);

  freg = fm->next_feature;
  while (freg)
    {
      vnet_feature_registration_t *next;
      uword *p = hash_get_mem (fm->arc_index_by_name, freg->arc_name);
      if (p == 0)
	{
	  /* Don't start vpp with broken features arcs */
	  clib_warning ("Unknown feature arc '%s'", freg->arc_name);
	  os_exit (1);
	}

      areg = uword_to_pointer (p[0], vnet_feature_arc_registration_t *);
      arc_index = areg->feature_arc_index;

      next = freg->next;
      freg->next = fm->next_feature_by_arc[arc_index];
      fm->next_feature_by_arc[arc_index] = freg;

      /* next */
      freg = next;
    }

  areg = fm->next_arc;
  while (areg)
    {
      clib_error_t *error;
      vnet_feature_config_main_t *cm;
      vnet_config_main_t *vcm;

      arc_index = areg->feature_arc_index;
      cm = &fm->feature_config_mains[arc_index];
      vcm = &cm->config_main;
      if ((error = vnet_feature_arc_init (vm, vcm,
					  areg->start_nodes,
					  areg->n_start_nodes,
					  fm->next_feature_by_arc[arc_index],
					  &fm->feature_nodes[arc_index])))
	{
	  clib_error_report (error);
	  os_exit (1);
	}

      fm->next_feature_by_name[arc_index] =
	hash_create_string (0, sizeof (uword));
      freg = fm->next_feature_by_arc[arc_index];

      while (freg)
	{
	  hash_set_mem (fm->next_feature_by_name[arc_index],
			freg->node_name, pointer_to_uword (freg));
	  freg = freg->next;
	}

      /* next */
      areg = areg->next;
      arc_index++;
    }

  return 0;
}

VLIB_INIT_FUNCTION (vnet_feature_init);

这里同样可以看出来feature初始化的时候走的也是INIT_FUNCTION的流程。

数据流程的初始化：如dpdk的初始化