玩转xenstore

1 xenstore简介

xenstore作为xen中的一种重要特性存在，主要起到个存储的作用，作为不同进程间，Dom0和DomU间的通信介质。它有个守护进程xenstored（ps -ef|grep xenstored可以查看），还有/var/lib/xenstored/tdb作为数据库用来存储xenstore键值。简单地理解，xenstored进程就像个邮差，而tdb就像个邮局，各个用户间的通信就通过xenstored来负责，xenstored把信息送到邮局，再传给需要的用户。

2 xenstore命令

在命令行可以直接使用xenstore命令，如：xenstore-ls –f。xenstore命令其实本质是/usr/bin下的由同一源文件生成的多个可执行文件。（l /usr/bin/xenstore-*）其源文件为xen-4.1.2\tools\xenstore\xenstore_client.c。
-rwxr-xr-x 9 root root 22757 Sep  4 01:34 /usr/bin/xenstore-chmod*
-rwxr-xr-x 1 root root 12036 Sep  4 01:34 /usr/bin/xenstore-control*
-rwxr-xr-x 9 root root 22757 Sep  4 01:34 /usr/bin/xenstore-exists*
-rwxr-xr-x 9 root root 22757 Sep  4 01:34 /usr/bin/xenstore-list*
-rwxr-xr-x 9 root root 22757 Sep  4 01:34 /usr/bin/xenstore-ls*
-rwxr-xr-x 9 root root 22757 Sep  4 01:34 /usr/bin/xenstore-read*
-rwxr-xr-x 9 root root 22757 Sep  4 01:34 /usr/bin/xenstore-rm*
-rwxr-xr-x 9 root root 22757 Sep  4 01:34 /usr/bin/xenstore-watch*
-rwxr-xr-x 9 root root 22757 Sep  4 01:34 /usr/bin/xenstore-write*

这个xenstore的命令有些个性的地方是使用同一份源码根据命令名字不同来执行不同的函数调用动作，完成不同的xenstore动作。
int
main(int argc, char **argv)
{
    const char *_command = strrchr(argv[0], '/');
    const char *command = _command ? &_command[1] : argv[0];
    int switch_argv = -1; /* which element of argv did we switch on */

    if (strncmp(command, "xenstore-", strlen("xenstore-")) == 0)
    {
 switch_argv = 0;
 command = command + strlen("xenstore-");
    }
    else if (argc < 2)
 usage(MODE_unknown, 0, argv[0]);
    else
    {
 command = argv[1];
 switch_argv = 1;
    }

    mode = lookup_mode(command);

注意：这里面用了事件机制即xs_transaction_xxx，即使是一个单一的xenstore动作也用了这个悲催的机制，该机制排斥并发，可能不断重试，耗费cpu资源严重，very bad！！！（后面单节详细讨论）
again:
    if (transaction) {
 xth = xs_transaction_start(xsh);
 if (xth == XBT_NULL)
     errx(1, "couldn't start transaction");
    }
   //根据mode解析的不同执行不同的xenstore动作，有可能是读、写、删除等……
    ret = perform(mode, optind, argc - switch_argv, argv + switch_argv, xsh, xth, prefix, tidy, upto, recurse, nr_watches);

    if (transaction && !xs_transaction_end(xsh, xth, ret)) {

3 xenstore通信

xenstore通信的本质就是socket通信，以xenstore-read为例说明：xenstore_client.c中（顾名思义，xenstore的客户端，用来对外的，接受命令后，和xenstored进行socket通信）首先在main中打开socket连接。
int
main(int argc, char **argv)
{
   xsh = xs_open(socket ? XS_OPEN_SOCKETONLY : 0);

static int
perform(enum mode mode, int optind, int argc, char **argv, struct xs_handle *xsh,
        xs_transaction_t xth, int prefix, int tidy, int upto, int recurse, int nr_watches)
{
    while (optind < argc) {
        switch (mode) {
        case MODE_unknown:
            /* CANNOT BE REACHED */
            errx(1, "invalid mode %d", mode);
        case MODE_read: {
            static struct expanding_buffer ebuf;
            unsigned len;
            char *val = xs_read(xsh, xth, argv[optind], &len);

tools\xenstore\xs.c
void *xs_read(struct xs_handle *h, xs_transaction_t t,
       const char *path, unsigned int *len)
{
 return xs_single(h, t, XS_READ, path, len);
}

static void *xs_single(struct xs_handle *h, xs_transaction_t t,
         enum xsd_sockmsg_type type,
         const char *string,
         unsigned int *len)
{
 struct iovec iovec;

 iovec.iov_base = (void *)string;
 iovec.iov_len = strlen(string) + 1;
 return xs_talkv(h, t, type, &iovec, 1, len);
}

static void *xs_talkv(struct xs_handle *h, xs_transaction_t t,
        enum xsd_sockmsg_type type,
        const struct iovec *iovec,
        unsigned int num_vecs,
        unsigned int *len)
{
    //向socket中写入请求
 if (!xs_write_all(h->fd, &msg, sizeof(msg)))
  goto fail;
    //从socket中读取结果
 ret = read_reply(h, &msg.type, len);

xenstored_core.c即xenstored中循环读取处理socket请求。

4 xenstore事件机制

在xen代码中可以看到以xs_transaction_start开头，xs_transaction_end结尾的代码段，形如：
retry_transaction:
t = xs_transaction_start(ctx->xsh);
……
    if (!xs_transaction_end(ctx->xsh, t, 0)) {
        if (errno == EAGAIN) {
            t = 0;
            /*add by c00209220 on 2012-04-20*/
            LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "need to retry xenstore transaction for domain %d", *domid);
            /*add by c00209220 end*/
            goto retry_transaction;
        }

上面的就是一个事件机制的实例。那其本质是什么？xenstore事件机制会在xs_transaction_start时建立一个临时数据库，
xenstored_transaction.c中
void do_transaction_start(struct connection *conn, struct buffered_data *in)
{
    //记录generation值
 trans->generation = generation;
    //在xenstore数据库同一目录下创建名为tdb.transID的备份数据库
 trans->tdb_name = talloc_asprintf(trans, "%s.%p",
       xs_daemon_tdb(), trans);
    //将源数据库的内容拷贝到备份的数据库里
 trans->tdb = tdb_copy(tdb_context(conn), trans->tdb_name);

在xs_transaction_start和xs_transaction_end代码中间进行对备份数据库的修改动作。
注意：一定要将tran_ID 即t值作为参数传入xs操作，若t对应参数为NULL，则是对源数据库进行操作。
    t = xs_transaction_start(ctx->xsh);

    xs_rm(ctx->xsh, t, dom_path);
    xs_mkdir(ctx->xsh, t, dom_path);

 

在xs_transaction_end中进行检查，看从start到end整个流程里，源数据库是否被其它人修改过，若修改过则返回错误EAGAIN，上层检测到该错误NO.，会重复进行retry。
void do_transaction_end(struct connection *conn, const char *arg)
{
 struct changed_node *i;
 struct changed_domain *d;
 struct transaction *trans;

 if (!arg || (!streq(arg, "T") && !streq(arg, "F"))) {
  send_error(conn, EINVAL);
  return;
 }

 if ((trans = conn->transaction) == NULL) {
  send_error(conn, ENOENT);
  return;
 }

 conn->transaction = NULL;
 list_del(&trans->list);
 conn->transaction_started--;

 /* Attach transaction to arg for auto-cleanup */
 talloc_steal(arg, trans);

 if (streq(arg, "T")) {
  /* FIXME: Merge, rather failing on any change. */
       //在start时会在trans->generation中记录generation值，与当前generation比较，如果不相等，说明源数据库被修改过，transaction动作要重做。
  if (trans->generation != generation) {
   send_error(conn, EAGAIN);
   return;
  }

注：generation是一个全局变量，xs_write，xs_rm，transaction成功完成都会增加generation值，换言之都会打断transaction机制，使其重试。
void add_change_node(struct transaction *trans, const char *node, bool recurse)
{
 struct changed_node *i;

 if (!trans) {
  /* They're changing the global database. */
  generation++;
  return;
 }

void do_transaction_end(struct connection *conn, const char *arg)
{
 if (streq(arg, "T")) {
  /* FIXME: Merge, rather failing on any change. */
  if (trans->generation != generation) {
   send_error(conn, EAGAIN);
   return;
  }
  if (!replace_tdb(trans->tdb_name, trans->tdb)) {
   send_error(conn, errno);
   return;
  }
  /* Don't close this: we won! */
  trans->tdb = NULL;

  /* fix domain entry for each changed domain */
  list_for_each_entry(d, &trans->changed_domains, list)
   domain_entry_fix(d->domid, d->nbentry);

  /* Fire off the watches for everything that changed. */
  list_for_each_entry(i, &trans->changes, list)
   fire_watches(conn, i->node, i->recurse);
  generation++;
 }

 

2013年5月24日上传