Cloud foundry warden 源码学习(4)

wshd是C写的一个程序，侦听在unix socket端口，wsh是其客户端。

int parent_run(wshd_t *w) {
  char path[MAXPATHLEN];
  int rv;
  pid_t pid;

  memset(path, 0, sizeof(path));

  strcpy(path + strlen(path), w->run_path);
  strcpy(path + strlen(path), "/");
  strcpy(path + strlen(path), "wshd.sock");

  w->fd = un_listen(path);

  rv = barrier_open(&w->barrier_parent);
  assert(rv == 0);

  rv = barrier_open(&w->barrier_child);
  assert(rv == 0);

  /* Unshare mount namespace, so the before clone hook is free to mount
   * whatever it needs without polluting the global mount namespace. */
  rv = unshare(CLONE_NEWNS);
  assert(rv == 0);

  rv = run(w->lib_path, "hook-parent-before-clone.sh");
  assert(rv == 0);

  pid = child_start(w);
  assert(pid > 0);

  parent_setenv_pid(w, pid);

  rv = run(w->lib_path, "hook-parent-after-clone.sh");
  assert(rv == 0);

  rv = barrier_signal(&w->barrier_parent);
  if (rv == -1) {
    fprintf(stderr, "Error waking up child process\n");
    exit(1);
  }

  rv = barrier_wait(&w->barrier_child);
  if (rv == -1) {
    fprintf(stderr, "Error waiting for acknowledgement from child process\n");
    exit(1);
  }

  return 0;
}

int main(int argc, char **argv) {
  wshd_t *w;
  int rv;

  /* Continue child execution in the context of the container */
  if (argc > 1 && strcmp(argv[1], "--continue") == 0) {
    return child_continue(argc, argv);
  }

  w = calloc(1, sizeof(*w));
  assert(w != NULL);

  rv = wshd__getopt(w, argc, argv);
  if (rv == -1) {
    exit(1);
  }

  if (strlen(w->run_path) == 0) {
    strcpy(w->run_path, "run");
  }

  if (strlen(w->lib_path) == 0) {
    strcpy(w->lib_path, "lib");
  }

  if (strlen(w->root_path) == 0) {
    strcpy(w->root_path, "root");
  }

  assert_directory(w->run_path);
  assert_directory(w->lib_path);
  assert_directory(w->root_path);

  parent_run(w);

  return 0;
}

Wshd进程首先启动在unix socket上的侦听，接下来调用hook script完成 warden虚拟机的创建.

Skeleton/lib/hook-parent-before-clone.sh

mount -n -t aufs -o br:tmp/rootfs=rw:$rootfs_path=ro+wh none<pre class="plain" name="code">cp bin/wshd mnt/sbin/wshd
chmod 700 mnt/sbin/wshd

Skeleton/lib/hook-parent-after-clone.sh

# Add new group for every subsystem
for system_path in /tmp/warden/cgroup/*
do
  instance_path=$system_path/instance-$id

  mkdir -p $instance_path

  if [ $(basename $system_path) == "cpuset" ]
  then
    cat $system_path/cpuset.cpus > $instance_path/cpuset.cpus
    cat $system_path/cpuset.mems > $instance_path/cpuset.mems
  fi
done

ip link add name $network_host_iface type veth peer name $network_container_iface
ip link set $network_host_iface netns 1
ip link set $network_container_iface netns $PID

ifconfig $network_host_iface $network_host_ip netmask $network_netmask

注意以上只是初始化cgroup，并没有对cpu/mem/disk/network做资源限制，这些会由DEA继续发limit request来完成。

紧接着，DEA会发送几个limit request，对warden container的资源做进一步的配置，这其中包括do_limit_memory, do_limit_cpu, do_limit_quota, do_limit_bandwith.
内存限制: 通过往cgroup mem里面memory.limit_in_bytes和memory.memsw.limit_in_bytes写入值限制该warden container的内存；
CPU限制: 通过往cgroup cpu里面cpu.shares写值。
Disk 限制: 通过调用setquota –u UID block_sock=xxx block_hard=xxx inode_soft=xxx inode_hard=xxx, 而UID是warden小虚机的/home/vcap目录。这就是说配给warden小虚机的disk配额。

至此，warden container创建完成。

总结一下：

Warden::Container::Linux在接收到create请求以后，拷贝skeleton文件系统，开启wshd进程，将wshd进程加入control group(cpu/memory/disk)

Warden container linux文件系统构成：

1./var/vcap/data/warden/depot/17r47v9oif1/tmp/rootfs 
(instance data以aufs方式readwrite挂载 home, etc,dev,sbin,var)
---->17vedc80gj3/mnt
2./var/vcap/data/packages/rootfs_lucid64/2.1 (rootfs  以aufs方式readonly挂载)  
---->17vedc80gj3/mnt
3.var/vcap/packages/buildpack_cache ---->/var/vcap/data/warden/depot/17vedc80gj3/mnt/var/vcap/packages/buildpack_cache (readonly)

/var/vcap/data/dea_next/droplets/850c48ada15591d8c6a56a0e1c64cb2846ea00dd---->
/var/vcap/data/warden/depot/17vedc80gj3/mnt/var/vcap/data/dea_next/droplets/850c48ada15591d8c6a56a0e1c64cb2846ea00dd (readonly)