1）复现流程及lxc的处理
demo1程序与执行结果如下，此时在容器内部看不到执行的程序。
int main()
{
int ret, fd, pid;
printf("father pid old:%d\n", getpid());
fd = open("/dev/ns", O_RDWR);
ret = ioctl(fd, 24635); // parm is dst ns process's pid
printf("father pid old:%d\n", getpid());
sleep(5);
return 0;
}
# ./a.out
father pid old:26169
father pid old:26169
demo2程序与执行结果如下，此时容器内还是看不到执行程序，但是这里getpid()获取到的值就为0了，对比上边，世界上没有这么玄乎的事，或许是缓存的问题？这是第一个疑问。
int main()
{
int ret, fd, pid;
fd = open("/dev/ns", O_RDWR);
ret = ioctl(fd, 24635); // parm is dst ns process's pid
printf("father pid old:%d\n", getpid());
sleep(5);
return 0;
}
# ./a.out
father pid old:0
demo3程序与执行结果如下，这时在容器内部能看到子进程，为什么子进程实现了pid ns的切换，父进程却没有实现？这是第二个疑问。
int main()
{
int ret, fd, pid;
fd = open("/dev/ns", O_RDWR);
ret = ioctl(fd, 24635); // parm is dst ns process's pid
printf("father pid old:%d\n", getpid());
pid = fork();
if(0 == pid) {
printf("son pid:%d\n", getpid());
sleep(5);
} else {
printf("father pid:%d\n", getpid());
wait();
}
return 0;
}
# ./a.out
father pid old:0
father pid:0
son pid:87
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 root 20 0 22036 2144 1624 S 0.0 0.1 0:00.01 bash
57 root 20 0 23672 1516 1160 R 0.0 0.1 0:00.66 top
87 root 20 0 3760 204 120 S 0.0 0.0 0:00.00 a.out
lxc attach流程如下，实现思路和我们上边的测试demo一致，看来这确实是linux机制的问题。
setns()
pid = fork();
if(!pid) { // son
execve(bash);
}
2）pid缓存问题
编写了demo4：
int main()
{
int ret, fd, pid;
printf("father pid:%d\n", getpid());
printf("father pid:%d\n", getpid());
printf("father pid:%d\n", getpid());
printf("father pid:%d\n", getpid());
fd = open("/dev/ns", O_RDWR);
ret = ioctl(fd, 24635); // parm is dst ns process's pid
printf("father pid old:%d\n", getpid());
pid = fork();
if(0 == pid) {
printf("son pid:%d\n", getpid());
sleep(5);
} else {
printf("father pid:%d\n", getpid());
wait();
}
return 0;
}
看下结果，fork前后打印一致
# ./a.out
father pid:23246
father pid:23246
father pid:23246
father pid:23246
father pid old:23246
father pid:23246
son pid:96
strace看下，果然和我们猜的一样，除了第一次getpid调用了syscall后，后边所有的返回值均是从缓存中获取的，所以这就能解释为什么不执行getpid并fork后执行getpid获取到的返回值是0，而执行了getpid并fork后再执行getpid获取到的返回值不变。
# strace ./a.out
execve("./a.out", ["./a.out"], [/* 29 vars */]) = 0
brk(0) = 0x18f4000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d79c000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=105026, ...}) = 0
mmap(NULL, 105026, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fa04d782000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\356\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1478056, ...}) = 0
mmap(NULL, 3586120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa04d215000
mprotect(0x7fa04d377000, 2097152, PROT_NONE) = 0
mmap(0x7fa04d577000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x162000) = 0x7fa04d577000
mmap(0x7fa04d57c000, 18504, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa04d57c000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d781000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d780000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d77f000
arch_prctl(ARCH_SET_FS, 0x7fa04d780700) = 0
mprotect(0x7fa04d577000, 16384, PROT_READ) = 0
mprotect(0x7fa04d79e000, 4096, PROT_READ) = 0
munmap(0x7fa04d782000, 105026) = 0
getpid() = 23253
fstat(1, {st_mode=S_IFCHR|0600, st_rdev=makedev(136, 4), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d79b000
write(1, "father pid:23253\n", 17father pid:23253
) = 17
write(1, "father pid:23253\n", 17father pid:23253
) = 17
write(1, "father pid:23253\n", 17father pid:23253
) = 17
write(1, "father pid:23253\n", 17father pid:23253
) = 17
open("/dev/ns", O_RDWR) = 3
ioctl(3, 0x603b, 0x7fa04d57cdf0) = 0
write(1, "father pid old:23253\n", 21father pid old:23253
) = 21
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7fa04d7809d0) = 97
write(1, "father pid:23253\n", 17father pid:23253
) = 17
wait4(-1, son pid:97
0xffffffff, 0, NULL) = -1 EFAULT (Bad address)
--- SIGCHLD (Child exited) @ 0 (0) ---
3）为什么子进程实现了pid ns的切换，父进程却没有实现？
先来看看setns的实现，在2.6.32中要支持setns很简单首先通过不创建新NS的方式调用copy_namespaces并传入dst ns，这会增加dst ns的引用，之后通过switch_task_namespaces传入current task与dst ns，在函数中会首先进行nsproxy指针的交换，将当前task切换到dst ns中，之后src ns减引用，这样就能保证引用数量的正确。因此这里实际上切换了current task的nsproxy。
copy_namespaces->switch_task_namespaces
int copy_namespaces(unsigned long flags, struct task_struct *tsk)
{
struct nsproxy *old_ns = tsk->nsproxy;
struct nsproxy *new_ns;
int err = 0;
if (!old_ns)
return 0;
get_nsproxy(old_ns);
if (!(flags & (CLONE_NEWNS | CLONE_NEWUTS | CLONE_NEWIPC |
CLONE_NEWPID | CLONE_NEWNET)))
return 0;
。。。
}
void switch_task_namespaces(struct task_struct *p, struct nsproxy *new)
{
struct nsproxy *ns;
might_sleep();
ns = p->nsproxy;
rcu_assign_pointer(p->nsproxy, new);
if (ns && atomic_dec_and_test(&ns->count)) {
/*
* wait for others to get what they want from this nsproxy.
*
* cannot release this nsproxy via the call_rcu() since
* put_mnt_ns() will want to sleep
*/
synchronize_rcu();
free_nsproxy(ns);
}
}
再看看getpid的调用流程(sys_getpid–>task_tgid_vnr->pid_vnr->pid_nr_ns)，这里考虑线程问题通过task_tgid获取该PID的tgid，因为应用层和内核对PID的定义不同，内核中进程与线程都拥有相同的结构体描述struct task_struct，因此进程与线程在内核中均拥有自己独立的PID，因此这时候找到“用户态PID”的关键是找到根进程，因为父根进程的pid与tgid是一致如下，因此在这里传入tgid来代替父根进程的PID。在pid_nr_ns中会进行两个判断，一是该pid ns的level应大于指定pid ns的level，这里的pid ns level如下图所示是一个树状结构，这里default pid ns中的pid level为0，而基于某进程创建的CLONE_NEWPID的进程，其pid ns level + 1，因此这里的判断会出现问题，因为setns只修改了current->nsproxy，如果是在default pid ns中执行的demo程序以及lxc程序，那么current->nsproxy->pid_ns->level = 1，而pid->level = 0，这会导致判断失败直接返回0，这也是我们在demo程序中通过getpid()得到0的原因。
<-- PID 43 --> <----------------- PID 42 ----------------->
+---------+
| process |
_| pid=42 |_
_/ | tgid=42 | \_ (new thread) _
_ (fork) _/ +---------+ \
/ +---------+
+---------+ | process |
| process | | pid=44 |
| pid=43 | | tgid=42 |
| tgid=43 | +---------+
+---------+
<-- PID 43 --> <--------- PID 42 --------> <--- PID 44 --->
SYSCALL_DEFINE0(getpid)
{
return task_tgid_vnr(current);
}
static inline pid_t task_tgid_vnr(struct task_struct *tsk)
{
return pid_vnr(task_tgid(tsk));
}
pid_t pid_vnr(struct pid *pid)
{
return pid_nr_ns(pid, current->nsproxy->pid_ns);
}
pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns)
{
struct upid *upid;
pid_t nr = 0;
if (pid && ns->level <= pid->level) {
upid = &pid->numbers[ns->level];
if (upid->ns == ns)
nr = upid->nr;
}
return nr;
}
如下，这里有对获取到的upid->nr的一个很好的解释，也就是说upid是pid ns有效的，另外这个实验是在2.6.32.5中实现的，而2.6.32.5中不支持setns(我对它进行了移植)，或许在3.x的某些支持setns的版本中也能复现，但4.9.0中pid_nr_ns的不同实现导致了实验效果的不同，在4.9.0中demo程序会返回default ns中的pid。
/*
* struct upid is used to get the id of the struct pid, as it is
* seen in particular namespace. Later the struct pid is found with
* find_pid_ns() using the int nr and struct pid_namespace *ns.
*/
struct upid {
/* Try to keep pid_chain in the same cacheline as nr for find_vpid */
int nr;
struct pid_namespace *ns;
struct hlist_node pid_chain;
};
最后再补充下，前面可以看到setns实现其实只修改了task_struct中的nsproxy，因此fork会通过调用链do_fork->copy_process->alloc_pid(p->nsproxy->pid_ns)也就是利用父进程的nsproxy->pid_ns来创建自己的pid，这样就说明了为什么lxc中必须fork出子进程来执行execve。
扫一扫
722
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
