libvirt和SanLock中的killpath源码分析

最新推荐文章于 2024-09-11 11:53:23 发布

weixin_34292402

最新推荐文章于 2024-09-11 11:53:23 发布

阅读量265

点赞数

文章标签：运维数据结构与算法 python

原文链接：https://my.oschina.net/LastRitter/blog/1539684

版权

2019独角兽企业重金招聘Python工程师标准>>>

sanlock守护进程初始化

守护进程主函数执行初始化。

src/main.c

int main(int argc, char *argv[])
{
...
	switch (com.type) {
	case COM_DAEMON:
		rv = do_daemon();
		break;

	case COM_CLIENT:
		rv = do_client();
		break;

	case COM_DIRECT:
		rv = do_direct();
		break;
	};
...
}

static int do_daemon(void)
{
	int fd, rv;


	/* This can take a while so do it before forking. */
	setup_groups();

	if (!com.debug) {
		/* TODO: copy comprehensive daemonization method from libvirtd */
		if (daemon(0, 0) < 0) {
			log_tool("cannot fork daemon\n");
			exit(EXIT_FAILURE);
		}
	}

	setup_limits();
	setup_helper();

	/* main task never does disk io, so we don't really need to set
	 * it up, but other tasks get their use_aio value by copying
	 * the main_task settings */

	sprintf(main_task.name, "%s", "main");
	setup_task_aio(&main_task, com.aio_arg, 0);

	rv = client_alloc();
	if (rv < 0)
		return rv;

	helper_ci = client_add(helper_status_fd, process_helper, helper_dead);
	if (helper_ci < 0)
		return rv;
	strcpy(client[helper_ci].owner_name, "helper");

	setup_signals();
	setup_logging();

	fd = lockfile(SANLK_RUN_DIR, SANLK_LOCKFILE_NAME, com.uid, com.gid);
	if (fd < 0) {
		close_logging();
		return fd;
	}

	setup_host_name();

	setup_uid_gid();

	log_level(0, 0, NULL, LOG_WARNING, "sanlock daemon started %s host %s",
		  VERSION, our_host_name_global);

	setup_priority();

	rv = thread_pool_create(DEFAULT_MIN_WORKER_THREADS, com.max_worker_threads);
	if (rv < 0)
		goto out;

	rv = setup_listener();
	if (rv < 0)
		goto out_threads;

	setup_token_manager();
	if (rv < 0)
		goto out_threads;

	/* initialize global eventfd for client_resume notification */
	if ((efd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK)) == -1) {
		log_error("couldn't create eventfd");
		goto out_threads;
	}

	main_loop();

	close_token_manager();

 out_threads:
	thread_pool_free();
 out:
	/* order reversed from setup so lockfile is last */
	close_logging();
	unlink_lockfile(fd, SANLK_RUN_DIR, SANLK_LOCKFILE_NAME);
	return rv;
}

创建helper子进程负责执行kill信号或者执行killpath指定的程序。

src/main.c

/*
 * first pipe for daemon to send requests to helper; they are not acknowledged
 * and the daemon does not get any result back for the requests.
 *
 * second pipe for helper to send general status/heartbeat back to the daemon
 * every so often to confirm it's not dead/hung.  If the helper gets stuck or
 * killed, the daemon will not get the status and won't bother sending requests
 * to the helper, and use SIGTERM instead
 */

static int setup_helper(void)
{
	int pid;
	int pw_fd = -1; /* parent write */
	int cr_fd = -1; /* child read */
	int pr_fd = -1; /* parent read */
	int cw_fd = -1; /* child write */
	int pfd[2];

	/* we can't allow the main daemon thread to block */
	if (pipe2(pfd, O_NONBLOCK | O_CLOEXEC))
		return -errno;

	/* uncomment for rhel7 where this should be available */
	/* fcntl(pfd[1], F_SETPIPE_SZ, 1024*1024); */

	cr_fd = pfd[0];
	pw_fd = pfd[1];

	if (pipe2(pfd, O_NONBLOCK | O_CLOEXEC)) {
		close(cr_fd);
		close(pw_fd);
		return -errno;
	}

	pr_fd = pfd[0];
	cw_fd = pfd[1];

	pid = fork();
	if (pid < 0) {
		close(cr_fd);
		close(pw_fd);
		close(pr_fd);
		close(cw_fd);
		return -errno;
	}

	if (pid) {
		close(cr_fd);
		close(cw_fd);
		helper_kill_fd = pw_fd;
		helper_status_fd = pr_fd;
		helper_pid = pid;
		return 0;
	} else {
		close(pr_fd);
		close(pw_fd);
		run_helper(cr_fd, cw_fd, (log_stderr_priority == LOG_DEBUG));
		exit(0);
	}
}

src/helper.c

通过和主进程之间的管道读取命令并执行：

int run_helper(int in_fd, int out_fd, int log_stderr)
{
	char name[16];
	struct pollfd pollfd;
	struct helper_msg hm;
	unsigned int fork_count = 0;
	unsigned int wait_count = 0;
	time_t now, last_send, last_good = 0;
	int timeout = STANDARD_TIMEOUT_MS;
	int rv, pid, status;

	memset(name, 0, sizeof(name));
	sprintf(name, "%s", "sanlock-helper");
	prctl(PR_SET_NAME, (unsigned long)name, 0, 0, 0);

	rv = setgroups(0, NULL);
	if (rv < 0)
		log_debug("error clearing helper groups errno %i", errno);

	memset(&pollfd, 0, sizeof(pollfd));
	pollfd.fd = in_fd;
	pollfd.events = POLLIN;

	now = monotime();
	last_send = now;
	rv = send_status(out_fd);
	if (!rv)
		last_good = now;

	while (1) {
		rv = poll(&pollfd, 1, timeout);
		if (rv == -1 && errno == EINTR)
			continue;

		if (rv < 0)
			exit(0);

		now = monotime();

		if (now - last_good >= HELPER_STATUS_INTERVAL &&
		    now - last_send >= 2) {
			last_send = now;
			rv = send_status(out_fd);
			if (!rv)
				last_good = now;
		}

		memset(&hm, 0, sizeof(hm));

		if (pollfd.revents & POLLIN) {
			rv = read_hm(in_fd, &hm);
			if (rv)
				continue;

			if (hm.type == HELPER_MSG_RUNPATH) {
				pid = fork();
				if (!pid) {
					run_path(&hm);
					exit(-1);
				}

				fork_count++;

				/*
				log_debug("helper fork %d count %d %d %s %s",
					  pid, fork_count, wait_count,
					  hm.path, hm.args);
				*/
			} else if (hm.type == HELPER_MSG_KILLPID) {
				kill(hm.pid, hm.sig);
			}
		}

		if (pollfd.revents & (POLLERR | POLLHUP | POLLNVAL))
			exit(0);

		/* collect child exits until no more children exist (ECHILD)
		   or none are ready (WNOHANG) */

		while (1) {
			rv = waitpid(-1, &status, WNOHANG);
			if (rv > 0) {
				wait_count++;

				/*
				log_debug("helper wait %d count %d %d",
					  rv, fork_count, wait_count);
				*/
				continue;
			}

			/* no more children to wait for or no children
			   have exited */

			if (rv < 0 && errno == ECHILD) {
				if (timeout == RECOVERY_TIMEOUT_MS) {
					log_debug("helper no children count %d %d",
						  fork_count, wait_count);
				}
				timeout = STANDARD_TIMEOUT_MS;
			} else {
				timeout = RECOVERY_TIMEOUT_MS;
			}
			break;
		}
	}

	return 0;
}

static void run_path(struct helper_msg *hm)
{
	char arg[SANLK_HELPER_ARGS_LEN];
	char *args = hm->args;
	char *av[MAX_AV_COUNT + 1]; /* +1 for NULL */
	int av_count = 0;
	int i, arg_len, args_len;

	for (i = 0; i < MAX_AV_COUNT + 1; i++)
		av[i] = NULL;

	av[av_count++] = strdup(hm->path);

	if (!args[0])
		goto pid_arg;

	/* this should already be done, but make sure */
	args[SANLK_HELPER_ARGS_LEN - 1] = '\0';

	memset(&arg, 0, sizeof(arg));
	arg_len = 0;
	args_len = strlen(args);

	for (i = 0; i < args_len; i++) {
		if (!args[i])
			break;

		if (av_count == MAX_AV_COUNT)
			break;

		if (args[i] == '\\') {
			if (i == (args_len - 1))
				break;
			i++;

			if (args[i] == '\\') {
				arg[arg_len++] = args[i];
				continue;
			}
			if (isspace(args[i])) {
				arg[arg_len++] = args[i];
				continue;
			} else {
				break;
			}
		}

		if (isalnum(args[i]) || ispunct(args[i])) {
			arg[arg_len++] = args[i];
		} else if (isspace(args[i])) {
			if (arg_len)
				av[av_count++] = strdup(arg);

			memset(arg, 0, sizeof(arg));
			arg_len = 0;
		} else {
			break;
		}
	}

	if ((av_count < MAX_AV_COUNT) && arg_len) {
		av[av_count++] = strdup(arg);
	}

 pid_arg:
	if ((av_count < MAX_AV_COUNT) && hm->pid) {
		memset(arg, 0, sizeof(arg));
		snprintf(arg, sizeof(arg)-1, "%d", hm->pid);
		av[av_count++] = strdup(arg);
	}

	execvp(av[0], av);
}

初始化并分配客户端管理数据结构。

src/main.c

/* FIXME: add a mutex for client array so we don't try to expand it
   while a cmd thread is using it.  Or, with a thread pool we know
   when cmd threads are running and can expand when none are. */

static int client_alloc(void)
{
	int i;

	/* pollfd is one element longer as we use an additional element for the
	 * eventfd notification mechanism */
	client = malloc(CLIENT_NALLOC * sizeof(struct client));
	pollfd = malloc((CLIENT_NALLOC+1) * sizeof(struct pollfd));

	if (!client || !pollfd) {
		log_error("can't alloc for client or pollfd array");
		return -ENOMEM;
	}

	for (i = 0; i < CLIENT_NALLOC; i++) {
		memset(&client[i], 0, sizeof(struct client));
		memset(&pollfd[i], 0, sizeof(struct pollfd));

		pthread_mutex_init(&client[i].mutex, NULL);
		client[i].fd = -1;
		client[i].pid = -1;

		pollfd[i].fd = -1;
		pollfd[i].events = 0;
	}
	client_size = CLIENT_NALLOC;
	return 0;
}

设置本地套接字监听端口及处理函数。

src/main.c

初始化套接字，并绑定端口，设置处理函数：

static int setup_listener(void)
{
	struct sockaddr_un addr;
	int rv, fd, ci;

	rv = sanlock_socket_address(&addr);
	if (rv < 0)
		return rv;

	fd = socket(AF_LOCAL, SOCK_STREAM, 0);
	if (fd < 0)
		return fd;

	unlink(addr.sun_path);
	rv = bind(fd, (struct sockaddr *) &addr, sizeof(struct sockaddr_un));
	if (rv < 0)
		goto exit_fail;

	rv = chmod(addr.sun_path, DEFAULT_SOCKET_MODE);
	if (rv < 0)
		goto exit_fail;

	rv = chown(addr.sun_path, com.uid, com.gid);
	if (rv < 0) {
		log_error("could not set socket %s permissions: %s",
			addr.sun_path, strerror(errno));
		goto exit_fail;
	}

	rv = listen(fd, 5);
	if (rv < 0)
		goto exit_fail;

	fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);

	ci = client_add(fd, process_listener, NULL);
	if (ci < 0)
		goto exit_fail;

	strcpy(client[ci].owner_name, "listener");
	return 0;

 exit_fail:
	close(fd);
	return -1;
}

src/sanlock_sock.c

初始化监听地址：

int sanlock_socket_address(struct sockaddr_un *addr)
{
	memset(addr, 0, sizeof(struct sockaddr_un));
	addr->sun_family = AF_LOCAL;
	snprintf(addr->sun_path, sizeof(addr->sun_path) - 1, "%s/%s",
		 SANLK_RUN_DIR, SANLK_SOCKET_NAME);
	return 0;
}

src/sanlock_sock.h

默认的本地监听套接字：

#define SANLK_RUN_DIR "/var/run/sanlock"
#define SANLK_SOCKET_NAME "sanlock.sock"

src/main.c

服务端处理函数，对于监听套接字，每接收一个新的连接则分配一个client结构与之对应，在SM_CMD_KILLPATH命令中处理设置killpath的功能：

static void process_listener(int ci GNUC_UNUSED)
{
	int fd;
	int on = 1;

	fd = accept(client[ci].fd, NULL, NULL);
	if (fd < 0)
		return;

	setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &on, sizeof(on));

	client_add(fd, process_connection, NULL);
}

static void process_connection(int ci)
{
	struct sm_header h;
	void (*deadfn)(int ci);
	int rv;

	memset(&h, 0, sizeof(h));

	rv = recv(client[ci].fd, &h, sizeof(h), MSG_WAITALL);
	if (!rv)
		return;
	if (rv < 0) {
		log_error("ci %d fd %d pid %d recv errno %d",
			  ci, client[ci].fd, client[ci].pid, errno);
		goto dead;
	}
	if (rv != sizeof(h)) {
		log_error("ci %d fd %d pid %d recv size %d",
			  ci, client[ci].fd, client[ci].pid, rv);
		goto dead;
	}
	if (h.magic != SM_MAGIC) {
		log_error("ci %d recv %d magic %x vs %x",
			  ci, rv, h.magic, SM_MAGIC);
		goto dead;
	}
	if (client[ci].restricted & SANLK_RESTRICT_ALL) {
		log_error("ci %d fd %d pid %d cmd %d restrict all",
			  ci, client[ci].fd, client[ci].pid, h.cmd);
		goto dead;
	}
	if (h.version && (h.cmd != SM_CMD_VERSION) &&
	    (h.version & 0xFFFF0000) > (SM_PROTO & 0xFFFF0000)) {
		log_error("ci %d recv %d proto %x vs %x",
			  ci, rv, h.version , SM_PROTO);
		goto dead;
	}

	client[ci].cmd_last = h.cmd;

	switch (h.cmd) {
	case SM_CMD_REGISTER:
	case SM_CMD_RESTRICT:
	case SM_CMD_VERSION:
	case SM_CMD_SHUTDOWN:
	case SM_CMD_STATUS:
	case SM_CMD_HOST_STATUS:
	case SM_CMD_RENEWAL:
	case SM_CMD_LOG_DUMP:
	case SM_CMD_GET_LOCKSPACES:
	case SM_CMD_GET_HOSTS:
	case SM_CMD_REG_EVENT:
	case SM_CMD_END_EVENT:
	case SM_CMD_SET_CONFIG:
		call_cmd_daemon(ci, &h, client_maxi);
		break;
	case SM_CMD_ADD_LOCKSPACE:
	case SM_CMD_INQ_LOCKSPACE:
	case SM_CMD_REM_LOCKSPACE:
	case SM_CMD_REQUEST:
	case SM_CMD_EXAMINE_RESOURCE:
	case SM_CMD_EXAMINE_LOCKSPACE:
	case SM_CMD_ALIGN:
	case SM_CMD_WRITE_LOCKSPACE:
	case SM_CMD_WRITE_RESOURCE:
	case SM_CMD_READ_LOCKSPACE:
	case SM_CMD_READ_RESOURCE:
	case SM_CMD_READ_RESOURCE_OWNERS:
	case SM_CMD_SET_LVB:
	case SM_CMD_GET_LVB:
	case SM_CMD_SHUTDOWN_WAIT:
	case SM_CMD_SET_EVENT:
		rv = client_suspend(ci);
		if (rv < 0)
			return;
		process_cmd_thread_unregistered(ci, &h);
		break;
	case SM_CMD_ACQUIRE:
	case SM_CMD_RELEASE:
	case SM_CMD_INQUIRE:
	case SM_CMD_CONVERT:
	case SM_CMD_KILLPATH:
		/* the main_loop needs to ignore this connection
		   while the thread is working on it */
		rv = client_suspend(ci);
		if (rv < 0)
			return;
		process_cmd_thread_registered(ci, &h);
		break;
	default:
		log_error("ci %d cmd %d unknown", ci, h.cmd);
	};

	return;

 dead:
	deadfn = client[ci].deadfn;
	if (deadfn)
		deadfn(ci);
}

最后进入主循环-main_loop函数中

守护进程主循环

src/main.c

循环poll每个套接字，如果有数据则执行相应的work函数，也就是之前设置好的process_connection函数。

static int main_loop(void)
{
	void (*workfn) (int ci);
	void (*deadfn) (int ci);
	struct space *sp, *safe;
	struct timeval now, last_check;
	int poll_timeout, check_interval;
	unsigned int ms;
	int i, rv, empty, check_all;
	char *check_buf = NULL;
	int check_buf_len = 0;
	uint64_t ebuf;

	gettimeofday(&last_check, NULL);
	poll_timeout = STANDARD_CHECK_INTERVAL;
	check_interval = STANDARD_CHECK_INTERVAL;

	while (1) {
		/* as well as the clients, check the eventfd */
		pollfd[client_maxi+1].fd = efd;
		pollfd[client_maxi+1].events = POLLIN;

		rv = poll(pollfd, client_maxi + 2, poll_timeout);
		if (rv == -1 && errno == EINTR)
			continue;
		if (rv < 0) {
			/* not sure */
		}
		for (i = 0; i <= client_maxi + 1; i++) {
			if (pollfd[i].fd == efd && pollfd[i].revents & POLLIN) {
				/* a client_resume completed */
				eventfd_read(efd, &ebuf);
				continue;
			}
			if (client[i].fd < 0)
				continue;
			if (pollfd[i].revents & POLLIN) {
				workfn = client[i].workfn;
				if (workfn)
					workfn(i);
			}
			if (pollfd[i].revents & (POLLERR | POLLHUP | POLLNVAL)) {
				deadfn = client[i].deadfn;
				if (deadfn)
					deadfn(i);
			}
		}


		gettimeofday(&now, NULL);
		ms = time_diff(&last_check, &now);
		if (ms < check_interval) {
			poll_timeout = check_interval - ms;
			continue;
		}
		last_check = now;
		check_interval = STANDARD_CHECK_INTERVAL;

		/*
		 * check the condition of each lockspace,
		 * if pids are being killed, have pids all exited?
		 * is its host_id being renewed?, if not kill pids
		 */

		pthread_mutex_lock(&spaces_mutex);
		list_for_each_entry_safe(sp, safe, &spaces, list) {

			if (sp->killing_pids && all_pids_dead(sp)) {
				/*
				 * move sp to spaces_rem so main_loop
				 * will no longer see it.
				 */
				log_space(sp, "set thread_stop");
				pthread_mutex_lock(&sp->mutex);
				sp->thread_stop = 1;
				deactivate_watchdog(sp);
				pthread_mutex_unlock(&sp->mutex);
				list_move(&sp->list, &spaces_rem);
				continue;
			}

			if (sp->killing_pids) {
				/*
				 * continue to kill the pids with increasing
				 * levels of severity until they all exit
				 */
				kill_pids(sp);
				check_interval = RECOVERY_CHECK_INTERVAL;
				continue;
			}

			/*
			 * check host_id lease renewal
			 */

			if (sp->align_size > check_buf_len) {
				if (check_buf)
					free(check_buf);
				check_buf_len = sp->align_size;
				check_buf = malloc(check_buf_len);
			}
			if (check_buf)
				memset(check_buf, 0, check_buf_len);

			check_all = 0;

			rv = check_our_lease(sp, &check_all, check_buf);
			if (rv)
				sp->renew_fail = 1;

			if (rv || sp->external_remove || (external_shutdown > 1)) {
				log_space(sp, "set killing_pids check %d remove %d",
					  rv, sp->external_remove);
				sp->space_dead = 1;
				sp->killing_pids = 1;
				kill_pids(sp);
				check_interval = RECOVERY_CHECK_INTERVAL;

			} else if (check_all) {
				check_other_leases(sp, check_buf);
			}
		}
		empty = list_empty(&spaces);
		pthread_mutex_unlock(&spaces_mutex);

		if (external_shutdown && empty)
			break;

		if (external_shutdown == 1) {
			log_debug("ignore shutdown, lockspace exists");
			external_shutdown = 0;
		}

		free_lockspaces(0);
		free_resources();

		gettimeofday(&now, NULL);
		ms = time_diff(&last_check, &now);
		if (ms < check_interval)
			poll_timeout = check_interval - ms;
		else
			poll_timeout = 1;
	}

	free_lockspaces(1);

	daemon_shutdown_reply();

	return 0;
}

同时定时检查每个Lockspace的租约是否正常更新。

src/lockspace.c

int check_our_lease(struct space *sp, int *check_all, char *check_buf)
{
	int id_renewal_fail_seconds, id_renewal_warn_seconds;
	uint64_t last_success;
	int corrupt_result;
	int gap;

	pthread_mutex_lock(&sp->mutex);
	last_success = sp->lease_status.renewal_last_success;
	corrupt_result = sp->lease_status.corrupt_result;

	if (sp->lease_status.renewal_read_count > sp->lease_status.renewal_read_check) {
		/*
		 * NB. it's unfortunate how subtle this is.
		 * main loop will pass this buf to check_other_leases next
		 */
		sp->lease_status.renewal_read_check = sp->lease_status.renewal_read_count;
		*check_all = 1;
		if (check_buf)
			memcpy(check_buf, sp->lease_status.renewal_read_buf, sp->align_size);
	}
	pthread_mutex_unlock(&sp->mutex);

	if (corrupt_result) {
		log_erros(sp, "check_our_lease corrupt %d", corrupt_result);
		return -1;
	}

	gap = monotime() - last_success;

	id_renewal_fail_seconds = calc_id_renewal_fail_seconds(sp->io_timeout);
	id_renewal_warn_seconds = calc_id_renewal_warn_seconds(sp->io_timeout);

	if (gap >= id_renewal_fail_seconds) {
		log_erros(sp, "check_our_lease failed %d", gap);
		return -1;
	}

	if (gap >= id_renewal_warn_seconds) {
		log_erros(sp, "check_our_lease warning %d last_success %llu",
			  gap, (unsigned long long)last_success);
	}

	if (com.debug_renew > 1) {
		log_space(sp, "check_our_lease good %d %llu",
		  	  gap, (unsigned long long)last_success);
	}

	return 0;
}

如果更新租约失败，并超过一段时间则调用kill_pids函数终止持有该Lockspace的租约的进程。

src/main.c

遍历这个Lockspace的所有client（每个client代表一个获取Resources的客户），向helper进程发送kill掉他的命令。

static void kill_pids(struct space *sp)
{
	struct client *cl;
	uint64_t now, last_success;
	int id_renewal_fail_seconds;
	int ci, sig;
	int do_kill, in_grace;

	/*
	 * all remaining pids using sp are stuck, we've made max attempts to
	 * kill all, don't bother cycling through them
	 */
	if (sp->killing_pids > 1)
		return;

	id_renewal_fail_seconds = calc_id_renewal_fail_seconds(sp->io_timeout);

	/*
	 * If we happen to renew our lease after we've started killing pids,
	 * the period we allow for graceful shutdown will be extended. This
	 * is an incidental effect, although it may be nice. The previous
	 * behavior would still be ok, where we only ever allow up to
	 * kill_grace_seconds for graceful shutdown before moving to sigkill.
	 */
	pthread_mutex_lock(&sp->mutex);
	last_success = sp->lease_status.renewal_last_success;
	pthread_mutex_unlock(&sp->mutex);

	now = monotime();

	for (ci = 0; ci <= client_maxi; ci++) {
		do_kill = 0;

		cl = &client[ci];
		pthread_mutex_lock(&cl->mutex);

		if (!cl->used)
			goto unlock;

		if (cl->pid <= 0)
			goto unlock;

		/* NB this cl may not be using sp, but trying to
		   avoid the expensive client_using_space check */

		if (cl->kill_count >= kill_count_max)
			goto unlock;

		if (cl->kill_count && (now - cl->kill_last < 1))
			goto unlock;

		if (!client_using_space(cl, sp))
			goto unlock;

		cl->kill_last = now;
		cl->kill_count++;

		/*
		 * the transition from using killpath/sigterm to sigkill
		 * is when now >=
		 * last successful lease renewal +
		 * id_renewal_fail_seconds +
		 * kill_grace_seconds
		 */

		in_grace = now < (last_success + id_renewal_fail_seconds + kill_grace_seconds);

		if (sp->external_remove || (external_shutdown > 1)) {
			sig = SIGKILL;
		} else if ((kill_grace_seconds > 0) && in_grace && cl->killpath[0]) {
			sig = SIGRUNPATH;
		} else if (in_grace) {
			sig = SIGTERM;
		} else {
			sig = SIGKILL;
		}

		/*
		 * sigterm will be used in place of sigkill if restricted
		 * sigkill will be used in place of sigterm if restricted
		 */

		if ((sig == SIGKILL) && (cl->restricted & SANLK_RESTRICT_SIGKILL))
			sig = SIGTERM;

		if ((sig == SIGTERM) && (cl->restricted & SANLK_RESTRICT_SIGTERM))
			sig = SIGKILL;

		do_kill = 1;
 unlock:
		pthread_mutex_unlock(&cl->mutex);

		if (!do_kill)
			continue;

		send_helper_kill(sp, cl, sig);
	}
}

向helper进程发送kill命令，helper进程进行kill，但是并不向主进程返回执行结果。

src/main.c

填充相应的数据结构，并向管道写入命令：

/*
 * We cannot block the main thread on this write, so the pipe is
 * NONBLOCK, and write fails with EAGAIN when the pipe is full.
 * With 512 msg size and 64k default pipe size, the pipe will be full
 * if we quickly send kill messages for 128 pids.  We retry
 * the kill once a second, so we'll retry the write again in
 * a second.
 *
 * By setting the pipe size to 1MB in setup_helper, we could quickly send 2048
 * msgs before getting EAGAIN.
 */

static void send_helper_kill(struct space *sp, struct client *cl, int sig)
{
	struct helper_msg hm;
	int rv;

	/*
	 * We come through here once a second while the pid still has
	 * leases.  We only send a single RUNPATH message, so after
	 * the first RUNPATH goes through we set CL_RUNPATH_SENT to
	 * avoid futher RUNPATH's.
	 */

	if ((cl->flags & CL_RUNPATH_SENT) && (sig == SIGRUNPATH))
		return;

	if (helper_kill_fd == -1) {
		log_error("send_helper_kill pid %d no fd", cl->pid);
		return;
	}

	memset(&hm, 0, sizeof(hm));

	if (sig == SIGRUNPATH) {
		hm.type = HELPER_MSG_RUNPATH;
		memcpy(hm.path, cl->killpath, SANLK_HELPER_PATH_LEN);
		memcpy(hm.args, cl->killargs, SANLK_HELPER_ARGS_LEN);

		/* only include pid if it's requested as a killpath arg */
		if (cl->flags & CL_KILLPATH_PID)
			hm.pid = cl->pid;
	} else {
		hm.type = HELPER_MSG_KILLPID;
		hm.sig = sig;
		hm.pid = cl->pid;
	}

	log_erros(sp, "kill %d sig %d count %d", cl->pid, sig, cl->kill_count);

 retry:
	rv = write(helper_kill_fd, &hm, sizeof(hm));
	if (rv == -1 && errno == EINTR)
		goto retry;

	/* pipe is full, we'll try again in a second */
	if (rv == -1 && errno == EAGAIN) {
		helper_full_count++;
		log_space(sp, "send_helper_kill pid %d sig %d full_count %u",
			  cl->pid, sig, helper_full_count);
		return;
	}

	/* helper exited or closed fd, quit using helper */
	if (rv == -1 && errno == EPIPE) {
		log_erros(sp, "send_helper_kill EPIPE");
		close_helper();
		return;
	}

	if (rv != sizeof(hm)) {
		/* this shouldn't happen */
		log_erros(sp, "send_helper_kill pid %d error %d %d",
			  cl->pid, rv, errno);
		close_helper();
		return;
	}

	if (sig == SIGRUNPATH)
		cl->flags |= CL_RUNPATH_SENT;
}

当进程被杀死后，主进程main_loop函数在与该进程连接poll套接字时，会收到POLLIN和POLLHUP事件，从而执行如下函数，释放client相关资源：

src/main.c

static int main_loop(void)
{
...
			if (pollfd[i].revents & (POLLERR | POLLHUP | POLLNVAL)) {
				deadfn = client[i].deadfn;
				if (deadfn)
					deadfn(i);
			}
...
}

void client_pid_dead(int ci)
{
	struct client *cl = &client[ci];
	int cmd_active;
	int i, pid;

	/* cmd_acquire_thread may still be waiting for the tokens
	   to be acquired.  if it is, cl->pid_dead tells it to release them
	   when finished.  Similarly, cmd_release_thread, cmd_inquire_thread
	   are accessing cl->tokens */

	pthread_mutex_lock(&cl->mutex);
	if (!cl->used || cl->fd == -1 || cl->pid == -1) {
		/* should never happen */
		pthread_mutex_unlock(&cl->mutex);
		log_error("client_pid_dead %d,%d,%d u %d a %d s %d bad state",
			  ci, cl->fd, cl->pid, cl->used,
			  cl->cmd_active, cl->suspend);
		return;
	}

	log_debug("client_pid_dead %d,%d,%d cmd_active %d suspend %d",
		  ci, cl->fd, cl->pid, cl->cmd_active, cl->suspend);

	if (cl->kill_count)
		log_error("dead %d ci %d count %d", cl->pid, ci, cl->kill_count);

	cmd_active = cl->cmd_active;
	pid = cl->pid;
	cl->pid = -1;
	cl->pid_dead = 1;

	/* when cmd_active is set and cmd_a,r,i_thread is done and takes
	   cl->mutex to set cl->cmd_active to 0, it will see cl->pid_dead is 1
	   and know they need to release cl->tokens and call client_free */

	/* make poll() ignore this connection */
	pollfd[ci].fd = -1;
	pollfd[ci].events = 0;

	pthread_mutex_unlock(&cl->mutex);

	/* it would be nice to do this SIGKILL as a confirmation that the pid
	   is really gone (i.e. didn't just close the fd) if we always had root
	   permission to do it */

	/* kill(pid, SIGKILL); */

	if (cmd_active) {
		log_debug("client_pid_dead %d,%d,%d defer to cmd %d",
			  ci, cl->fd, pid, cmd_active);
		return;
	}

	/* use async release here because this is the main thread that we don't
	   want to block doing disk lease i/o */

	pthread_mutex_lock(&cl->mutex);
	for (i = 0; i < cl->tokens_slots; i++) {
		if (cl->tokens[i]) {
			release_token_async(cl->tokens[i]);
			free(cl->tokens[i]);
		}
	}

	_client_free(ci);
	pthread_mutex_unlock(&cl->mutex);
}

处理设置killpath命令的函数

src/cmd.c

命令处理函数：

void call_cmd_thread(struct task *task, struct cmd_args *ca)
{
	switch (ca->header.cmd) {
	case SM_CMD_ACQUIRE:
		cmd_acquire(task, ca);
		break;
	case SM_CMD_RELEASE:
		cmd_release(task, ca);
		break;
	case SM_CMD_INQUIRE:
		cmd_inquire(task, ca);
		break;
	case SM_CMD_CONVERT:
		cmd_convert(task, ca);
		break;
	case SM_CMD_REQUEST:
		cmd_request(task, ca);
		break;
	case SM_CMD_ADD_LOCKSPACE:
		strcpy(client[ca->ci_in].owner_name, "add_lockspace");
		cmd_add_lockspace(ca);
		break;
	case SM_CMD_INQ_LOCKSPACE:
		strcpy(client[ca->ci_in].owner_name, "inq_lockspace");
		cmd_inq_lockspace(ca);
		break;
	case SM_CMD_REM_LOCKSPACE:
		strcpy(client[ca->ci_in].owner_name, "rem_lockspace");
		cmd_rem_lockspace(ca);
		break;
	case SM_CMD_ALIGN:
		cmd_align(task, ca);
		break;
	case SM_CMD_WRITE_LOCKSPACE:
		cmd_write_lockspace(task, ca);
		break;
	case SM_CMD_WRITE_RESOURCE:
		cmd_write_resource(task, ca);
		break;
	case SM_CMD_READ_LOCKSPACE:
		cmd_read_lockspace(task, ca);
		break;
	case SM_CMD_READ_RESOURCE:
		cmd_read_resource(task, ca);
		break;
	case SM_CMD_READ_RESOURCE_OWNERS:
		cmd_read_resource_owners(task, ca);
		break;
	case SM_CMD_EXAMINE_LOCKSPACE:
	case SM_CMD_EXAMINE_RESOURCE:
		cmd_examine(task, ca);
		break;
	case SM_CMD_KILLPATH:
		cmd_killpath(task, ca);
		break;
	case SM_CMD_SET_LVB:
		cmd_set_lvb(task, ca);
		break;
	case SM_CMD_GET_LVB:
		cmd_get_lvb(task, ca);
		break;
	case SM_CMD_SHUTDOWN_WAIT:
		cmd_shutdown_wait(task, ca);
		break;
	case SM_CMD_SET_EVENT:
		cmd_set_event(task, ca);
		break;
	};
}

一个进程只能设置自己的killpath，而不能设置其他进程的：

/* N.B. the api doesn't support one client setting killpath for another
   pid/client */

static void cmd_killpath(struct task *task, struct cmd_args *ca)
{
	struct client *cl;
	int cl_ci = ca->ci_target;
	int cl_fd = ca->cl_fd;
	int cl_pid = ca->cl_pid;
	int rv, result, pid_dead;

	cl = &client[cl_ci];

	log_debug("cmd_killpath %d,%d,%d flags %x",
		  cl_ci, cl_fd, cl_pid, ca->header.cmd_flags);

	rv = recv(cl_fd, cl->killpath, SANLK_HELPER_PATH_LEN, MSG_WAITALL);
	if (rv != SANLK_HELPER_PATH_LEN) {
		log_error("cmd_killpath %d,%d,%d recv path %d %d",
			  cl_ci, cl_fd, cl_pid, rv, errno);
		memset(cl->killpath, 0, SANLK_HELPER_PATH_LEN);
		memset(cl->killargs, 0, SANLK_HELPER_ARGS_LEN);
		result = -ENOTCONN;
		goto done;
	}

	rv = recv(cl_fd, cl->killargs, SANLK_HELPER_ARGS_LEN, MSG_WAITALL);
	if (rv != SANLK_HELPER_ARGS_LEN) {
		log_error("cmd_killpath %d,%d,%d recv args %d %d",
			  cl_ci, cl_fd, cl_pid, rv, errno);
		memset(cl->killpath, 0, SANLK_HELPER_PATH_LEN);
		memset(cl->killargs, 0, SANLK_HELPER_ARGS_LEN);
		result = -ENOTCONN;
		goto done;
	}

	cl->killpath[SANLK_HELPER_PATH_LEN - 1] = '\0';
	cl->killargs[SANLK_HELPER_ARGS_LEN - 1] = '\0';

	if (ca->header.cmd_flags & SANLK_KILLPATH_PID)
		cl->flags |= CL_KILLPATH_PID;

	result = 0;
 done:
	pthread_mutex_lock(&cl->mutex);
	pid_dead = cl->pid_dead;
	cl->cmd_active = 0;
	pthread_mutex_unlock(&cl->mutex);

	if (pid_dead) {
		/* release tokens in case a client sets/changes its killpath
		   after it has acquired leases */
		release_cl_tokens(task, cl);
		client_free(cl_ci);
		return;
	}

	send_result(cl_fd, &ca->header, result);
	client_resume(ca->ci_in);
}

发送修改killpath的消息给sanlock身份的守护进程；
src/client.c

int sanlock_killpath(int sock, uint32_t flags, const char *path, char *args)
{
	char path_max[SANLK_HELPER_PATH_LEN];
	char args_max[SANLK_HELPER_ARGS_LEN];
	int rv, datalen;

	datalen = SANLK_HELPER_PATH_LEN + SANLK_HELPER_ARGS_LEN;

	memset(path_max, 0, sizeof(path_max));
	memset(args_max, 0, sizeof(args_max));

	snprintf(path_max, SANLK_HELPER_PATH_LEN-1, "%s", path);
	snprintf(args_max, SANLK_HELPER_ARGS_LEN-1, "%s", args);

	rv = send_header(sock, SM_CMD_KILLPATH, flags, datalen, 0, -1);
	if (rv < 0)
		return rv;

	rv = send_data(sock, path_max, SANLK_HELPER_PATH_LEN, 0);
	if (rv < 0) {
		rv = -errno;
		goto out;
	}

	rv = send_data(sock, args_max, SANLK_HELPER_ARGS_LEN, 0);
	if (rv < 0) {
		rv = -errno;
		goto out;
	}

	rv = recv_result(sock);
 out:
	return rv;
}

libvirt中lockfailure相关的源码分析

libvirt中使用on_lockfailure标签来设置虚拟机的锁失败后的处理，其处理方法如下（src/conf/domain_conf.h）：

typedef enum {
    VIR_DOMAIN_LOCK_FAILURE_DEFAULT,
    VIR_DOMAIN_LOCK_FAILURE_POWEROFF,
    VIR_DOMAIN_LOCK_FAILURE_RESTART,
    VIR_DOMAIN_LOCK_FAILURE_PAUSE,
    VIR_DOMAIN_LOCK_FAILURE_IGNORE,

    VIR_DOMAIN_LOCK_FAILURE_LAST
} virDomainLockFailureAction;

目前只支持“默认”、“关机”和“暂停”的处理方式，如果是默认则不做任何处理，相当于让sanlock自己处理（src/locking/lock_driver_sanlock.c）。

virLockManagerSanlockRegisterKillscript(int sock,
                                        const char *vmuri,
                                        const char *uuidstr,
                                        virDomainLockFailureAction action)
{
    virBuffer buf = VIR_BUFFER_INITIALIZER;
    char *path;
    char *args = NULL;
    int ret = -1;
    int rv;

    switch (action) {
    case VIR_DOMAIN_LOCK_FAILURE_DEFAULT:
        return 0;

    case VIR_DOMAIN_LOCK_FAILURE_POWEROFF:
    case VIR_DOMAIN_LOCK_FAILURE_PAUSE:
        break;

    case VIR_DOMAIN_LOCK_FAILURE_RESTART:
    case VIR_DOMAIN_LOCK_FAILURE_IGNORE:
    case VIR_DOMAIN_LOCK_FAILURE_LAST:
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                       _("Failure action %s is not supported by sanlock"),
                       virDomainLockFailureTypeToString(action));
        goto cleanup;
    }

    virBufferEscape(&buf, '\\', "\\ ", "%s", vmuri);
    virBufferAddLit(&buf, " ");
    virBufferEscape(&buf, '\\', "\\ ", "%s", uuidstr);
    virBufferAddLit(&buf, " ");
    virBufferEscape(&buf, '\\', "\\ ", "%s",
                    virDomainLockFailureTypeToString(action));

    if (virBufferCheckError(&buf) < 0)
        goto cleanup;

    /* Unfortunately, sanlock_killpath() does not use const for either
     * path or args even though it will just copy them into its own
     * buffers.
     */
    path = (char *) VIR_LOCK_MANAGER_SANLOCK_KILLPATH;
    args = virBufferContentAndReset(&buf);

    VIR_DEBUG("Register sanlock killpath: %s %s", path, args);

    /* sanlock_killpath() would just crop the strings */
    if (strlen(path) >= SANLK_HELPER_PATH_LEN) {
        virReportError(VIR_ERR_INTERNAL_ERROR,
                       _("Sanlock helper path is longer than %d: '%s'"),
                       SANLK_HELPER_PATH_LEN - 1, path);
        goto cleanup;
    }
    if (strlen(args) >= SANLK_HELPER_ARGS_LEN) {
        virReportError(VIR_ERR_INTERNAL_ERROR,
                       _("Sanlock helper arguments are longer than %d:"
                         " '%s'"),
                       SANLK_HELPER_ARGS_LEN - 1, args);
        goto cleanup;
    }

    if ((rv = sanlock_killpath(sock, 0, path, args)) < 0) {
        if (rv <= -200) {
            virReportError(VIR_ERR_INTERNAL_ERROR,
                           _("Failed to register lock failure action:"
                             " error %d"), rv);
        } else {
            virReportSystemError(-rv, "%s",
                                 _("Failed to register lock failure"
                                   " action"));
        }
        goto cleanup;
    }

    ret = 0;

 cleanup:
    VIR_FREE(args);
    return ret;
}

如果是“关机”或“暂停”的处理方式则执行如下程序，且不可更改（src/locking/lock_driver_sanlock.c）:

#define VIR_LOCK_MANAGER_SANLOCK_KILLPATH LIBEXECDIR "/libvirt_sanlock_helper"

libvirt_sanlock_helper程序目前只支持“关机”和“暂停”（src/locking/sanlock_helper.c）。

int
main(int argc, char **argv)
{
    const char *uri;
    const char *uuid;
    virDomainLockFailureAction action;
    char *xml = NULL;
    virConnectPtr conn = NULL;
    virDomainPtr dom = NULL;
    int ret = EXIT_FAILURE;

    int authTypes[] = {
        VIR_CRED_AUTHNAME,
        VIR_CRED_ECHOPROMPT,
        VIR_CRED_PASSPHRASE,
        VIR_CRED_NOECHOPROMPT,
    };
    virConnectAuth auth = {
        .credtype = authTypes,
        .ncredtype = ARRAY_CARDINALITY(authTypes),
        .cb = authCallback,
    };

    if (virGettextInitialize() < 0)
        exit(EXIT_FAILURE);

    if (getArgs(argc, argv, &uri, &uuid, &action) < 0)
        goto cleanup;

    if (!(conn = virConnectOpenAuth(uri, &auth, 0)) ||
        !(dom = virDomainLookupByUUIDString(conn, uuid)))
        goto cleanup;

    switch (action) {
    case VIR_DOMAIN_LOCK_FAILURE_POWEROFF:
        if (virDomainDestroy(dom) == 0 ||
            virDomainIsActive(dom) == 0)
            ret = EXIT_SUCCESS;
        break;

    case VIR_DOMAIN_LOCK_FAILURE_PAUSE:
        if (virDomainSuspend(dom) == 0)
            ret = EXIT_SUCCESS;
        break;

    case VIR_DOMAIN_LOCK_FAILURE_DEFAULT:
    case VIR_DOMAIN_LOCK_FAILURE_RESTART:
    case VIR_DOMAIN_LOCK_FAILURE_IGNORE:
    case VIR_DOMAIN_LOCK_FAILURE_LAST:
        fprintf(stderr, _("unsupported failure action: '%s'\n"),
                virDomainLockFailureTypeToString(action));
        break;
    }

 cleanup:
    virObjectUnref(dom);
    if (conn)
        virConnectClose(conn);
    VIR_FREE(xml);

    return ret;
}