haproxy无缝热加载的配置和操作[下]

上接：haproxy无缝热加载的配置和操作[上]

3.2. systemd启动方式

  本来以为systemd的启动方式和命令行方式启动是一样的。但是实践下来却行不通，我最初是这么来配置system service文件的：

[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target

[Service]
ExecStartPre=/usr/sbin/haproxy -f /var/lib/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy -f /var/lib/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid -Ws
ExecReload=/usr/sbin/haproxy -f /var/lib/haproxy/haproxy-443.cfg -p /var/lib/haproxy/haproxy.pid -Ws -sf $MAINPID -x /var/lib/haproxy/stats/haproxy.sock
KillMode=mixed
Restart=always
SuccessExitStatus=143
Type=notify
LimitCORE=infinity

  配置完成后，启动和停止haproxy包括重启haproxy都没有问题，但是在命令行用：

systemctl reload haproxy

  命令执行的时候发现会卡住，必须用ctrl-c强制退出才行。用ps查看进程，发现老的进程已经退出了，新的进程也已经起来了，新的连接请求也没有问题。而且过了大概90s以后，系统日志里面会报重启超时的错误信息。

  后来想到应该systemd设计者认为reload操作，它的老进程是持续运行的，而不是用新进程来替换老进程，这样子导致systemd对服务的状态检测逻辑有问题。

  既然这样，我们完全可以在haproxy外面再套一个壳，由这个壳和systemd进行交互，reload的时候保证这个壳进程不变，由这个壳进程来实现模拟非systemd启动方式，这样子就可以完美适配了。于是就有了下面的壳程序，haproxy-systemd-wrapper，源码如下：

/*
 * Wrapper to make haproxy systemd-compliant.
 *
 * Copyright 2013 Marc-Antoine Perennou <Marc-Antoine@Perennou.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 *
 */

#include <errno.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/wait.h>
#include <systemd/sd-daemon.h>

#define REEXEC_FLAG "HAPROXY_SYSTEMD_REEXEC"
#define SD_DEBUG "<7>"
#define SD_NOTICE "<5>"

static volatile sig_atomic_t caught_signal;

static char *pid_file = "/var/run/haproxy.pid";
static char *unix_socket_file = NULL;
static int wrapper_argc;
static char **wrapper_argv;

/* returns the path to the haproxy binary into <buffer>, whose size indicated
 * in <buffer_size> must be at least 1 byte long.
 */
static void locate_haproxy(char *buffer, size_t buffer_size)
{
    char *end = NULL;
    int len;

    len = readlink("/proc/self/exe", buffer, buffer_size - 1);
    if (len == -1)
        goto fail;

    buffer[len] = 0;
    end = strrchr(buffer, '/');
    if (end == NULL)
        goto fail;

    if (strcmp(end + strlen(end) - 16, "-systemd-wrapper") == 0) {
        end[strlen(end) - 16] = '\0';
        return;
    }

    end[1] = '\0';
    strncpy(end + 1, "haproxy", buffer + buffer_size - (end + 1));
    buffer[buffer_size - 1] = '\0';
    return;
 fail:
    strncpy(buffer, "/usr/sbin/haproxy", buffer_size);
    buffer[buffer_size - 1] = '\0';
    return;
}

static void spawn_haproxy(char **pid_strv, int nb_pid)
{
    char haproxy_bin[512];
    pid_t pid;
    int main_argc;
    char **main_argv;

    main_argc = wrapper_argc - 1;
    main_argv = wrapper_argv + 1;

    pid = fork();
    if (!pid) {
        /* 3 for "haproxy -Ds -sf" */
        char **argv = calloc(4 + main_argc + nb_pid + 1, sizeof(char *));
        int i;
        int argno = 0;
        locate_haproxy(haproxy_bin, 512);
        argv[argno++] = haproxy_bin;

        for (i = 0; i < main_argc; ++i) {            
            if (nb_pid <= 0 && main_argv[i][0] == '-' && main_argv[i][1] == 'x') 
            {
                i++;
            }else {
                argv[argno++] = main_argv[i];
            }
        }
        //argv[argno++] = "-W";
        if (nb_pid > 0) {
            argv[argno++] = "-sf";
            for (i = 0; i < nb_pid; ++i)
                argv[argno++] = pid_strv[i];
        }
        argv[argno] = NULL;

        fprintf(stderr, SD_DEBUG "haproxy-systemd-wrapper: executing ");
        for (i = 0; argv[i]; ++i)
            fprintf(stderr, "%s ", argv[i]);
        fprintf(stderr, "\n");

        execv(argv[0], argv);
        exit(0);
    }
}

static int read_pids(char ***pid_strv)
{
    FILE *f = fopen(pid_file, "r");
    int read = 0, allocated = 8;
    char pid_str[10];

    if (!f)
        return 0;

    *pid_strv = malloc(allocated * sizeof(char *));
    while (1 == fscanf(f, "%s\n", pid_str)) {
        if (read == allocated) {
            allocated *= 2;
            *pid_strv = realloc(*pid_strv, allocated * sizeof(char *));
        }
        (*pid_strv)[read++] = strdup(pid_str);
    }

    fclose(f);

    return read;
}

static void signal_handler(int signum)
{
    caught_signal = signum;
}

static void do_restart(void)
{
    setenv(REEXEC_FLAG, "1", 1);
    fprintf(stderr, SD_NOTICE "haproxy-systemd-wrapper: re-executing\n");

    execv(wrapper_argv[0], wrapper_argv);
}

static void do_shutdown(void)
{
    int i, pid;
    char **pid_strv = NULL;
    int nb_pid = read_pids(&pid_strv);
    for (i = 0; i < nb_pid; ++i) {
        pid = atoi(pid_strv[i]);
        if (pid > 0) {
            fprintf(stderr, SD_DEBUG "haproxy-systemd-wrapper: SIGINT -> %d\n", pid);
            kill(pid, SIGINT);
            free(pid_strv[i]);
        }
    }
    free(pid_strv);
}

static void init(int argc, char **argv)
{
    while (argc > 1) {
        if ((*argv)[0] == '-' && (*argv)[1] == 'p') {
            pid_file = *(argv + 1);
        }

        if ((*argv)[0] == '-' && (*argv)[1]  == 'x') {
            unix_socket_file = *(argv + 1);
        }

        --argc; ++argv;
    }
}

int main(int argc, char **argv)
{
    int status;
    struct sigaction sa;

    wrapper_argc = argc;
    wrapper_argv = argv;

    --argc; ++argv;
    init(argc, argv);

    memset(&sa, 0, sizeof(struct sigaction));
    sa.sa_handler = &signal_handler;
    sigaction(SIGUSR2, &sa, NULL);
    sigaction(SIGHUP, &sa, NULL);
    sigaction(SIGINT, &sa, NULL);
    sigaction(SIGTERM, &sa, NULL);

    sd_notifyf(0, "STATUS=Starting...");


    if (getenv(REEXEC_FLAG) != NULL) {
        /* We are being re-executed: restart HAProxy gracefully */
        int i;
        char **pid_strv = NULL;
        int nb_pid = read_pids(&pid_strv);

        unsetenv(REEXEC_FLAG);
        spawn_haproxy(pid_strv, nb_pid);

        for (i = 0; i < nb_pid; ++i)
            free(pid_strv[i]);
        free(pid_strv);
    }
    else {
        /* Start a fresh copy of HAProxy */
        spawn_haproxy(NULL, 0);
    }

    sd_notifyf(0, "READY=1\nMAINPID=%lu", (unsigned long)getpid());

    fprintf(stderr, SD_NOTICE"PID=%lu", (unsigned long)getpid());

    status = -1;
    while (-1 != wait(&status) || errno == EINTR) {
        if (caught_signal == SIGUSR2 || caught_signal == SIGHUP) {
            caught_signal = 0;
            do_restart();
        }
        else if (caught_signal == SIGINT || caught_signal == SIGTERM) {
            caught_signal = 0;
            do_shutdown();
        }
    }

    sd_notifyf(0, "STOPPING=1");

    fprintf(stderr, SD_NOTICE "haproxy-systemd-wrapper: exit, haproxy RC=%d\n",
            status);

    sd_notifyf(0, "STATUS=Stopped");
    return status;
}

  对源码的实现逻辑不做过多的解释。这里说明一下，haproxy-systemd-wrapper接收和haproxy命令一样的参数，在启动haproxy进程的时候会将命令行参数传递给haproxy。
  实现了haproxy-systemd-wrapper以后，需要配置haproxy以非daemon方式启动，
因此，在配置文件的global节中，需要把daemon配置指令去掉。

  然后，配置下面的systemd service文件：

[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target

[Service]
ExecStartPre=/usr/sbin/haproxy -f /var/lib/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy-systemd-wrapper -f /var/lib/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid -Ws
ExecReload=/bin/kill -USR2 $MAINPID
KillMode=mixed
Restart=always
SuccessExitStatus=143
Type=notify
LimitCORE=infinity

   当systemctl reload haproxy的时候，不再是通过ExecReload的命令启动一个haproxy进程，而是发送一个kill -USR2消息给haproxy-systemd-wrapper，由haproxy-systemd-wrapper模拟启动以下命令：

 /haproxy -f haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid) -x /var/lib/haproxy/stats/haproxy.sock

  从而完美实现了systemd运行模式下的无缝热加载。

4. 总结

   以上介绍了两种haproxy运行模式下实现无缝热加载的配置方法，经过测试，达到预期效果。上述方法供大家参考。