管理 AirFlow 方法

管理 AirFlow 方法

进程管理工具Supervisord

1. 安装进程管理工具Supervisord管理airflow进程

easy_install supervisor  #此方法不适用于python3安装（会出现很多问题）
echo_supervisord_conf > /etc/supervisord.conf

2. 编辑文件supervisord.conf，添加启动命令

vi /etc/supervisord.conf

[program:airflow_web]
command=/usr/bin/airflow webserver -p 8080

[program:airflow_worker]
command=/usr/bin/airflow worker

[program:airflow_scheduler]
command=/usr/bin/airflow scheduler

3. 启动supervisord服务

/usr/bin/supervisord -c /etc/supervisord.conf

4. 此时可以用 supervisorctl 来管理airflow服务了

supervisorctl start airflow_web
supervisorctl stop airflow_web
supervisorctl restart airflow_web
supervisorctl stop all

进程管理工具 systemd

1. vim /etc/sysconfig/airflow # systemd需要调用此文件，一般定义的是airflow的变量

AIRFLOW_CONFIG=/root/airflow/airflow.cfg
AIRFLOW_HOME=/root/airflow

2. vim /usr/lib/systemd/system/airflow-webserver.service #systemctl 管理的服务名
   其他的服务也可以使用此方式进行定义

[Unit]
Description=Airflow webserver daemon
After=network.target postgresql.service mysql.service redis.service
Wants=postgresql.service mysql.service redis.service

[Service]
EnvironmentFile=/etc/sysconfig/airflow
User=root
Group=root
Type=simple
ExecStart=/bin/bash -c "export PATH=${PATH}:/usr/local/python3/bin/ ; /usr/local/python3/bin/airflow webserver -p 8080 --pid /root/airflow/service/webserver.pid  -A /root/airflow/service/webserver.out  -E /root/airflow/service/webserver.err -l /root/airflow/service/webserver.log"

KillMode=process
Restart=on-failure
RestartSec=5s
PrivateTmp=true

[Install]
WantedBy=multi-user.target

3. systemctl daemon-reload #加载服务
4. systemctl status airflow-webserver.service #查看服务状态，后期就可以用这种方式进行管理

使用脚本管理airflow

#!/bin/bash
#=== This is the function about airflow webserver service ===
webserver_status(){
   
    echo -e "\e[36m  Checking service status, please wait ... \e[0m"
    sleep  3
    Status=`ps -elf| grep "airflow[ -]webserver" |wc -l`
    if [ $Status -eq 0 ] ;then
        echo -e "\e[31m webserver is stop !!! \e[0m"
    else 
        echo -e "\e[32m webserver is running... \e[0m"
    fi
}
webserver_start(){
   
    echo  -e "\e[36m Starting airflow webserver ... \e[0m"
    sleep 1
    nohup /usr/local/python3/bin/airflow  webserver >> /root/airflow/service/webserver.log 2>&1 &
    webserver_status
}
webserver_stop(){
   
    echo  -e "\e[36m Stopping airflow webserver ... \e[0m"
    sleep 1
    /usr/bin/kill -9 `ps -elf| grep "airflow[ -]webserver" | grep -v grep |awk -F" " '{ print $4 }'`
    rm -rf /root/airflow/airflow-webserver.pid
    webserver_status
}
#=== This is the function about airflow scheduler service ===
scheduler_status(){
   
    echo -e "\e[36m  Checking service status, please wait ... \e[0m"
    sleep  3
    Status=`ps -elf| grep "airflow[ -]scheduler" |wc -l`
    if [ $Status -eq 0 ] ;then
        echo -e "\e[31m scheduler is stop !!! \e[0m"
    else 
        echo -e "\e[32m scheduler is running... \e[0m"
    fi
}
scheduler_start(){
   
    echo  -e "\e[36m Starting airflow scheduler ... \e[0m"
    sleep 1
    nohup /usr/local/python3/bin/airflow  scheduler >> /root/airflow/service/scheduler.log 2>&1 &
    scheduler_status
}
scheduler_stop(){
   
    echo  -e "\e[36m Stopping airflow scheduler ... \e[0m"
    sleep 1
    /usr/bin/kill -9 `ps -elf| grep "airflow[ -]scheduler" | grep -v grep |awk -F" " '{ print $4 }'`
    rm -rf /root/airflow/airflow-scheduler.pid
    scheduler_status
}
#=== This is the function about airflow flower service ===
flower_status(){
   
    echo -e "\e[36m  Checking service status, please wait ... \e[0m"
    sleep  3
    Status=`netstat  -anputl| grep 5555 | grep LISTEN | awk -F" " '{ print $7 }' | awk -F"/" '{ print $1 }' |wc -l`
    if [ $Status -eq 0 ] ;then
        echo -e "\e[31m flower is stop !!! \e[0m"
    else 
        echo -e "\e[32m flower is running... \e[0m"
    fi
}
flower_start(){
   
    echo  -e "\e[36m Starting airflow flower ... \e[0m"
    sleep 1
    nohup /usr/local/python3/bin/airflow  flower >> /root/airflow/service/flower.log 2>&1 &
    flower_status
}
flower_stop(){
   
    echo  -e "\e[36m Stopping airflow flower ... \e[0m"
    sleep 1
    /usr/bin/kill -9 `netstat  -anputl| grep 5555 | grep LISTEN | awk -F" " '{ print $7 }' | awk -F"/" '{ print $1 }'`
    rm -rf /root/airflow/airflow-flower.pid
    flower_status
}
#=== This is the function about airflow worker service ===
worker_status(){
   
    echo