管理 AirFlow 方法
进程管理工具Supervisord
- 安装进程管理工具Supervisord管理airflow进程
easy_install supervisor #此方法不适用于python3安装(会出现很多问题)
echo_supervisord_conf > /etc/supervisord.conf
- 编辑文件supervisord.conf,添加启动命令
vi /etc/supervisord.conf
[program:airflow_web]
command=/usr/bin/airflow webserver -p 8080
[program:airflow_worker]
command=/usr/bin/airflow worker
[program:airflow_scheduler]
command=/usr/bin/airflow scheduler
- 启动supervisord服务
/usr/bin/supervisord -c /etc/supervisord.conf
- 此时可以用 supervisorctl 来管理airflow服务了
supervisorctl start airflow_web
supervisorctl stop airflow_web
supervisorctl restart airflow_web
supervisorctl stop all
进程管理工具 systemd
- vim /etc/sysconfig/airflow # systemd需要调用此文件,一般定义的是airflow的变量
AIRFLOW_CONFIG=/root/airflow/airflow.cfg
AIRFLOW_HOME=/root/airflow
- vim /usr/lib/systemd/system/airflow-webserver.service #systemctl 管理的服务名
其他的服务也可以使用此方式进行定义
[Unit]
Description=Airflow webserver daemon
After=network.target postgresql.service mysql.service redis.service
Wants=postgresql.service mysql.service redis.service
[Service]
EnvironmentFile=/etc/sysconfig/airflow
User=root
Group=root
Type=simple
ExecStart=/bin/bash -c "export PATH=${PATH}:/usr/local/python3/bin/ ; /usr/local/python3/bin/airflow webserver -p 8080 --pid /root/airflow/service/webserver.pid -A /root/airflow/service/webserver.out -E /root/airflow/service/webserver.err -l /root/airflow/service/webserver.log"
KillMode=process
Restart=on-failure
RestartSec=5s
PrivateTmp=true
[Install]
WantedBy=multi-user.target
- systemctl daemon-reload #加载服务
- systemctl status airflow-webserver.service #查看服务状态,后期就可以用这种方式进行管理
使用脚本管理airflow
#!/bin/bash
webserver_status(){
echo -e "\e[36m Checking service status, please wait ... \e[0m"
sleep 3
Status=`ps -elf| grep "airflow[ -]webserver" |wc -l`
if [ $Status -eq 0 ] ;then
echo -e "\e[31m webserver is stop !!! \e[0m"
else
echo -e "\e[32m webserver is running... \e[0m"
fi
}
webserver_start(){
echo -e "\e[36m Starting airflow webserver ... \e[0m"
sleep 1
nohup /usr/local/python3/bin/airflow webserver >> /root/airflow/service/webserver.log 2>&1 &
webserver_status
}
webserver_stop(){
echo -e "\e[36m Stopping airflow webserver ... \e[0m"
sleep 1
/usr/bin/kill -9 `ps -elf| grep "airflow[ -]webserver" | grep -v grep |awk -F" " '{ print $4 }'`
rm -rf /root/airflow/airflow-webserver.pid
webserver_status
}
scheduler_status(){
echo -e "\e[36m Checking service status, please wait ... \e[0m"
sleep 3
Status=`ps -elf| grep "airflow[ -]scheduler" |wc -l`
if [ $Status -eq 0 ] ;then
echo -e "\e[31m scheduler is stop !!! \e[0m"
else
echo -e "\e[32m scheduler is running... \e[0m"
fi
}
scheduler_start(){
echo -e "\e[36m Starting airflow scheduler ... \e[0m"
sleep 1
nohup /usr/local/python3/bin/airflow scheduler >> /root/airflow/service/scheduler.log 2>&1 &
scheduler_status
}
scheduler_stop(){
echo -e "\e[36m Stopping airflow scheduler ... \e[0m"
sleep 1
/usr/bin/kill -9 `ps -elf| grep "airflow[ -]scheduler" | grep -v grep |awk -F" " '{ print $4 }'`
rm -rf /root/airflow/airflow-scheduler.pid
scheduler_status
}
flower_status(){
echo -e "\e[36m Checking service status, please wait ... \e[0m"
sleep 3
Status=`netstat -anputl| grep 5555 | grep LISTEN | awk -F" " '{ print $7 }' | awk -F"/" '{ print $1 }' |wc -l`
if [ $Status -eq 0 ] ;then
echo -e "\e[31m flower is stop !!! \e[0m"
else
echo -e "\e[32m flower is running... \e[0m"
fi
}
flower_start(){
echo -e "\e[36m Starting airflow flower ... \e[0m"
sleep 1
nohup /usr/local/python3/bin/airflow flower >> /root/airflow/service/flower.log 2>&1 &
flower_status
}
flower_stop(){
echo -e "\e[36m Stopping airflow flower ... \e[0m"
sleep 1
/usr/bin/kill -9 `netstat -anputl| grep 5555 | grep LISTEN | awk -F" " '{ print $7 }' | awk -F"/" '{ print $1 }'`
rm -rf /root/airflow/airflow-flower.pid
flower_status
}
worker_status(){
echo