在 CentOS7 上安装 Airflow1.10.11

1, 安装配置

# 看一下可以安装的 gcc 的版本
[root@do-airflow ~]# yum list | grep gcc.x86_64
libgcc.x86_64                               4.8.5-16.el7               @anaconda
gcc.x86_64                                  4.8.5-39.el7               base
libgcc.x86_64                               4.8.5-39.el7               base

# 安装gcc
[root@do-airflow ~]# yum -y install gcc

# 确认gcc安装成功
[root@do-airflow ~]# yum list installed | grep gcc
gcc.x86_64                            4.8.5-39.el7                     @base
libgcc.x86_64                         4.8.5-39.el7                     @base

# 看一下可以安装的 python3 的版本
[root@do-airflow ~]# yum list | grep python3.x86_64
python3.x86_64                              3.6.8-13.el7               base

# 安装 python3
[root@do-airflow ~]# yum -y install python3 python3-devel

# 检查安装成功
[root@do-airflow ~]# whereis python3
python3: /usr/bin/python3 /usr/bin/python3.6 /usr/bin/python3.6m /usr/bin/python3.6-config /usr/bin/python3.6m-config /usr/bin/python3.6m-x86_64-config /usr/lib/python3.6 /usr/lib64/python3.6 /usr/include/python3.6m /usr/share/man/man1/python3.1.gz
[root@do-airflow ~]# ll /usr/bin/python3
lrwxrwxrwx. 1 root root 9 Jul 14 23:31 /usr/bin/python3 -> python3.6
[root@do-airflow ~]# /usr/bin/python3 --version
Python 3.6.8

#------- 以下可以跳过，如果想输入python命令，直接执行的是python3，才需要配置

# 看一下之前的 python 命令软连接
[root@do-airflow ~]# ll /usr/bin/python
lrwxrwxrwx. 1 root root 7 Jul 14 20:48 /usr/bin/python -> python2
[root@do-airflow ~]# ll /usr/bin/python2
lrwxrwxrwx. 1 root root 9 Jul 14 20:48 /usr/bin/python2 -> python2.7
[root@do-airflow ~]# ll /usr/bin/python2.7
-rwxr-xr-x. 1 root root 7136 Aug  3  2017 /usr/bin/python2.7

# 创建指向 python3 的软连接
[root@do-airflow ~]# rm -f /usr/bin/python
[root@do-airflow ~]# ln -s /usr/bin/python3 /usr/bin/python
[root@do-airflow ~]# python --version
Python 3.6.8

# 因为升级了版本以后，yum 就会报错，要修复一下
[root@do-airflow ~]# vi /usr/bin/yum
[root@do-airflow ~]# cat /usr/bin/yum
#!/usr/bin/python2

[root@do-airflow ~]# vi /usr/libexec/urlgrabber-ext-down
[root@do-airflow ~]# cat /usr/libexec/urlgrabber-ext-down
#!/usr/bin/python2

# 因为升级了版本以后，firewalld 服务会启动失败，也要修复一下
[root@do-airflow ~]# vi /usr/sbin/firewalld
[root@do-airflow ~]# cat /usr/sbin/firewalld
#!/usr/bin/python2 -Es

[root@do-airflow ~]# vi /usr/bin/firewall-cmd
[root@do-airflow ~]# cat /usr/bin/firewall-cmd
#!/usr/bin/python2 -Es

#------- 以上可以跳过，如果想输入python命令，直接执行的是python3，才需要配置

# 升级 pip3
## 官网下载超慢，现提供几个镜像站点
## 阿里云 http://mirrors.aliyun.com/pypi/simple/
## 中国科技大学 https://pypi.mirrors.ustc.edu.cn/simple/
## 豆瓣(douban) http://pypi.douban.com/simple/
## 清华大学 https://pypi.tuna.tsinghua.edu.cn/simple/
## 中国科学技术大学 http://pypi.mirrors.ustc.edu.cn/simple/
[root@do-airflow ~]# pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple --default-timeout=100 --upgrade pip

# 安装 airflow
[root@do-airflow ~]# pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple --default-timeout=100 apache-airflow

# 确认安装的 airflow 的版本
[root@do-airflow ~]# airflow version
1.10.11

# 安装 pymysql cryptography
[root@do-airflow ~]# pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple pymysql cryptography

# 配置环境变量，指明 airflow dag所在的目录
[root@do-airflow ~]# vi /etc/profile
[root@do-airflow ~]# cat /etc/profile
export AIRFLOW_HOME=/opt/airflow
[root@do-airflow ~]# source /etc/profile

2, 初始化

# 初始化 airflow
## 默认使用内置的 sqlite
[root@do-airflow ~]# airflow initdb

# 配置时区
[root@do-airflow ~]# vi /opt/airflow/airflow.cfg
[root@do-airflow ~]# cat /opt/airflow/airflow.cfg | grep default_timezone
default_timezone = Asia/Shanghai

[root@do-airflow ~]# vi /usr/local/lib/python3.6/site-packages/airflow/utils/timezone.py
## 在27行 utc = pendulum.timezone('UTC') 下添加如下代码
from airflow.configuration import conf
try:
        tz = conf.get("core", "default_timezone")
        if tz == "system":
                utc = pendulum.local_timezone()
        else:
                utc = pendulum.timezone(tz)
except Exception:
        pass
## 添加上述代码的基础上，转往72行 d = dt.datetime.utcnow()，修改为
d = dt.datetime.now()

[root@do-airflow ~]# vi /usr/local/lib/python3.6/site-packages/airflow/utils/sqlalchemy.py
## 在38行 utc = pendulum.timezone('UTC') 下添加如下代码
try:
	tz = conf.get("core", "default_timezone")
	if tz == "system":
		utc = pendulum.local_timezone()
	else:
		utc = pendulum.timezone(tz)
except Exception:
	pass

[root@do-airflow ~]# vi /usr/local/lib/python3.6/site-packages/airflow/www/templates/admin/master.html
## 注释掉40行 var UTCseconds = (x.getTime() + x.getTimezoneOffset()*60*1000); 再增加一行
<!-- var UTCseconds = (x.getTime() + x.getTimezoneOffset()*60*1000); -->
var UTCseconds = x.getTime();
## 添加上述代码的基础上，注释掉44行，"timeFormat":"H:i:s %UTC%", 再增加一行
<!-- "timeFormat":"H:i:s %UTC%", -->
"timeFormat":"H:i:s",

# 配置 airflow 使用 mysql 库
## 创库
[root@do-airflow ~]# mysql -uroot -p
Enter password:
mysql> create database airflow;
Query OK, 1 row affected (0.00 sec)

## 修改配置
[root@do-airflow ~]# vi /opt/airflow/airflow.cfg
[root@do-airflow ~]# cat /opt/airflow/airflow.cfg
sql_alchemy_conn = mysql+pymysql://root:Qwer#1234@localhost:3306/airflow
## 顺便修改 Executor
executor = LocalExecutor

## 重新初始化
[root@do-airflow ~]# airflow initdb

3, 服务启动

# 开放通过 tcp 访问 8080
[root@do-airflow ~]# firewall-cmd --add-port=8080/tcp --permanent 
[root@do-airflow ~]# firewall-cmd --reload

# 确认端口已开放
[root@do-airflow ~]# firewall-cmd --list-ports
8080/tcp

# 配置 systemd 服务
# 具体参考这里：https://github.com/apache/airflow/tree/master/scripts/systemd
## 创建环境变量文件
[root@do-airflow ~]# vi /etc/sysconfig/airflow
[root@do-airflow ~]# cat /etc/sysconfig/airflow
AIRFLOW_CONFIG=/opt/airflow/airflow.cfg
AIRFLOW_HOME=/opt/airflow

## 创建 airflow-webserver 服务
[root@do-airflow ~]# vi /usr/lib/systemd/system/airflow-webserver.service
[root@do-airflow ~]# cat /usr/lib/systemd/system/airflow-webserver.service
[Unit]
Description=Airflow webserver daemon
After=network.target mysqld.service
Wants=mysqld.service

[Service]
EnvironmentFile=/etc/sysconfig/airflow
User=root
Group=root
Type=simple
ExecStart=/usr/local/bin/airflow webserver
Restart=on-failure
RestartSec=5s
PrivateTmp=true

[Install]
WantedBy=multi-user.target

## 创建 airflow-scheduler 服务
[root@do-airflow ~]# vi /usr/lib/systemd/system/airflow-scheduler.service
[root@do-airflow ~]# cat /usr/lib/systemd/system/airflow-scheduler.service
[Unit]
Description=Airflow scheduler daemon
After=network.target mysqld.service
Wants=mysqld.service

[Service]
EnvironmentFile=/etc/sysconfig/airflow
User=root
Group=root
Type=simple
ExecStart=/usr/local/bin/airflow scheduler
Restart=always
RestartSec=5s

[Install]
WantedBy=multi-user.target

# 启动服务
[root@do-airflow ~]# systemctl daemon-reload
[root@do-airflow ~]# systemctl start airflow-webserver
[root@do-airflow ~]# systemctl enable airflow-webserver
[root@do-airflow ~]# systemctl status airflow-webserver
[root@do-airflow ~]#
[root@do-airflow ~]# systemctl start airflow-scheduler
[root@do-airflow ~]# systemctl enable airflow-scheduler
[root@do-airflow ~]# systemctl status airflow-scheduler
[root@do-airflow ~]#

# 访问以下url，确认 airflow webserver 正常运行
http://192.168.109.131:8080/admin/