Airflow 常见问题汇总
1- 中文注释导致dag 不显示
2- task 依赖关系
3- dag 依赖
ExternalTaskSensor
参考以下代码:
# -*- coding: utf-8 -*-
import pendulum
import airflow
from airflow import DAG
from airflow.decorators import task
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from airflow.operators.empty import EmptyOperator
from airflow.sensors.external_task import ExternalTaskSensor
import pandas as pd
from datetime import timedelta
from impala.dbapi import connect
import configparser
#-------------------------------------------------------------------------------
# password meta file
ini_path = "/root/tmp/qiu_tmp/profile_kw.ini"
def config_read_ini(db_name):
config=configparser.ConfigParser()
config.read(ini_path)
db_host_o=config.get(db_name,"HOST")
db_port_o=config.get(db_name,"PORT")
db_user_o=config.get(db_name,"USERNAME")
db_pass_o=config.get(db_name,"PASSWORD")
db_base_o=config.get(db_name,"DB")
db_ini = [db_host_o,db_port_o,db_user_o,db_pass_o,db_base_o]
return db_ini
# impala host_port_user_password read
impala_ini = config_read_ini("IMPALA")
# read meta list
def read_meta_lst(db_nam):
meta_sql = f'''select id,table_nam
from bi_ods.dask_bi_meta
where run_flag = 1
and table_nam like '{db_nam}%'
order by id asc ; '''
cursor = connect(host=impala_ini[0],port=impala_ini[1],user=impala_ini[2],password=impala_ini[3],database=impala_ini[4]).cursor()
cursor.execute(meta_sql)
meta_lst = cursor.fetchall()
print(meta_lst)
meta_lst = [i[1] for i in meta_lst]
cursor.close()
print(meta_lst)
return meta_lst
dws_meta_lst = read_meta_lst('bi_data_dws')
ads_meta_lst = read_meta_lst('bi_ads_ads')
default_args = {
'owner': 'david.qiu',
'depends_on_past': False,
'start_date': pendulum.datetime(2022, 12, 6, 22, tz="Asia/Shanghai"),
'email': ['qiuswh@126.com'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5)
}
#-------------------------------------------------------------------------------
with DAG(
dag_id="kw_dws_dag",
default_args=default_args,
description='kw_dws_dag',
schedule='11 4 * * *'
) as dag:
kw_dwd_dag_check=ExternalTaskSensor(
task_id="kw_dwd_dag_check",
external_dag_id='kw_dwd_dag',
external_task_id=None,
allowed_states=['success'],
execution_delta=timedelta(minutes=10)
)
start_task = EmptyOperator(task_id="start")
mid_task = EmptyOperator(task_id="mid_task" )
end_task = EmptyOperator(task_id="end" )
kw_dwd_dag_check >> start_task
for i in range(len(dws_meta_lst)):
dws_meta_lst[i] = BashOperator(
task_id=dws_meta_lst[i],
depends_on_past=False,
bash_command=f'''ssh root@100.0.0.1 "impala-shell -f /root/bi/bi_data/{dws_meta_lst[i]} ;" '''
)
start_task >> dws_meta_lst[i] >> mid_task
for j in range(len(ads_meta_lst)):
ads_meta_lst[j] = BashOperator(
task_id=ads_meta_lst[j],
depends_on_past=False,
bash_command=f'''ssh root@100.0.0.1 "impala-shell -f /root/bi/bi_ads/{ads_meta_lst[j]} ;" '''
)
mid_task >> ads_meta_lst[j] >> end_task
dws_meta_lst[0] >> dws_meta_lst[2]
ads_meta_lst[3] >> ads_meta_lst[0]
ads_meta_lst[4] >> ads_meta_lst[1]