Airflow 常见问题汇总

Airflow 常见问题汇总
1- 中文注释导致dag 不显示

2- task 依赖关系

3- dag 依赖
ExternalTaskSensor

参考以下代码：

# -*- coding: utf-8 -*-
import pendulum 
import airflow
from airflow import DAG
from airflow.decorators import task
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from airflow.operators.empty import EmptyOperator
from airflow.sensors.external_task import ExternalTaskSensor
import pandas as pd
from datetime import timedelta
from impala.dbapi import connect
import configparser
 
#-------------------------------------------------------------------------------
# password meta file 
ini_path =  "/root/tmp/qiu_tmp/profile_kw.ini"

def config_read_ini(db_name):
    
    config=configparser.ConfigParser()
    config.read(ini_path)
    
    db_host_o=config.get(db_name,"HOST")
    db_port_o=config.get(db_name,"PORT")
    db_user_o=config.get(db_name,"USERNAME")
    db_pass_o=config.get(db_name,"PASSWORD")
    db_base_o=config.get(db_name,"DB")
    
    db_ini = [db_host_o,db_port_o,db_user_o,db_pass_o,db_base_o]
    return db_ini

# impala host_port_user_password read 
impala_ini = config_read_ini("IMPALA")

# read meta list
def read_meta_lst(db_nam):
    meta_sql = f'''select id,table_nam 
                    from  bi_ods.dask_bi_meta 
                   where  run_flag = 1 
                     and  table_nam like '{db_nam}%' 
                order by  id asc  ; '''                     
                
    cursor = connect(host=impala_ini[0],port=impala_ini[1],user=impala_ini[2],password=impala_ini[3],database=impala_ini[4]).cursor()
    cursor.execute(meta_sql)
    meta_lst = cursor.fetchall() 
    print(meta_lst)
    meta_lst = [i[1] for i in meta_lst]
    cursor.close()  
    print(meta_lst)    
    return meta_lst

dws_meta_lst = read_meta_lst('bi_data_dws')  
ads_meta_lst = read_meta_lst('bi_ads_ads') 

default_args = {
    'owner': 'david.qiu',
    'depends_on_past': False,
    'start_date': pendulum.datetime(2022, 12, 6, 22, tz="Asia/Shanghai"),
    'email': ['qiuswh@126.com'],
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=5)
}

#-------------------------------------------------------------------------------
with DAG(
    dag_id="kw_dws_dag",
    default_args=default_args,
    description='kw_dws_dag',
    schedule='11 4 * * *' 
) as dag:
    kw_dwd_dag_check=ExternalTaskSensor(
        task_id="kw_dwd_dag_check", 
        external_dag_id='kw_dwd_dag',     
        external_task_id=None, 
        allowed_states=['success'], 
        execution_delta=timedelta(minutes=10)
        )
    
    start_task = EmptyOperator(task_id="start")
    mid_task   = EmptyOperator(task_id="mid_task" )
    end_task   = EmptyOperator(task_id="end" ) 
    
    kw_dwd_dag_check >> start_task
    
    for i in range(len(dws_meta_lst)):
        dws_meta_lst[i] = BashOperator(
            task_id=dws_meta_lst[i],
            depends_on_past=False,
            bash_command=f'''ssh root@100.0.0.1 "impala-shell -f /root/bi/bi_data/{dws_meta_lst[i]} ;" '''  
            )
        start_task >> dws_meta_lst[i] >> mid_task            

    for j in range(len(ads_meta_lst)):
        ads_meta_lst[j] = BashOperator(
            task_id=ads_meta_lst[j],
            depends_on_past=False,
            bash_command=f'''ssh root@100.0.0.1 "impala-shell -f /root/bi/bi_ads/{ads_meta_lst[j]} ;" '''  
            )   
        mid_task >> ads_meta_lst[j] >> end_task
    
    dws_meta_lst[0] >> dws_meta_lst[2] 
    ads_meta_lst[3] >> ads_meta_lst[0] 
    ads_meta_lst[4] >> ads_meta_lst[1]