Airflow调度程序开发

一叶知秋--

已于 2022-09-09 14:56:54 修改

阅读量268

点赞数 1

文章标签：大数据

于 2022-09-09 14:53:59 首次发布

本文链接：https://blog.csdn.net/qq_41420150/article/details/125760869

版权

需求：外界传入参数，使得Airflow的调度程序将参数值带入至相关执行程序中执行。

程序代码：

from datetime import timedelta,datetime
from airflow.operators.python_operator import PythonOperator
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
import pendulum
local_tz = pendulum.timezone("Asia/Shanghai")

default_args = {
    'owner': 'Test',
    'start_date': datetime(2022,7,3, tzinfo=local_tz)
}
with DAG(
    'customer_sale_order',
    default_args=default_args,
    description='customer_sale_order',
    schedule_interval='10 8 * * *',
    tags=['customer_sale_order']
) as dag:
    def dateStart(ds, **kwargs):       #根据前端DAG页面获取date_start值
        try:
            if len(kwargs.items()) == 0:
                return ''
            else:
                return kwargs['dag_run'].conf['date_start']
        except Exception:
            return ''
            pass


    t_get_data = PythonOperator(      #task执行dateStart function获取到值
        task_id='dateStart',
        provide_context=True,
        python_callable=dateStart,
        dag=dag)
    t0 = BashOperator(
        task_id='check_dependency',
        bash_command='/data/app/airflow/dag/script/customer_sale_order/customer_sale_order_check_dependency.sh ',
        dag=dag
    )
    t1 = BashOperator(
        task_id='customer_sale_order',
        bash_command='/data/app/airflow/dag/script/customer_sale_order/customer_sale_order.sh ',
        dag=dag
    )
    t2 = BashOperator(
        task_id='update_end_time',
        bash_command='/data/app/airflow/dag/script/customer_sale_order/customer_sale_order_update_end_time.sh ',
        dag=dag
    )

    t_replace = BashOperator(      #将前端所获取值传入到后端shell脚本完成脚本内容替换
        task_id='replace',
        bash_command="/data/app/airflow/dag/script/customer_sale_order/replace.sh {{ task_instance.xcom_pull(task_ids='dateStart') }} ",
        dag=dag,
    )
    t_restore = BashOperator(
        task_id='restore',
        bash_command="/data/app/airflow/dag/script/customer_sale_order/restore.sh {{ task_instance.xcom_pull(task_ids='dateStart') }} ",
        dag=dag
    )
    t_get_data>>t_replace>> t0 >> t1 >> t2 >>t_restore

reliace.sh

if ["$1" = ""]
then
echo '参数为空' >> /data/app/airflow/dag/script/1.txt
else
echo $1 >> /data/app/airflow/dag/script/1.txt
sed -i "s/now()/toDate('12345678900987654321')/g" /data/app/airflow/dag/sql/customer_sale_order/*
sed -i "s/today()/toDate('12345678900987654321')/g" /data/app/airflow/dag/sql/customer_sale_order/*
sed -i "s/12345678900987654321/$1/g" /data/app/airflow/dag/sql/customer_sale_order/*
sed -i "s/now()/toDate('12345678900987654321')/g" /data/check_dependency/customer_sale_order/*
sed -i "s/today()/toDate('12345678900987654321')/g" /data/check_dependency/customer_sale_order/*
sed -i "s/12345678900987654321/$1/g" /data/check_dependency/customer_sale_order/*