airflow添加新任务

配置执行命令

path:/home/airflow/dsp-airflow/dsp_airflow/conf/dsp_airflow.cfg

#anti-fraud
dir_anti_fraud = %(project_dir)santi-fraud
script_anti_fraud = run_daily.sh

修改调度文件

path:/home/airflow/dsp-airflow/dsp_airflow/dags/dsp_daily_production.py

1
dir_dsp_simple = get_dsp_config(section=command_tag, option='dir_dsp_simple')
script_dsp_simple = get_dsp_config(section=command_tag, option='script_dsp_simple')
dir_anti_fraud = get_dsp_config(section=command_tag, option='dir_anti_fraud')
script_anti_fraud = get_dsp_config(section=command_tag, option='script_anti_fraud')


2
local_dsp_simple_daily_taskid = "local_dsp_simple_daily"
local_anti_fraud_daily_taskid = "local_anti_fraud_daily"


3
local_dsp_simple_daily = gen_report_task(parent_dag=dag, task_id_str=local_dsp_simple_daily_taskid,
                                         command=command_template.format(username=LINUX_USER,
                                                                         dir=dir_dsp_simple,
                                                                         script=script_dsp_simple,
                                                                         paras=" -%sd %s" % (test_option,
                                                                                             day_interval_between_execution_and_now))
                                         )
local_anti_fraud_daily = gen_report_task(parent_dag=dag, task_id_str=local_anti_fraud_daily_taskid,
                                         command=command_template.format(username=LINUX_USER,
                                                                         dir=dir_anti_fraud,
                                                                         script=script_anti_fraud,
                                                                         paras="{{ ds }}")
                                         )


4
# local dsp simple daily
local_dsp_simple_daily.set_upstream(is_local_dsp_simple_hourly_all_ready)
local_dsp_simple_daily.set_downstream(end_task)
# local anti fraud daily
local_anti_fraud_daily.set_upstream([is_local_dsp_info_hourly_all_ready, is_local_dsp_simple_hourly_all_ready])
local_anti_fraud_daily.set_downstream(end_task)