python中标准库模板operator_使用PythonOperator的模板文件的气流

获取 BashOperator 或 SqlOperator 为其模板获取外部文件的方法有点清楚地记录在案，但是看看我对文档中的理解的测试不起作用 . 我不确定 templates_exts 和 templates_dict 参数如何正确交互以获取文件 .

在我的dags文件夹中，我创建了： pyoptemplate.sql 和 pyoptemplate.t 以及 test_python_operator_template.py ：

pyoptemplate.sql：

SELECT * FROM {{params.table}};

pyoptemplate.t：

SELECT * FROM {{params.table}};

test_python_operator_template.py：

# coding: utf-8

# vim:ai:si:et:sw=4 ts=4 tw=80

"""

# A Test of Templates in PythonOperator

"""

from airflow import DAG

from airflow.operators.python_operator import PythonOperator

from datetime import datetime

import pprint

pp = pprint.PrettyPrinter(indent=4)

def templated_function(ds, **kwargs):

"""This function will try to use templates loaded from external files"""

pp.pprint(ds)

pp.pprint(kwargs)

# Define the DAG

dag = DAG(dag_id='test_python_operator_template_dag',

default_args={"owner": "lamblin",

"start_date": datetime.now()},

template_searchpath=['/Users/daniellamblin/airflow/dags'],

schedule_interval='@once')

# Define the single task in this controller example DAG

op = PythonOperator(task_id='test_python_operator_template',

provide_context=True,

python_callable=templated_function,

templates_dict={

'pyoptemplate': '',

'pyoptemplate.sql': '',

'sql': 'pyoptemplate',

'file1':'pyoptemplate.sql',

'file2':'pyoptemplate.t',

'table': '{{params.table}}'},

templates_exts=['.sql','.t'],

params={'condition_param': True,

'message': 'Hello World',

'table': 'TEMP_TABLE'},

dag=dag)

运行的结果显示 table 被正确模板化为字符串，但其他人没有提取任何文件进行模板化 .

dlamblin$ airflow test test_python_operator_template_dag test_python_operator_template 2017-01-18

[2017-01-18 23:58:06,698] {__init__.py:36} INFO - Using executor SequentialExecutor

[2017-01-18 23:58:07,342] {models.py:154} INFO - Filling up the DagBag from /Users/daniellamblin/airflow/dags

[2017-01-18 23:58:07,620] {models.py:1196} INFO -

--------------------------------------------------------------------------------

Starting attempt 1 of 1

--------------------------------------------------------------------------------

[2017-01-18 23:58:07,620] {models.py:1219} INFO - Executing on 2017-01-18 00:00:00

'2017-01-18'

{ u'END_DATE': '2017-01-18',

u'conf': ,

u'dag': ,

u'dag_run': None,

u'ds_nodash': u'20170118',

u'end_date': '2017-01-18',

u'execution_date': datetime.datetime(2017, 1, 18, 0, 0),

u'latest_date': '2017-01-18',

u'macros': ,

u'params': { 'condition_param': True,

'message': 'Hello World',

'table': 'TEMP_TABLE'},

u'run_id': None,

u'tables': None,

u'task': ,

u'task_instance': ,

u'task_instance_key_str': u'test_python_operator_template_dag__test_python_operator_template__20170118',

'templates_dict': { 'file1': u'pyoptemplate.sql',

'file2': u'pyoptemplate.t',

'pyoptemplate': u'',

'pyoptemplate.sql': u'',

'sql': u'pyoptemplate',

'table': u'TEMP_TABLE'},

u'test_mode': True,

u'ti': ,

u'tomorrow_ds': '2017-01-19',

u'tomorrow_ds_nodash': u'20170119',

u'ts': '2017-01-18T00:00:00',

u'ts_nodash': u'20170118T000000',

u'yesterday_ds': '2017-01-17',

u'yesterday_ds_nodash': u'20170117'}

[2017-01-18 23:58:07,634] {python_operator.py:67} INFO - Done. Returned value was: None