1、首先创建一个workflow工作流,拖取HiveServer2 和Sub-workflow 控件
HiveServer2控件针对Hsql脚本,Sub-workflow复用Shell sqoop同步脚本
如图:
2、shell sqoop同步脚本
参考1,拖取shell 脚本控件,如图
这里涉及到参数传递以及环境设置
特别是FILE那块的设置
–CDH版本
3、shell脚本如下:
#!/bin/bash
#下面这段参数只针对腾讯云,CDH版本不需要
basepath=$(cd dirname $0
; pwd)
export SQOOP_HOME=
b
a
s
e
p
a
t
h
/
a
p
p
l
i
c
a
t
i
o
n
/
s
q
o
o
p
e
x
p
o
r
t
S
Q
O
O
P
C
O
N
F
D
I
R
=
basepath/application/sqoop export SQOOP_CONF_DIR=
basepath/application/sqoopexportSQOOPCONFDIR=SQOOP_HOME/conf
export PATH=
S
Q
O
O
P
H
O
M
E
/
b
i
n
:
SQOOP_HOME/bin:
SQOOPHOME/bin:PATH
表名
table_name=$1
pd_sum_date=$2
pd_date=$3
sqoop eval
–connect jdbc:mysql://xx.xx.xx.xx:xx/xx
–username xx
–password xx
-e “delete from
t
a
b
l
e
n
a
m
e
w
h
e
r
e
s
u
m
d
a
t
e
=
′
{table_name} where sum_date = '
tablenamewheresumdate=′{pd_sum_date}’”
sqoop export
–connect “jdbc:mysql://xx.xx.xx.xx:xx/xx ?useUnicode=true&characterEncoding=utf-8”
–username xxx
–password xxx
–table KaTeX parse error: Undefined control sequence: \ at position 14: {table_name} \̲ ̲--export-dir /a…{table_name}/dt=${pd_date}
–input-fields-terminated-by ‘\001’
–input-null-string ‘\N’ --input-null-non-string ‘\N’
4、调度设置
Parameters
pd_date ${coord:formatTime(coord:nominalTime(), ‘yyyyMMdd’)}
last_kjqj ${coord:formatTime(coord:dateOffset(coord:nominalTime(), -2, ‘MONTH’), ‘yyyyMM’)}
cur_kjqj ${coord:formatTime(coord:dateOffset(coord:nominalTime(), -1, ‘MONTH’), ‘yyyyMM’)}
pd_sum_date ${coord:formatTime(coord:nominalTime(), ‘yyyy-MM-dd’)}
pm_mon ${coord:formatTime(coord:dateOffset(coord:nominalTime(), -1, ‘MONTH’), ‘yyyyMM’)}
k_kjnd ${coord:formatTime(coord:nominalTime(), ‘yyyy’)}
up_kjqj ${coord:formatTime(coord:dateOffset(coord:nominalTime(), -3, ‘MONTH’), ‘yyyyMM’)}
k_beginofmon ${coord:formatTime(coord:nominalTime(), ‘yyyyMM’)}01