workflow.xml
<workflow-app xmlns="uri:oozie:workflow:0.4" name="job_ods_a_xdr_ps_gn_dns_wf">
<start to="CHECK_DATA" />
<decision name="CHECK_DATA">
<switch>
<case to="HIVE_PARTITION">
${fs:exists(concat("cache/O_RE_ST_XDR_PS_GN_DNS/",inputtime))}
</case>
<default to="end" />
</switch>
</decision>
<action name="HIVE_PARTITION">
<hive xmlns="uri:oozie:hive-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
<property>
<name>oozie.hive.defaults</name>
<value>my-hive-default.xml</value>
</property>
<property>
<name>hive.metastore.local</name>
<value>false</value>
</property>
<property>
<name>hive.metastore.uris</name>
<value>thrift://GZ-HADOOP37:9083</value>
</property>
<property>
<name>hive.metastore.warehouse.dir</name>
<value>/user/hive/warehouse</value>
</property>
</configuration>
<script>addpartition_script.q</script>
<param>tablename=ODS_RE_ST_XDR_PS_GN_DNS_15MI</param>
<param>day=${day}</param>
<param>hour=${hour}</param>
</hive>
<ok to="ODS_RE_ST_XDR_PS_GN_DNS_15MI" />
<error to="fail" />
</action>
<action name="ODS_RE_ST_XDR_PS_GN_DNS_15MI">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<main-class>com.boco.BSSystem.etl.ods.OdsReStXdrPsGnDns15Mi.OdsReStXdrPsGnDns15MiDriver</main-class>
<arg>cache/O_RE_ST_XDR_PS_GN_DNS/${inputtime}</arg>
<arg>ods/ODS_RE_ST_XDR_PS_GN_DNS_15MI/${wf:id()}/</arg>
<arg>${wf:id()}</arg>
</java>
<ok to="IMAPAL_REFRESH" />
<error to="fail" />
</action>
<action name="IMAPAL_REFRESH">
<shell xmlns="uri:oozie:shell-action:0.1">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<exec>impala-shell</exec>
<argument>-q</argument>
<argument>refresh ODS_RE_ST_XDR_PS_GN_DNS_15MI</argument>
<capture-output />
</shell>
<ok to="LAST_15MI" />
<error to="fail" />
</action>
<decision name="LAST_15MI">
<switch>
<case to="ODS_A_XDR_PS_GN_DNS_H">
${mi eq 45 }
</case>
<default to="CHECK_ODS" />
</switch>
</decision>
<action name="ODS_A_XDR_PS_GN_DNS_H">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<main-class>com.boco.BSSystem.impala.ImpalaCollect</main-class>
<arg>ODS_A_XDR_PS_GN_DNS_H</arg>
<arg>${day}</arg>
<arg>${hour}</arg>
</java>
<ok to="DW_F_XDR_PS_GN_DNS_SUBAPP_H" />
<error to="fail" />
</action>
<action name="DW_F_XDR_PS_GN_DNS_SUBAPP_H">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<main-class>com.boco.BSSystem.impala.ImpalaCollect</main-class>
<arg>DW_F_XDR_PS_GN_DNS_SUBAPP_H</arg>
<arg>${day}</arg>
<arg>${hour}</arg>
</java>
<ok to="DW_F_XDR_PS_GN_DNS_CELL_H" />
<error to="fail" />
</action>
<action name="DW_F_XDR_PS_GN_DNS_CELL_H">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<main-class>com.boco.BSSystem.impala.ImpalaCollect</main-class>
<arg>DW_F_XDR_PS_GN_DNS_CELL_H</arg>
<arg>${day}</arg>
<arg>${hour}</arg>
</java>
<ok to="ODS_RE_ST_XDR_PS_CAUSE_DNS_H" />
<error to="fail" />
</action>
<action name="ODS_RE_ST_XDR_PS_CAUSE_DNS_H">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<main-class>com.boco.BSSystem.impala.ImpalaCollect</main-class>
<arg>ODS_RE_ST_XDR_PS_CAUSE_DNS_H</arg>
<arg>${day}</arg>
<arg>${hour}</arg>
</java>
<ok to="DEL_ODS_RE_ST_XDR_PS_CAUSE_DNS_H" />
<error to="fail" />
</action>
<action name="DEL_ODS_RE_ST_XDR_PS_CAUSE_DNS_H">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<main-class>com.boco.BSSystem.utils.TruncateTable</main-class>
<arg>ODS_RE_ST_XDR_PS_CAUSE_DNS_H</arg>
<arg>${deltime}</arg>
<arg>${deltime}</arg>
</java>
<ok to="DEL_DW_F_XDR_PS_GN_DNS_CELL_H" />
<error to="fail" />
</action>
<action name="DEL_DW_F_XDR_PS_GN_DNS_CELL_H">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<main-class>com.boco.BSSystem.utils.TruncateTable</main-class>
<arg>DW_F_XDR_PS_GN_DNS_CELL_H</arg>
<arg>${deltime}</arg>
<arg>${deltime}</arg>
</java>
<ok to="DEL_DW_F_XDR_PS_GN_DNS_SUBAPP_H" />
<error to="fail" />
</action>
<action name="DEL_DW_F_XDR_PS_GN_DNS_SUBAPP_H">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<main-class>com.boco.BSSystem.utils.TruncateTable</main-class>
<arg>DW_F_XDR_PS_GN_DNS_SUBAPP_H</arg>
<arg>${deltime}</arg>
<arg>${deltime}</arg>
</java>
<ok to="DEL_ODS_A_XDR_PS_GN_DNS_H" />
<error to="fail" />
</action>
<action name="DEL_ODS_A_XDR_PS_GN_DNS_H">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<main-class>com.boco.BSSystem.utils.TruncateTable</main-class>
<arg>ODS_A_XDR_PS_GN_DNS_H</arg>
<arg>${deltime}</arg>
<arg>${deltime}</arg>
</java>
<ok to="fork_sqoop_DW" />
<error to="fail" />
</action>
<fork name="fork_sqoop_DW">
<path start="SQOOP_DW_F_XDR_PS_GN_DNS_SUBAPP_H" />
<path start="SQOOP_DW_F_XDR_PS_GN_DNS_CELL_H" />
<path start="SQOOP_ODS_A_XDR_PS_GN_DNS_H" />
<path start="SQOOP_ODS_RE_ST_XDR_PS_CAUSE_DNS_H" />
</fork>
<action name="SQOOP_ODS_RE_ST_XDR_PS_CAUSE_DNS_H">
<sqoop xmlns="uri:oozie:sqoop-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<arg>export</arg>
<arg>--connect</arg>
<arg>${strUrl}</arg>
<arg>--username</arg>
<arg>${odsdbuser}</arg>
<arg>--password</arg>
<arg>${odsdbpwd}</arg>
<arg>-m</arg>
<arg>2</arg>
<arg>--table</arg>
<arg>ODS_RE_ST_XDR_PS_CAUSE_DNS_H</arg>
<arg>--export-dir</arg>
<arg>impala/ODS_RE_ST_XDR_PS_CAUSE_DNS_H/day=${day}/hour=${hour}/*</arg>
<arg>--input-fields-terminated-by</arg>
<arg>|</arg>
<arg>--columns</arg>
<arg>day_id,hour_id,lac,cid,cell_type,dns_resp_code,dns_req_cnt,resp_cnt</arg>
</sqoop>
<ok to="joining_sqoop_DW" />
<error to="fail" />
</action>
<action name="SQOOP_ODS_A_XDR_PS_GN_DNS_H">
<sqoop xmlns="uri:oozie:sqoop-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<arg>export</arg>
<arg>--connect</arg>
<arg>${strUrl}</arg>
<arg>--username</arg>
<arg>${odsdbuser}</arg>
<arg>--password</arg>
<arg>${odsdbpwd}</arg>
<arg>-m</arg>
<arg>2</arg>
<arg>--table</arg>
<arg>ODS_A_XDR_PS_GN_DNS_H</arg>
<arg>--export-dir</arg>
<arg>impala/ODS_A_XDR_PS_GN_DNS_H/day=${day}/hour=${hour}/*</arg>
<arg>--input-fields-terminated-by</arg>
<arg>|</arg>
<arg>--columns</arg>
<arg>day_id,hour_id,u_province_desc,u_region_desc,bscrnc_desc,rat,lac,cid,cell_type,app_type,app_sub_type,ul_data,dl_data,ul_ip_packet,dl_ip_packet,ul_tcp_discordnum,dl_tcp_discordnum,ul_tcp_renum,dl_tcp_renum,ul_ip_frag_packets,dl_ip_frag_packets,dns_req_cnt,response_cnt,auth_content_cnt,addi_content_cnt,timedelay,req_succ_num</arg>
</sqoop>
<ok to="joining_sqoop_DW" />
<error to="fail" />
</action>
<action name="SQOOP_DW_F_XDR_PS_GN_DNS_SUBAPP_H">
<sqoop xmlns="uri:oozie:sqoop-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<arg>export</arg>
<arg>--connect</arg>
<arg>${strUrl}</arg>
<arg>--username</arg>
<arg>${dwdbuser}</arg>
<arg>--password</arg>
<arg>${dwdbpwd}</arg>
<arg>-m</arg>
<arg>2</arg>
<arg>--table</arg>
<arg>DW_F_XDR_PS_GN_DNS_SUBAPP_H</arg>
<arg>--export-dir</arg>
<arg>impala/DW_F_XDR_PS_GN_DNS_SUBAPP_H/day=${day}/hour=${hour}/*</arg>
<arg>--input-fields-terminated-by</arg>
<arg>|</arg>
<arg>--columns</arg>
<arg>day_id,hour_id,region_id,region_desc,net_type,app_type,app_sub_type,ul_data,dl_data,ul_ip_packet,dl_ip_packet,ul_tcp_discordnum,dl_tcp_discordnum,ul_tcp_renum,dl_tcp_renum,ul_ip_frag_packets,dl_ip_frag_packets,dns_req_cnt,response_cnt,auth_content_cnt,addi_content_cnt,timedelay,req_succ_num</arg>
</sqoop>
<ok to="joining_sqoop_DW" />
<error to="fail" />
</action>
<action name="SQOOP_DW_F_XDR_PS_GN_DNS_CELL_H">
<sqoop xmlns="uri:oozie:sqoop-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<arg>export</arg>
<arg>--connect</arg>
<arg>${strUrl}</arg>
<arg>--username</arg>
<arg>${dwdbuser}</arg>
<arg>--password</arg>
<arg>${dwdbpwd}</arg>
<arg>-m</arg>
<arg>2</arg>
<arg>--table</arg>
<arg>DW_F_XDR_PS_GN_DNS_CELL_H</arg>
<arg>--export-dir</arg>
<arg>impala/DW_F_XDR_PS_GN_DNS_CELL_H/day=${day}/hour=${hour}/*</arg>
<arg>--input-fields-terminated-by</arg>
<arg>|</arg>
<arg>--columns</arg>
<arg>day_id,hour_id,region_id,region_desc,net_type,bscrnc_id,bscrnc_desc,lac,cid,cell_id,ul_data,dl_data,ul_ip_packet,dl_ip_packet,ul_tcp_discordnum,dl_tcp_discordnum,ul_tcp_renum,dl_tcp_renum,ul_ip_frag_packets,dl_ip_frag_packets,dns_req_cnt,response_cnt,auth_content_cnt,addi_content_cnt,timedelay,req_succ_num</arg>
</sqoop>
<ok to="joining_sqoop_DW" />
<error to="fail" />
</action>
<join name="joining_sqoop_DW" to="CHECK_ODS" />
<decision name="CHECK_ODS">
<switch>
<case to="del_ods_node">
${fs:exists(concat("ods/ODS_RE_ST_XDR_PS_GN_DNS_15MI/", wf:id()))}
</case>
<default to="end" />
</switch>
</decision>
<action name="del_ods_node">
<fs>
<delete path='${nameNode}/user/${wf:user() }/ods/ODS_RE_ST_XDR_PS_GN_DNS_15MI/${wf:id()}/'/>
</fs>
<ok to="end"/>
<error to="fail"/>
</action>
<kill name="fail">
<message>job failed, error message[${wf:errorMessage(wf:lastErrorNode())}] </message>
</kill>
<end name="end" />
</workflow-app>
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
- 113
- 114
- 115
- 116
- 117
- 118
- 119
- 120
- 121
- 122
- 123
- 124
- 125
- 126
- 127
- 128
- 129
- 130
- 131
- 132
- 133
- 134
- 135
- 136
- 137
- 138
- 139
- 140
- 141
- 142
- 143
- 144
- 145
- 146
- 147
- 148
- 149
- 150
- 151
- 152
- 153
- 154
- 155
- 156
- 157
- 158
- 159
- 160
- 161
- 162
- 163
- 164
- 165
- 166
- 167
- 168
- 169
- 170
- 171
- 172
- 173
- 174
- 175
- 176
- 177
- 178
- 179
- 180
- 181
- 182
- 183
- 184
- 185
- 186
- 187
- 188
- 189
- 190
- 191
- 192
- 193
- 194
- 195
- 196
- 197
- 198
- 199
- 200
- 201
- 202
- 203
- 204
- 205
- 206
- 207
- 208
- 209
- 210
- 211
- 212
- 213
- 214
- 215
- 216
- 217
- 218
- 219
- 220
- 221
- 222
- 223
- 224
- 225
- 226
- 227
- 228
- 229
- 230
- 231
- 232
- 233
- 234
- 235
- 236
- 237
- 238
- 239
- 240
- 241
- 242
- 243
- 244
- 245
- 246
- 247
- 248
- 249
- 250
- 251
- 252
- 253
- 254
- 255
- 256
- 257
- 258
- 259
- 260
- 261
- 262
- 263
- 264
- 265
- 266
- 267
- 268
- 269
- 270
- 271
- 272
- 273
- 274
- 275
- 276
- 277
- 278
- 279
- 280
- 281
- 282
- 283
- 284
- 285
- 286
- 287
- 288
- 289
- 290
- 291
- 292
- 293
- 294
- 295
- 296
- 297
- 298
- 299
- 300
- 301
- 302
- 303
- 304
- 305
- 306
- 307
- 308
- 309
- 310
- 311
- 312
- 313
- 314
- 315
- 316
- 317
- 318
- 319
- 320
- 321
- 322
- 323
- 324
- 325
- 326
- 327
- 328
- 329
- 330
- 331
- 332
- 333
- 334
- 335
- 336
- 337
- 338
- 339
- 340
- 341
- 342
- 343
- 344
- 345
- 346
- 347
- 348
- 349
- 350
- 351
- 352
- 353
- 354
- 355
- 356
- 357
- 358
- 359
- 360
- 361
- 362
- 363
- 364
- 365
- 366
- 367
- 368
- 369
- 370
- 371
- 372
- 373
- 374
- 375
- 376
- 377
- 378
- 379
- 380
- 381
- 382
- 383
- 384
- 385
- 386
- 387
- 388
- 389
- 390
- 391
- 392
- 393
- 394
- 395
- 396
- 397
- 398
- 399
- 400
- 401
- 402
- 403
- 404
- 405
- 406
- 407
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
- 113
- 114
- 115
- 116
- 117
- 118
- 119
- 120
- 121
- 122
- 123
- 124
- 125
- 126
- 127
- 128
- 129
- 130
- 131
- 132
- 133
- 134
- 135
- 136
- 137
- 138
- 139
- 140
- 141
- 142
- 143
- 144
- 145
- 146
- 147
- 148
- 149
- 150
- 151
- 152
- 153
- 154
- 155
- 156
- 157
- 158
- 159
- 160
- 161
- 162
- 163
- 164
- 165
- 166
- 167
- 168
- 169
- 170
- 171
- 172
- 173
- 174
- 175
- 176
- 177
- 178
- 179
- 180
- 181
- 182
- 183
- 184
- 185
- 186
- 187
- 188
- 189
- 190
- 191
- 192
- 193
- 194
- 195
- 196
- 197
- 198
- 199
- 200
- 201
- 202
- 203
- 204
- 205
- 206
- 207
- 208
- 209
- 210
- 211
- 212
- 213
- 214
- 215
- 216
- 217
- 218
- 219
- 220
- 221
- 222
- 223
- 224
- 225
- 226
- 227
- 228
- 229
- 230
- 231
- 232
- 233
- 234
- 235
- 236
- 237
- 238
- 239
- 240
- 241
- 242
- 243
- 244
- 245
- 246
- 247
- 248
- 249
- 250
- 251
- 252
- 253
- 254
- 255
- 256
- 257
- 258
- 259
- 260
- 261
- 262
- 263
- 264
- 265
- 266
- 267
- 268
- 269
- 270
- 271
- 272
- 273
- 274
- 275
- 276
- 277
- 278
- 279
- 280
- 281
- 282
- 283
- 284
- 285
- 286
- 287
- 288
- 289
- 290
- 291
- 292
- 293
- 294
- 295
- 296
- 297
- 298
- 299
- 300
- 301
- 302
- 303
- 304
- 305
- 306
- 307
- 308
- 309
- 310
- 311
- 312
- 313
- 314
- 315
- 316
- 317
- 318
- 319
- 320
- 321
- 322
- 323
- 324
- 325
- 326
- 327
- 328
- 329
- 330
- 331
- 332
- 333
- 334
- 335
- 336
- 337
- 338
- 339
- 340
- 341
- 342
- 343
- 344
- 345
- 346
- 347
- 348
- 349
- 350
- 351
- 352
- 353
- 354
- 355
- 356
- 357
- 358
- 359
- 360
- 361
- 362
- 363
- 364
- 365
- 366
- 367
- 368
- 369
- 370
- 371
- 372
- 373
- 374
- 375
- 376
- 377
- 378
- 379
- 380
- 381
- 382
- 383
- 384
- 385
- 386
- 387
- 388
- 389
- 390
- 391
- 392
- 393
- 394
- 395
- 396
- 397
- 398
- 399
- 400
- 401
- 402
- 403
- 404
- 405
- 406
- 407
addpartition_script.q
alter table ${tablename} add if not exists partition(day='${day}',hour='${hour}') location 'day=${day}/hour=${hour}';
my-hive-default.xml
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<property>
<name>hive.metastore.local</name>
<value>false</value>
</property>
<property>
<name>hive.metastore.uris</name>
<value>thrift://GZ-HADOOP37:9083</value>
</property>
<property>
<name>hive.metastore.warehouse.dir</name>
<value>/user/hive/warehouse</value>
</property>
<property>
<name>hive.warehouse.subdir.inherit.perms</name>
<value>true</value>
</property>
<property>
<name>mapred.reduce.tasks</name>
<value>-1</value>
</property>
<property>
<name>hive.exec.reducers.bytes.per.reducer</name>
<value>1073741824</value>
</property>
<property>
<name>hive.exec.reducers.max</name>
<value>999</value>
</property>
<property>
<name>hive.metastore.execute.setugi</name>
<value>true</value>
</property>
</configuration>