hadoop学习工作总结（二）之hive流程

最新推荐文章于 2022-04-18 00:05:01 发布

xiaozhuangfeng

最新推荐文章于 2022-04-18 00:05:01 发布

阅读量654

点赞数

分类专栏： hadoop 文章标签： hadoop hive

本文链接：https://blog.csdn.net/xiaozhuangfeng/article/details/39006139

版权

hadoop 专栏收录该内容

18 篇文章 0 订阅

订阅专栏

coordinator.xml

<coordinator-app name="gbd-360-jkdm-oracle-cd"
frequency="${coord:months(1)}" start="${job_start}" end="${job_end}"
timezone="GMT+08:00" xmlns="uri:oozie:coordinator:0.2">
<controls>
     <timeout>2</timeout>
  <concurrency>1</concurrency>
</controls>
<datasets>
  <dataset name="bankLogDataset" frequency="${coord:months(1)}"
   initial-instance="${datesets_initial}" timezone="GMT+08:00">
   <uri-template>${hdfs_sqoop_tmp}/${YEAR}${MONTH}${DAY}</uri-template>
  </dataset>
</datasets>

<output-events>
      <data-out name="logData" dataset="bankLogDataset">
          <instance>${coord:current(-1)}</instance>
      </data-out>
   </output-events>

    <action>
        <workflow>
            <app-path>${application_path}</app-path>
            <configuration>
                <property>
                    <name>sqoop_import_path</name>
                    <value>${coord:dataOut('logData')}</value>
                </property>
                <property>
                    <name>nominalformateDate</name>
                    <value>${coord:formatTime(coord:dateOffset(coord:nominalTime(), -1, 'MONTH'), "yyyyMMdd")}</value>
                </property>
            </configuration>
        </workflow>
    </action>
</coordinator-app>

gbd-360-jkdm-oracle.properties

###################ods import properties file###################
#sqoop import config
hdfs_sqoop_tmp=${namenode_address}/apps-data/hduser0522/gbd_360_safe/gbd-360-jkdm-oracle
hdfs_address_prefix=${namenode_address}/apps/hduser0522

#coordinator config
job_start=2014-08-24T02:00+0800
job_end=2099-12-30T02:00+0800
datesets_initial=2000-01-01T02:00+0800

#shell
seq1=seq1.sh
seq2=seq2.sh
seq3=seq3.sh
seq4=seq4.sh
seq5=seq5.sh

##oozie config
#use system library
oozie.use.system.libpath=true
application_path=${hdfs_address_prefix}/gbd_360_safe/gbd-360-jkdm-oracle

#oozie.bundle.application.path=${application_path}
oozie.coord.application.path=${application_path}

removeJob.sh

#!/bin/sh

oozie -Dheader:j_username=$j_username -Dheader:j_password=$j_password job --oozie $oozie_url -auth ldap -kill $1 -doas $doas_user;

runJob.sh

#!/bin/sh

j_username=`grep j_username /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`
j_password=`grep j_password /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`
oozie_url=`grep oozie_url /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`
doas_user=`grep doas_user /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`
namenode_address=`grep namenode_address /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`
jobtracker_address=`grep jobtracker_address /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`
mapred_job_queue_name=`grep mapred_job_queue_name /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`
jk_connection=`grep jk_connection /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`
jk_username=`grep jk_username /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`
jk_password=`grep jk_password /appcom/apps/hduser0522/gbd_360_safe/server.env|awk 'BEGIN{FS=":::::"}{print $NF}'`

hadoop fs -rmr /apps-data/hduser0522/gbd_360_safe/gbd-360-jkdm-oracle;
hadoop fs -rmr /apps/hduser0522/gbd_360_safe/gbd-360-jkdm-oracle;
hadoop fs -put /appcom/apps/hduser0522/gbd_360_safe/gbd-360-jkdm-oracle/ /apps/hduser0522/gbd_360_safe/gbd-360-jkdm-oracle/;

oozie -Dheader:j_username=$j_username -Dheader:j_password=$j_password job -Dnamenode_address=$namenode_address -Djobtracker_address=$jobtracker_address -Dmapred_job_queue_name=$mapred_job_queue_name -Djk_connection=$jk_connection -Djk_username=$jk_username -Djk_password=$jk_password --oozie $oozie_url -config /appcom/apps/hduser0522/gbd_360_safe/gbd-360-jkdm-oracle/gbd-360-jkdm-oracle.properties -auth ldap -run -doas $doas_user

workflow.xml

<workflow-app xmlns="uri:oozie:workflow:0.4" name="gbd-360-jkdm-oracle-wf">
<start to="pre_clean_path_node"/>

     <action name="seq1">
         <shell xmlns="uri:oozie:shell-action:0.2">
             <job-tracker>${jobtracker_address}</job-tracker>
             <name-node>${namenode_address}</name-node>
             <configuration>
                 <property>
                   <name>mapred.job.queue.name</name>
                   <value>${mapred_job_queue_name}</value>
                 </property>
             </configuration>
             <exec>${seq1}</exec>
             <argument>${nominalformateDate}</argument>
             <argument>${mapred_job_queue_name}</argument>
              <argument>${jk_connection}</argument>
              <argument>${jk_username}</argument>
              <argument>${jk_password}</argument>

             <file>shell/${seq1}#${seq1}</file>
         </shell>
         <ok to="process_import_forking2"/>
         <error to="fail"/>
    </action>

     <fork name="process_import_forking2">
        <path start="seq2"/>
        <path start="seq3"/>
        <path start="seq4"/>
        <path start="seq5"/>
     </fork>

     <action name="seq2">
         <shell xmlns="uri:oozie:shell-action:0.2">
             <job-tracker>${jobtracker_address}</job-tracker>
             <name-node>${namenode_address}</name-node>
             <configuration>
                 <property>
                   <name>mapred.job.queue.name</name>
                   <value>${mapred_job_queue_name}</value>
                 </property>
             </configuration>
             <exec>${seq2}</exec>
             <argument>${nominalformateDate}</argument>
             <argument>${mapred_job_queue_name}</argument>
              <argument>${jk_connection}</argument>
              <argument>${jk_username}</argument>
              <argument>${jk_password}</argument>

             <file>shell/${seq2}#${seq2}</file>
         </shell>
          <ok to="process_import_joining"/>
          <error to="fail"/>
    </action>

.......

    <kill name="fail">
        <message>gbd import wf failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <end name="end"/>
</workflow-app>