oozie5.1 编译安装

# 解压缩

tar xzvf oozie-5.1.0.tar.gz

# 依赖工具安装（已安装，请忽略）

sudo yum install maven

# 编译（非常耗时）

可在我的上传资源中下载编译结果oozie-5.1.0-distro.tar.gz

https://download.csdn.net/download/zphyy1988/11095148

https://download.csdn.net/download/zphyy1988/11095150

https://download.csdn.net/download/zphyy1988/11095154

bin/mkdistro.sh -DskipTests -Puber

# 编译成功结果

/opt/bigdata/oozie-5.1.0/distro/target/oozie-5.1.0-distro.tar.gz

# 拷贝至目录/opt/bigdata并解压缩

tar zxvf oozie-5.1.0-distro.tar.gz

# 配置

## mysql

create database oozie;

grant all privileges on oozie.* to 'root'@'%' identified by '123456';

FLUSH PRIVILEGES;

## cd $OOZIE_HOME

cd /opt/bigdata/oozie-5.1.0

## 创建目录libext

mkdir libext

## 将下载的ext-2.2.zip(http://archive.cloudera.com/gplextras/misc/ext-2.2.zip)拷贝到libext目录里

cp ext-2.2.zip libext

## 配置oozie-site.xml

cd /opt/bigdata/oozie-5.1.0/conf

# spark on yarn

mkdir spark-conf

vi spark-defaults.conf

# spark on yarn log

spark.yarn.historyServer   http://xx:18080

spark.eventLog.dir        hdfs://xx:9000/spark/historyLog

spark.eventLog.enabled    true

# oozie

vi oozie-site.xml

<property>

   <name>oozie.service.ProxyUserService.proxyuser.root.hosts</name>

   <value>*</value>

</property>

<property>

<name>oozie.service.ProxyUserService.proxyuser.root.groups</name>

  <value>*</value>

</property>

<property>

 <name>oozie.service.JPAService.create.db.schema</name>

 <value>false</value>

</property>

<property>

<name>oozie.service.JPAService.jdbc.driver</name>

<value>com.mysql.jdbc.Driver</value>

</property>

<property>

        <name>oozie.service.JPAService.jdbc.url</name>

        <value>jdbc:mysql://master:3306/oozie?createDatabaseIfNotExist=true</value>

</property>

<property>

        <name>oozie.service.JPAService.jdbc.username</name>

        <value>root</value>

</property>

<property>

        <name>oozie.service.JPAService.jdbc.password</name>

        <value>password</value>

</property>

<property>

        <name>oozie.service.HadoopAccessorService.hadoop.configurations</name>

        <value>*=/opt/bigdata/hadoop-2.7.7/etc/hadoop</value>

 </property>

<property>

        <name>oozie.service.HadoopAccessorService.action.configurations</name>

        <value>*=/opt/bigdata/hadoop-2.7.7/etc/hadoop</value>

</property>

<property>

         <name>oozie.service.SparkConfigurationService.spark.configurations</name>

         <value>*=spark-conf</value>

</property>

<property>

         <name>oozie.service.WorkflowAppService.system.libpath</name>

         <value>/user/root/share/lib</value>

</property>

<property>

        <name>oozie.use.system.libpath</name>

        <value>true</value>

</property>

<property>

        <name>oozie.subworkflow.classpath.inheritance</name>

        <value>true</value>

</property>

## 环境准备

bin/oozie-setup.sh

bin/oozie-setup.sh sharelib create -fs hdfs://master:9000 -locallib oozie-sharelib-5.1.0.tar.gz

cp mysql-connector-java-8.0.15.jar lib

## 创建元数据

bin/ooziedb.sh create -sqlfile oozie.sql -run

# 配置hadoop所有节点

## hadoop core-site.xml

<!-- OOZIE -->

  <property>

    <name>hadoop.proxyuser.[OOZIE_SERVER_USER].hosts</name>

    <value>[OOZIE_SERVER_HOSTNAME]</value>

  </property>

  <property>

    <name>hadoop.proxyuser.[OOZIE_SERVER_USER].groups</name>

    <value>[USER_GROUPS_THAT_ALLOW_IMPERSONATION]</value>

  </property>

ps: 用特定的值替换大写字母部分，然后重启Hadoop

<!-- OOZIE -->

<property>

<name>hadoop.proxyuser.root.hosts</name>

<value>master</value>

</property>

<property>

<name>hadoop.proxyuser.root.groups</name>

<value>*</value>

</property>

 

## 启动

bin/oozied.sh start

## 检查

http://master:11000/oozie/

 

# 例子1：spark

## 解压缩

tar zxvf oozie-examples.tar.gz

## 创建hdfs目录

bin/hadoop dfs -mkdir /user/root/examples

## 修改job.properties、workflow.xml

vi job.properties

nameNode=hdfs://master:9000

resourceManager=cluster1

master=yarn

vi workflow.xml

## 上传至hdfs

bin/hadoop dfs -put /opt/bigdata/oozie-5.1.0/examples/* /user/root/examples

## 运行

bin/oozie job -oozie http://master:11000/oozie -config examples/apps/spark/job.properties -run

## 检查状态

bin/oozie job –oozie http://oozie-host:11000/oozie -info ${jobid}

 

## 一个任务执行中，其他任务accept，需要优化capacity-scheduler.xml，详情见 《spark 调优》第9条