制作Livy的Parcel包和csd文件

1. 配置java，maven等环境变量

   java:

   export JAVA_HOME=/usr/local/java
   export PATH=$JAVA_HOME/bin:$PATH
   export CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tool.jar:$CLASSPATH
   

   maven:

   export MVN_HOME=/usr/local/maven
   export PATH=$MVN_HOME/bin:$PATH
   

2. 安装及配置R和Python环境变量
   Centos7 配置R环境

3. 下载cm_ext

   Cloudera提供的cm_ext工具,对生成的csd和parcel进行校验

   [root@node01 ~]# mkdir -p ~/github/cloudera
   [root@node01 ~]# cd ~/github/cloudera
   [root@node01 ~]# git clone https://github.com/cloudera/cm_ext.git
   [root@node01 ~]# cd cm_ext
   [root@node01 ~]# mvn package
   

4. 制作Livy Spark Server 的Parcel包并校验

   ./build_parcel.sh <Version> <Distro>
   

   Distro 参数：
   https://github.com/cloudera/cm_ext/wiki/Parcel-distro-suffixes

   Distro	系统版本	备注
   el5	Redhat Enterprise Linux 5 and clones	CentOS, Scientific Linux, etc
   el6	Redhat Enterprise Linux 6 and clones	CentOS, Scientific Linux, etc
   el7	Redhat Enterprise Linux 7 and clones	CentOS, Scientific Linux, etc
   sles11	SuSE Linux Enterprise Server 11.x
   lucid	Ubuntu Linux 10.04 LTS	No CDH 5.x parcel provided
   precise	Ubuntu Linux 12.04 LTS
   trusty	Ubuntu Linux 14.04 LTS	Newly supported in CM 5.2. No CDH 4.x parcel provided
   squeeze	Debian 6.x	No CDH 5.x parcel provided
   wheezy	Debian 7.x	Newly supported in CM 5.0. No CDH 4.x parcel provided

   [root@node01 ~]# lsb_release -a                #查看版本
   LSB Version:	:core-4.1-amd64:core-4.1-noarch
   Distributor ID:	CentOS
   Description:	CentOS Linux release 7.6.1810 (Core) 
   Release:	7.6.1810
   Codename:	Core
   [root@node01 ~]# git clone https://github.com/Automattic/cm-livy-scripts.git
   [root@node01 ~]# cd cm-livy-scripts
   [root@node01 ~]# ./build_parcel.sh 1.0 el7
   

   日志

   ...
   [INFO] ------------------------------------------------------------------------
   [INFO] Reactor Summary:
   [INFO] 
   [INFO] livy-main .......................................... SUCCESS [  2.289 s]
   [INFO] livy-api ........................................... SUCCESS [  6.053 s]
   [INFO] livy-client-common ................................. SUCCESS [  1.820 s]
   [INFO] livy-test-lib ...................................... SUCCESS [  4.647 s]
   [INFO] livy-rsc ........................................... SUCCESS [  5.001 s]
   [INFO] livy-core .......................................... SUCCESS [  4.732 s]
   [INFO] livy-repl .......................................... SUCCESS [ 16.335 s]
   [INFO] livy-server ........................................ SUCCESS [ 18.624 s]
   [INFO] livy-assembly ...................................... SUCCESS [  1.961 s]
   [INFO] livy-client-http ................................... SUCCESS [  8.189 s]
   [INFO] livy-integration-test .............................. SUCCESS [ 11.908 s]
   [INFO] ------------------------------------------------------------------------
   [INFO] BUILD SUCCESS                                   #打包
   [INFO] ------------------------------------------------------------------------
   [INFO] Total time: 01:21 min
   [INFO] Finished at: 2019-04-09T17:28:51+08:00
   [INFO] Final Memory: 87M/1675M
   [INFO] ------------------------------------------------------------------------
   ...
   + java -jar /root/github/cloudera/cm_ext/validator/target/validator.jar -d ./LIVY-1.0
   Validating: ./LIVY-1.0
   Validating: ./LIVY-1.0/meta/parcel.json
   Validation succeeded.                                   #校验
   ...
   + java -jar /root/github/cloudera/cm_ext/validator/target/validator.jar -f ./LIVY-1.0-el7.parcel
   Validating: ./LIVY-1.0-el7.parcel
   Validating: LIVY-1.0/meta/parcel.json
   Validation succeeded.                                  #校验
   + rm -rf ./LIVY-1.0
   + /root/github/cloudera/cm_ext/make_manifest/make_manifest.py .
   Scanning directory: .
   Found parcel LIVY-1.0-el7.parcel
   

5. 制作Livy Spark Server 的CSD文件

   [root@node01 cm-livy-scripts]# ./build_csd.sh 1.0
   + JARNAME=LIVY-1.0.jar
   + java -jar /root/github/cloudera/cm_ext/validator/target/validator.jar -s ./csd-src/descriptor/service.sdl
   Validating: ./csd-src/descriptor/service.sdl
   Validation succeeded.
   + jar -cvf ./LIVY-1.0.jar -C ./csd-src .
   added manifest
   adding: scripts/(in = 0) (out= 0)(stored 0%)
   adding: scripts/control.sh(in = 524) (out= 309)(deflated 41%)
   adding: aux/(in = 0) (out= 0)(stored 0%)
   adding: aux/spark-user-configurable-options.conf(in = 1957) (out= 665)(deflated 66%)
   adding: descriptor/(in = 0) (out= 0)(stored 0%)
   adding: descriptor/service.sdl(in = 5438) (out= 1558)(deflated 71%)
   + echo 'Created LIVY-1.0.jar'
   Created LIVY-1.0.jar
   

6. 查看Parcel包和csd文件

   [root@node01 cm-livy-scripts]# ll
   total 91008
   drwxr-xr-x  6 root root     4096 Apr  9 17:29 ./
   drwxr-xr-x  4 root root     4096 Apr  9 17:25 ../
   -rwxr-xr-x  1 root root      348 Apr  9 17:25 build_csd.sh
   -rwxr-xr-x  1 root root     1271 Apr  9 17:25 build_parcel.sh
   drwxr-xr-x  5 root root     4096 Apr  9 17:25 csd-src/
   drwxr-xr-x  8 root root     4096 Apr  9 17:25 .git/
   -rw-r--r--  1 root root       51 Apr  9 17:25 .gitignore
   -rw-r--r--  1 root root    15199 Apr  9 17:25 LICENSE.md
   drwxr-xr-x 17 root root     4096 Apr  9 17:27 livy/
   -rw-r--r--  1 root root     3584 Apr  9 17:41 LIVY-1.0.jar
   -rw-r--r--  1 root root 93121920 Apr  9 17:28 LIVY-1.0-el7.parcel
   -rw-r--r--  1 root root      313 Apr  9 17:29 manifest.json
   drwxr-xr-x  3 root root     4096 Apr  9 17:25 parcel-src/
   -rw-r--r--  1 root root     2370 Apr  9 17:25 README.md
   -rwxr-xr-x  1 root root       92 Apr  9 17:25 serve_parcel.sh