Livy的CDH环境parcel和csd制作

制作parcel和csd需要用到Cloudera cm_ext，所以先用Git下载cm_ext，然后用Maven安装。

这里主要是制作Livy Spark server，用于Spark on Hue，暂时没有用到R所以就先不安装了。

因为使用平台是CDH6.0.1，所以需要修改脚本。

1. 安装支持语言和环境变量

Java、Scala、Python、Maven，并配置好环境变量。

2. 安装Cloudera cm_ext

cm_ext git地址

安装Git下载 cm_ext

yum -y install git

mkdir -p ~/github/cloudera; cd ~/github/cloudera
git clone git@github.com:cloudera/cm_ext.git; cd cm_ext
mvn package

 

3. 查看Linux版本

lsb_release -a | grep Description

Linux版本代号参考

el5	Redhat Enterprise Linux 5 and clones (CentOS, Scientific Linux, etc)
el6	Redhat Enterprise Linux 6 and clones (CentOS, Scientific Linux, etc)
el7	Redhat Enterprise Linux 7 and clones (CentOS, Scientific Linux, etc)
sles11	SuSE Linux Enterprise Server 11.x
lucid	Ubuntu Linux 10.04 LTS (No CDH 5.x parcel provided)
precise	Ubuntu Linux 12.04 LTS
trusty	Ubuntu Linux 14.04 LTS (Newly supported in CM 5.2. No CDH 4.x parcel provided)
squeeze	Debian 6.x (No CDH 5.x parcel provided))
wheezy	Debian 7.x (Newly supported in CM 5.0. No CDH 4.x parcel provided)

4. 制作Livy parcel

git clone https://github.com/Automattic/cm-livy-scripts.git
cd cm-livy-scripts
./build_parcel.sh 1.0 el7

5. 制作Livy csd

./build_csd.sh 1.0

最后的文件： 

parcel：LIVY-0.6.0-el7.parcel  manifest.json

csd：LIVY-0.6.0.jar

参考：cm-livy-scripts

 

其他

参考：livy_zeppelin_cdh_csd_parcels

因为不需要zeppelin，修改了脚本，可以一键安装。

#!/bin/bash
set -x
set -e

JAR_PATH=/jars/apache-livy-0.6.0-incubating-bin.zip
DEST_PATH=/opt/apps/livy_cdh_parcels_csd/

LIVY_URL=https://mirrors.tuna.tsinghua.edu.cn/apache/incubator/livy/0.6.0-incubating/apache-livy-0.6.0-incubating-bin.zip
LIVY_MD5="55f966dc4b25ba595428dbc4634fac01"
LIVY_VERSION=0.6.0

livy_service_name="LIVY"
livy_service_name_lower="$( echo $livy_service_name | tr '[:upper:]' '[:lower:]' )"
livy_archive="$( basename $LIVY_URL )"
livy_folder="$( basename $livy_archive .zip )"
livy_parcel_folder="${livy_service_name}-${LIVY_VERSION}"
livy_parcel_name="$livy_parcel_folder-el7.parcel"
livy_built_folder="${livy_parcel_folder}_build"
livy_csd_build_folder="livy_csd_build"

function build_cm_ext {
  #Checkout if dir does not exist
  if [ ! -d cm_ext ]; then
    git clone https://github.com/cloudera/cm_ext.git
  fi
  if [ ! -f cm_ext/validator/target/validator.jar ]; then
    cd cm_ext
    mvn package
    cd ..
  fi
}

function get_livy {
  if [ ! -f "$livy_archive" ]; then
    cp $JAR_PATH $DEST_PATH
  fi
  livy_md5="$( md5sum $livy_archive | cut -d' ' -f1 )"
  if [ "$livy_md5" != "$LIVY_MD5" ]; then
    echo ERROR: md5 of $livy_archive is not correct
    exit 1
  fi
  if [ ! -d "$livy_folder" ]; then
    unzip $livy_archive
  fi
}

function build_livy_parcel {
  if [ -f "$livy_built_folder/$livy_parcel_name" ] && [ -f "$livy_built_folder/manifest.json" ]; then
    return
  fi
  if [ ! -d $livy_parcel_folder ]; then
    get_livy
    mv $livy_folder $livy_parcel_folder
  fi
  cp -r livy-parcel-src/meta $livy_parcel_folder
  sed -i -e "s/%VERSION%/$LIVY_VERSION/" ./$livy_parcel_folder/meta/parcel.json
  sed -i -e "s/%SERVICENAME%/$livy_service_name/" ./$livy_parcel_folder/meta/parcel.json
  sed -i -e "s/%SERVICENAMELOWER%/$livy_service_name_lower/" ./$livy_parcel_folder/meta/parcel.json
  java -jar cm_ext/validator/target/validator.jar -d ./$livy_parcel_folder
  mkdir -p $livy_built_folder
  tar zcvhf ./$livy_built_folder/$livy_parcel_name $livy_parcel_folder --owner=root --group=root
  java -jar cm_ext/validator/target/validator.jar -f ./$livy_built_folder/$livy_parcel_name
  python cm_ext/make_manifest/make_manifest.py ./$livy_built_folder
}

function build_livy_csd {
  JARNAME=${livy_service_name}-${LIVY_VERSION}.jar
  if [ -f "$JARNAME" ]; then
    return
  fi
  rm -rf ${livy_csd_build_folder}
  cp -rf ./livy-csd-src ${livy_csd_build_folder}
  sed -i -e "s/%SERVICENAME%/$livy_service_name/" ${livy_csd_build_folder}/descriptor/service.sdl
  sed -i -e "s/%SERVICENAMELOWER%/$livy_service_name_lower/" ${livy_csd_build_folder}/descriptor/service.sdl
  sed -i -e "s/%SERVICENAMELOWER%/$livy_service_name_lower/" ${livy_csd_build_folder}/scripts/control.sh
  java -jar cm_ext/validator/target/validator.jar -s ${livy_csd_build_folder}/descriptor/service.sdl -l "SPARK_ON_YARN SPARK2_ON_YARN"

  jar -cvf ./$JARNAME -C ${livy_csd_build_folder} .
}

case $1 in
parcel)
  build_cm_ext
  build_livy_parcel
  ;;
csd)
  build_livy_csd
  ;;
*)
  echo "Usage: $0 [parcel|csd]"
  ;;
esac