spark启动脚本:
###hadoop配置文件
export HADOOP_CONF_DIR=/etc/hadoop/conf
#spark-submit路径
sparksubmits="/opt/cloudera/parcels/SPARK2/bin/spark2-submit"
#jar包所在本地目录
jars="/usr/java/checkpoint/SSE_ST2_ANALYSIS_SPARK.jar"
echo "begin running NewsStreamimgClusterDriver model"
#使用spark-submit提交spark程序
su root -c "$sparksubmits --class cn.com.trs.topic.news.streaming.NewsStreamingClusterDriver \
--master yarn \
--driver-cores 1 \
--driver-memory 2g \
--deploy-mode cluster \
--executor-cores 1 \
--num-executors 16 \
--executor-memory 1g \
--name NewsStreamingClusterDriver \
$jars XZ 4 20 \
"
echo "finished!"
l里面用到的spark2-submit的命令脚本
#!/bin/bash
# Reference: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in
SOURCE="${BASH_SOURCE[0]}"
BIN_DIR="$( dirname "$SOURCE" )"
while [ -h "$SOURCE" ]
do
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
done
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
CDH_LIB_DIR=$BIN_DIR/../../CDH/lib
LIB_DIR=$BIN_DIR/../lib
export HADOOP_HOME=$CDH_LIB_DIR/hadoop
# Autodetect JAVA_HOME if not defined
. $CDH_LIB_DIR/bigtop-utils/bigtop-detect-javahome
exec $LIB_DIR/spark2/bin/spark-submit "$@"