Spark User-guide Summary - Building Script

Package install

Basic Software Dependence, pre-install.sh

#!/usr/bin/env bash 
apt install software-properties-common 
add-apt-repository ppa:jonathonf/vim 
apt update apt install vim 

# git, expect, openssh-server 
apt install git expect openssh-server 
# Java8 
apt install openjdk-8-jre openjdk-8-jdk 
# Set JAVA_HOME 
JAVA_PATH=$(update-alternatives --list java) 
JAVA_HOME=${JAVA_PATH%/jre/bin*} 
echo "export JAVA_HOME=$JAVA_HOME" >> "$HOME/.bashrc"

Install Spark, spark-install.sh

#!/usr/bin/env bash 
SPARK_DOWNLOAD_URL="SPARK_URL"
SPARK_TGZ=${SPARK_DOWNLOAD_URL##*/} # spark-2.2.2-bin-hadoop2.7.tgz 
SPARK_VER=${SPARK_TGZ%%.tgz} # spark-2.2.2-bin-hadoop2.7 
if [ ! -d /usr/local/spark ]; then 
    [ ! -f /tmp/$SPARK_TGZ ] && wget -P /tmp --no-check-certificate $SPARK_DOWNLOAD_URL 
    tar -zxf /tmp/$SPARK_TGZ -C /usr/local 
    mv /usr/local/$SPARK_VER /usr/local/spark 
else 
    echo ">>>> /usr/local/spark already exists." 
fi

Install HBase, hbase-install.sh

#!/usr/bin/env bash 
HBASE_DOWNLOAD_URL="HBASE_URL"
HBASE_TAR_GZ=${HBASE_DOWNLOAD_URL##*/} # hbase-1.1.1-bin.tar.gz 
HBASE_VER=${HBASE_TAR_GZ%%-bin.tar.gz} # hbase-1.1.1 
if [ ! -d /usr/local/hbase ]; then 
    [ ! -f /tmp/$HBASE_TAR_GZ ] && wget -P /tmp --no-check-certificate $HBASE_DOWNLOAD_URL 
    tar -zxf /tmp/$HBASE_TAR_GZ -C /usr/local 
    mv /usr/local/$HBASE_VER /usr/local/hbase 
else 
    echo ">>>> /usr/local/hbase already exists." 
fi

Hostname Config

modify two files, /etc/hostname,/etc/hosts

first, create a ip_hostname.txt,slaves.txt for master

# ip_hostname.txt
192.168.1.1:Slave1
192.168.1.2:Slave2

# slaves.txt
Slave1
Slave2

for the script hostname, every node is different

#!/usr/bin/env bash
for line in $(cat ip_hostname.txt); do 
    ip=${line%%:*} 
    hostname=${line##*:} 
    ssh "root@$ip" "echo "$hostname" > /etc/hostname" 
done

for the script host, every node is same

#!/usr/bin/env bash 
for line in $(cat ip_hostname.txt); do 
    ip=${line%%:*} 
    hostname=${line##*:} echo "$ip $hostname" >> /etc/hosts 
done 
MASTER="MASTER IP" 
HOSTANME="Master" 
echo "$MASTER $HOSTANME" >> /etc/hosts 
echo "/etc/hosts has been updated!"

Package Copying

cp.sh

#!/usr/bin/env bash 
# DIR of Spark(,etc) on master
DIR='/usr/local/spark /usr/local/hbase' 
for slave in $(cat slaves.txt); do 
    rsync -avz $HOME/.bashrc "root@$slave":$HOME/.bashrc 
    rsync -avz $HOME/bin "root@$slave":$HOME 
    rsync -avz /etc/hosts "root@$slave":/etc 
    for dir in $DIR; do 
        rsync -avz $dir "root@$slave":/usr/local 
    done 
done