在Ubuntu上搭建hadoop和spark集群,1台master(namenode),3台slave(datanode)
1. 安装Java
sudo mkdir /usr/local/java/
sudo tar xvf jdk-7u79-linux-x64.tgz -C /usr/local/java/
2. 安装scala
$ sudo mkdir /usr/local/src/scala
$ sudo tar xvf scala-2.10.5.tgz -C /usr/local/src/scala/
3. 安装hadoop
sudo mkdir /usr/local/hadoop
sudo tar xvf hadoop-2.7.1.tar.gz -C /usr/local/hadoop/
4. 安装spark
Spark runs on Java 6+, Python 2.6+ and R 3.1+. For the Scala API, Spark 1.4.1 uses Scala 2.10. You will need to use a compatible Scala version (2.10.x).
sudo mkdir /usr/local/spark
sudo tar xvf spark-1.4.1-bin-hadoop2.6.tgz -C /usr/local/spark/
5. 添加环境路径
# config /etc/profile
export JAVA_HOME=/usr/local/java/jdk1.7.0_79
export SCALA_HOME=/usr/local/scala/scala-2.10.5
export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
export HADOOP_HOME=/usr/local/hadoop
export SPARK_HOME=/usr/local/spark/spark-1.4.1-bin-hadoop2.6
export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$SCALA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin
------------------------------
source /etc/profile
6. 创建hadoop用户,从master发布到slave
sudo chown -R hadoop:hadoop hadoop
sudo chown -R hadoop:hadoop spark
sudo chown -R hadoop:hadoop scala
sudo scp -r /usr/local/hadoop hadoop@slave1:~/
sudo mv ~/hadoop /usr/local/
sudo scp -r /usr/local/scala hadoop@slave1:~/
sudo mv ~/scala /usr/local/
sudo scp -r /usr/local/spark hadoop@slave1:~/
sudo mv ~/spark /usr/local/
7. 配置hadoop
fs.defaultFS
hdfs://master_ip:9000
dfs.name.dir
/usr/local/hadoop/datalog1
dfs.data.dir
/usr/local/hadoop/data1
dfs.replication
3
mapred.job.tracker
master:9001
# The java implementation to use.
export JAVA_HOME=/usr/local/java/jdk1.7.0_79
8. 配置spark
# /usr/local/spark/spark-1.4.1-bin-hadoop2.6/conf/spark-env.sh
#jdk
export JAVA_HOME=/usr/local/java/jdk1.7.0_79
#scala
export SCALA_HOME=/usr/local/scala/scala-2.10.5
#spark master ip
export SPARK_MASTER_IP=192.168.1.1
export SPARK_WORKER_MEMORY=2g
#hadoop config folder
export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop
# /usr/local/spark/spark-1.4.1-bin-hadoop2.6/conf/slaves
master
slave1
slave2
slave3
#