Ubuntu上搭建hadoop和spark集群

在Ubuntu上搭建hadoop和spark集群，1台master(namenode)，3台slave(datanode)

1. 安装Java

sudo mkdir /usr/local/java/
sudo tar xvf jdk-7u79-linux-x64.tgz -C /usr/local/java/

2. 安装scala

$ sudo mkdir /usr/local/src/scala
$ sudo tar xvf scala-2.10.5.tgz -C /usr/local/src/scala/

3. 安装hadoop

sudo mkdir /usr/local/hadoop
sudo tar xvf hadoop-2.7.1.tar.gz -C /usr/local/hadoop/

4. 安装spark

Spark runs on Java 6+, Python 2.6+ and R 3.1+. For the Scala API, Spark 1.4.1 uses Scala 2.10. You will need to use a compatible Scala version (2.10.x).

sudo mkdir /usr/local/spark
sudo tar xvf spark-1.4.1-bin-hadoop2.6.tgz -C /usr/local/spark/

5. 添加环境路径

# config /etc/profile

export JAVA_HOME=/usr/local/java/jdk1.7.0_79
export SCALA_HOME=/usr/local/scala/scala-2.10.5
export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
export HADOOP_HOME=/usr/local/hadoop
export SPARK_HOME=/usr/local/spark/spark-1.4.1-bin-hadoop2.6
export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$SCALA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin


------------------------------

source /etc/profile

6. 创建hadoop用户，从master发布到slave

sudo chown -R hadoop:hadoop hadoop
sudo chown -R hadoop:hadoop spark
sudo chown -R hadoop:hadoop scala

sudo scp -r /usr/local/hadoop hadoop@slave1:~/
sudo mv ~/hadoop /usr/local/

sudo scp -r /usr/local/scala hadoop@slave1:~/
sudo mv ~/scala /usr/local/

sudo scp -r /usr/local/spark hadoop@slave1:~/
sudo mv ~/spark /usr/local/

7. 配置hadoop

      
      


      
      


       
       

        
        
         
         fs.defaultFS
        
        

        
        
         
         hdfs://master_ip:9000
        
        

       
       


      
      


      
      


      
      


       
       

        
        
         
         dfs.name.dir
        
        

        
        
         
         /usr/local/hadoop/datalog1
        
        

       
       


       
       

        
        
         
         dfs.data.dir
        
        

        
        
         
         /usr/local/hadoop/data1
        
        

       
       


       
       

        
        
         
         dfs.replication
        
        

        
        
         
         3
        
        

       
       


      
      


      
      


      
      


       
       

        
        
         
         mapred.job.tracker
        
        

        
        
         
         master:9001
        
        

       
       


      
      


      
      

# The java implementation to use.
export JAVA_HOME=/usr/local/java/jdk1.7.0_79

8.  配置spark

# /usr/local/spark/spark-1.4.1-bin-hadoop2.6/conf/spark-env.sh

#jdk
export JAVA_HOME=/usr/local/java/jdk1.7.0_79

#scala
export SCALA_HOME=/usr/local/scala/scala-2.10.5

#spark master ip
export SPARK_MASTER_IP=192.168.1.1

export SPARK_WORKER_MEMORY=2g
#hadoop config folder
export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop

# /usr/local/spark/spark-1.4.1-bin-hadoop2.6/conf/slaves
master
slave1
slave2
slave3

#