Hadoop环境搭建(集群)
- 环境准备(centos7 x 3)
- Hadoop参数配置
- 启动Hadoop
环境准备
- Linux新增hadoop用户(当前用户root):
useradd -m hadoop -s /bin/bash
passwd hadoop
visudo
hadoop ALL=(ALL) NOPASSWD:ALL # 追加配置
sudo yum install openssh-clients
sudo yum install openssh-server
rpm -qa | grep ssh
cd ~/.ssh/
ssh-keygen -t rsa
cat id_rsa.pub >> authorized_keys
chmod 600 ./authorized_keys
ssh localhost
vi /etc/hosts
cd ~/.ssh/
rm -f id_rsa*
ssh-keygen -t rsa
cat id_rsa.pub >> authorized_keys
ssh-copy-id -i Master
ssh-copy-id -i Master
more authorized_keys
chmod 600 authorized_keys
scp /home/hadoop/.ssh/authorized_keys Slave1:/home/hadoop/.ssh/
scp /home/hadoop/.ssh/authorized_keys Slave2:/home/hadoop/.ssh/
rpm -ql java-1.7.0-openjdk-devel | grep '/bin/javac'
vi ~/.bashrc
export JAVA_HOME=/usr/java/jdk1.7.0_79
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin
source ~/.bashrc
echo $JAVA_HOME
java -version
Hadoop安装及配置
cat ~/download/hadoop-2.x.x.tar.gz.mds | grep 'MD5'
md5sum ~/下载/hadoop-2.x.x.tar.gz | tr "a-z" "A-Z"
sudo tar -zxf ~/下载/hadoop-2.x.x.tar.gz -C /usr
cd /usr/
sudo mv ./hadoop-2.x.x/ ./hadoop
sudo chown -R hadoop:hadoop ./hadoop
cd /usr/hadoop
./bin/hadoop version
- 向conf下的slaves文件中添加所有的DataNode节点的hostname
- 修改core-site.xml,配置如下:
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://Master:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/usr/hadoop/tmp</value>
</property>
</configuration>
<configuration>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>Master:50090</value>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/usr/hadoop/tmp/dfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/usr/hadoop/tmp/dfs/data</value>
</property>
</configuration>
- 修改mapred-site.xml (默认文件名为 mapred-site.xml.template),配置如下:
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>Master:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>Master:19888</value>
</property>
<property>
<name>mapreduce.tasktracker.map.tasks.maximum</name>
<value>2</value>
<final>true</final>
</property>
<property>
<name>mapreduce.tasktracker.reduce.tasks.maximum</name>
<value>2</value>
<final>true</final>
</property>
</configuration>
<configuration>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>Master</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>8192</value>
</property>
<property>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>8</value>
</property>
</configuration>
- 将Master节点上的hadoop文件打包并分发至其他节点
cd /usr
sudo rm -r ./hadoop/tmp
sudo rm -r ./hadoop/logs/*
tar -zcf ~/hadoop.master.tar.gz ./hadoop
cd ~
scp ./hadoop.master.tar.gz Slave1:/home/hadoop
scp ./hadoop.master.tar.gz Slave2:/home/hadoop
sudo rm -r /usr/hadoop
sudo tar -zxf ~/hadoop.master.tar.gz -C /usr
sudo chown -R hadoop:hadoop /usr/hadoop
vi ~/.bashrc
export HADOOP_HOME=/usr/hadoop
export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin
export JAVA_LIBRARY_PATH=/usr/hadoop/lib/native
source ~/.bashrc
启动Hadoop
hdfs namenode -format
start-dfs.sh
start-yarn.sh
mr-jobhistory-daemon.sh start historyserver
jps
hdfs dfsadmin -report
***DataNode无法启动,删除所有节点(包括Slave节点)上的/usr/hadoop/tmp文件夹,重新执行一次 hdfs namenode -format
hdfs dfs -mkdir -p /user/hadoop
hdfs dfs -mkdir input
hdfs dfs -put XXX.txt input
hadoop jar /wordcount.jar input output
stop-yarn.sh
stop-dfs.sh
mr-jobhistory-daemon.sh stop historyserver