1.创建Hadoop用户
useradd hadoop
2.安装jdk1.8
rpm -qa|grep jdk
rpm -e java-1.8.0-openjdk-1.8.0.312.b07-1.el7_9.x86_64 --nodeps
rpm -e java-1.8.0-openjdk-headless-1.8.0.312.b07-1.el7_9.x86_64 --nodeps
tar -zxvf jdk-8u112-linux-x64.tar.gz -C /opt/cdh-5.14.2/
3.修改配置文件
vim /etc/profile
export JAVA_HOME=/opt/cdh-5.14.2/jdk1.8.0_112
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/rt.jar
4.配置Hadoop用户环境变量
export HADOOP_HOME=/opt/cdh-5.14.2/hadoop-2.6.0-cdh5.14.2
export JAVA_LIBRAY_PATH=$HADOOP_HOME/lib/native
5.解压hadoop-2.6.tar.gz(cdh版本)
$ tar -zxvf hadoop-2.6.0-cdh5.14.2.tar.gz -C /opt/cdh-5.14.2/
6.配置hadoop的java环境支持(hadoop-env.sh,mapred-env.sh,yarn-env.sh)
export JAVA_HOME=/opt/cdh-5.14.2/jdk1.8.0_112
7.配置core-site.xml
<property>
<name>fs.defaultFS</name>
<value>hdfs://192.168.1.231:8020</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/cdh-5.14.2/hadoop-2.6.0-cdh5.14.2/data</value>
</property>
8.配置hdfs-site.xml
存放到hdfs上的文件的副本数,生产环境至少为3
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
9.格式化namenode
$ bin/hdfs namenode -format
10.启动hdfs守护进程
$ sbin/hadoop-daemon.sh start namenode
$ sbin/hadoop-daemon.sh start datanode
11.查看守护进程
$ jps
3097 Jps
2931 NameNode
3023 DataNode
12.web访问界面
http://192.168.1.231:50070
13.HDFS文件系统常用命令
$ bin/hdfs dfs
$ bin/hdfs dfs -ls /
$ bin/hdfs dfs -mkdir -p /input
$ bin/hdfs dfs -rmdir /input
$ bin/hdfs dfs -put /u01/soft/jdk-8u112-linux-x64.tar.gz /input
14.解决warning(Unable to load native-hadoop library for your platform)
出现此警告的原因是本机库默认编译为32位
tar -xvf hadoop-native-64-2.6.0.tar -C $HADOOP_HOME/lib/native
15.配置YARN
${HADOOP_HOME}/etc/hadoop/yarn-site.xml
指定ResorceManager所在服务器的主机名
指明在执行MapReduce的时候使用shuffle机制
<property>
<name>yarn.resourcemanager.hostname</name>
<value>192.168.1.231</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
16.复制并重名模板文件
$ cp etc/hadoop/mapred-site.xml.template etc/hadoop/mapred-site.xml
17.配置${HADOOP_HOME}/etc/hadoop/mapred-site.xml
指定MapReduce基于Yarn来运行
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
18.启动hdfs yarn进程
$ sbin/yarn-daemon.sh start resourcemanager
$ sbin/yarn-daemon.sh start nodemanager
19.检查hdfs yarn启动状态
$ jps
4595 ResourceManager
4419 NameNode
4836 NodeManager
5896 Jps
4506 DataNode
20.web访问界面
http://192.168.1.231:8088
http://192.168.1.231:19888