Hadoop完全分布式安装
1.1 安装JDK
- 解压
tar-xzvf jdk-8u212-linux-x64.tar.gz
- 配置环境变量
vim /etc/profile
3. 添加如下配置
export JAVA_HOME=/opt/module/jdk1.8.0_212
export PATH=$PATH:$JAVA_HOME/bin
4. 使配置生效
source /etc/profile
1.2 免密登录
(每个节点都需要执行)
ssh-keygen -t rsa
其中会让输入密码等操作,直接不输入,按enter键
会在/root/.ssh产生id_rsa和id_rsa.pub文件
(每个节点需要执行)
cd /root/.ssh
(每个节点需要执行)
cat id_rsa.pub >>authorized_keys
将其他节点的id_rsa.pub内容添加到本节点的authorized_keys文件中(每个节点需要执行)
1.3 解压hadoop
tar -xzvf hadoop-3.1.3.tar.gz
1.4 配置Hadoop配置
切换到/opt/module/hadoop-3.1.3/etc/hadoop目录下
cd /opt/module/hadoop-3.1.3/etc/hadoop
1.vim core-site.xml
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/opt/module/hadoop-3.1.3/tmp</value>
<description>Abase for other temporary directories.</description>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://bigdata1:9000</value>
</property>
<property>
<name>hadoop.proxyuser.root.hosts</name>
<description>对root用户不进行限制</description>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.root.groups</name>
<value>*</value>
<description>对root群组不限制</description>
</property>
</configuration>
2.vim hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/opt/module/hadoop-3.1.3/tmp/dfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/opt/module/hadoop-3.1.3/tmp/dfs/data</value>
</property>
</configuration>
3.vim mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>
</configuration>
4.vim yarn-env.sh
export JAVA_HOME=/opt/module/jdk1.8.0_212
5.vim yarn-site.xml
<configuration>
<!-- Site specific YARN configuration properties -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>bigdata1</value>
</property>
</configuration>
6.vim workers
bigdata1
bigdata2
bigdata3
7.修改 /opt/module/hadoop-3.1.3/sbin/start-dfs.sh和 /opt/module/hadoop-3.1.3/sbin/stop-dfs.sh
vim /opt/module/hadoop-3.1.3/sbin/start-dfs.sh
vim /opt/module/hadoop-3.1.3/sbin/stop-dfs.sh
HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root
8.修改 /opt/module/hadoop-3.1.3/sbin/start-yarn.sh和 /opt/module/hadoop-3.1.3/sbin/stop-yarn.sh
vim /opt/module/hadoop-3.1.3/sbin/start-yarn.sh
vim /opt/module/hadoop-3.1.3/sbin/stop-yarn.sh
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root
1.5 配置环境变量
vim /etc/profile
#HADOOP_HOME
export HADOOP_HOME=/opt/module/hadoop-3.1.3
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
使变量生效
source /etc/profile
1.6 分发安装包
scp -r /opt/module/hadoop-3.1.3 bigdata2:/opt/module/
scp -r /opt/module/hadoop-3.1.3 bigdata3:/opt/module/
1.7 Hdfs格式化
hdfs namenode -format
1.8 Hadoop启动
在…/etc/hadoop目录下
start-all.sh
使用jps查看java进程会看到 如下进程
在bigdata1上可看到
NameNode
SecondaryNameNode
ResourceManager
NodeManager
DataNode
在bigdata2上可看到
NodeManager
DataNode
在bigdata3上可看到
NodeManager
DataNode