hadoop HA 搭建

组件部署

IP	主机名称	JDK	ZK	NN	RM	DN	NM	JN
10.30.21.58	master	y		y	y
10.30.21.59	masterbak	y		y	y
10.30.21.60	slave1	y	y			y	y	y
10.30.21.61	slave2	y	y			y	y	y
10.30.21.62	slave3	y	y			y	y	y

 

 

 

 

 

 

 

 

 

一 账号创建

新建用户
adduser hadoop
passwd hadoop

添加用户组
usermod -a -G hadoop hadoop

赋予root权限
vi /etc/sudoers
root    ALL=(ALL)       ALL
hadoop　　ALL=(ALL)　　ALL  (新增)

二 下载源码

创建hadoop目录
mkdir -p /data/hadoop/soft
chown -R hadoop:hadoop /data/hadoop
切换用户
[root@master hadoop]# su hadoop
[hadoop@master soft]$ pwd
/data/hadoop/soft
下载源码
wget https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5-src.tar.gz

三 部署相关组件

1 jdk
mkdir /opt/java
jdk-8u241-linux-x64.tar.gz
/opt/java/jdk1.8.0_241
ln -s  /opt/java/jdk1.8.0_241 /opt/jdk

vi /etc/profile

export JAVA_HOME=/opt/java/jdk1.8.0_241
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib:$CLASSPATH
export JAVA_PATH=${JAVA_HOME}/bin:${JRE_HOME}/bin
export PATH=$PATH:${JAVA_PATH}

source /etc/profile

2 安装 glibc-headers 和 g++ 命令
yum install glibc-headers
yum install gcc-c++

3 安装 make 和 cmake
yum install make
yum install cmake

4 安装protobuf
tar -zxvf protobuf-2.5.0.tar.gz
cd /opt/protobuf-2.5.0
./configure
make
make check
make install
ldconfig

vi /etc/profile
export LD_LIBRARY_PATH=/opt/protobuf-2.5.0
export PATH=$PATH:${JAVA_PATH}:${LD_LIBRARY_PATH}
source /etc/profile

protoc --version
[root@master protobuf-2.5.0]# protoc --version
libprotoc 2.5.0

5 安装 openssl 库
yum install openssl-devel

6 安装 ncurses-devel 库
yum install ncurses-devel

7 maven
tar -zxvf apache-maven-3.5.3-bin.tar.gz
cd /opt/apache-maven-3.5.3
vi conf/settings.xml 

  <mirrors>
    <!-- mirror
     | Specifies a repository mirror site to use instead of a given repository. The repository that
     | this mirror serves has an ID that matches the mirrorOf element of this mirror. IDs are used
     | for inheritance and direct lookup purposes, and must be unique across the set of mirrors.
     |
    <mirror>
      <id>mirrorId</id>
      <mirrorOf>repositoryId</mirrorOf>
      <name>Human Readable Name for this Mirror.</name>
      <url>http://my.repository.com/repo/path</url>
    </mirror>
     -->
    <mirror>
       <id>nexus-aliyun</id>                        
       <mirrorOf>central</mirrorOf>                       
       <name>Nexus aliyun</name>                       
       <url>http://maven.aliyun.com/nexus/content/groups/public</url>
    </mirror>
  </mirrors>

vi /etc/profile
export MAVEN_HOME=/opt/apache-maven-3.5.3
export PATH=$PATH:${JAVA_PATH}:${LD_LIBRARY_PATH}:${MAVEN_HOME}/bin
source /etc/profile

[root@master apache-maven-3.5.3]# mvn -version
Apache Maven 3.5.3 (3383c37e1f9e9b3bc3df5050c29c8aff9f295297; 2018-02-25T03:49:05+08:00)
Maven home: /opt/apache-maven-3.5.3
Java version: 1.8.0_241, vendor: Oracle Corporation
Java home: /opt/java/jdk1.8.0_241/jre
Default locale: zh_CN, platform encoding: UTF-8
OS name: "linux", version: "3.10.0-1160.15.2.el7.x86_64", arch: "amd64", family: "unix"

8 apache-ant
tar -zxvf apache-ant-1.9.15-bin.tar.gz

vi /etc/profile
export ANT_HOME=/opt/apache-ant-1.9.15
export PATH=$PATH:${JAVA_PATH}:${LD_LIBRARY_PATH}:${MAVEN_HOME}/bin:${ANT_HOME}/bin
source /etc/profile

[root@master apache-ant-1.9.15]# ant -version        
Apache Ant(TM) version 1.9.15 compiled on May 10 2020


关闭防火墙
systemctl stop firewalld.serivce
systemctl disable firewalld.service

四 编译源码

cd /data/hadoop/soft
tar -zxvf hadoop-2.8.5-src.tar.gz
cd /data/hadoop/soft/hadoop-2.8.5-src
mvn package -Pdist,native -DskipTests -Dtar

[INFO] Reactor Summary:
[INFO] 
[INFO] Apache Hadoop Main 2.8.5 ........................... SUCCESS [01:14 min]
[INFO] Apache Hadoop Build Tools .......................... SUCCESS [ 54.307 s]
[INFO] Apache Hadoop Project POM .......................... SUCCESS [ 17.853 s]
[INFO] Apache Hadoop Annotations .......................... SUCCESS [ 11.368 s]
[INFO] Apache Hadoop Assemblies ........................... SUCCESS [  0.142 s]
[INFO] Apache Hadoop Project Dist POM ..................... SUCCESS [ 35.581 s