编译
- Hadoop的下载及解压
# hadoop源代码的下载可以为不同版本,推荐hadoop-2.8.1版本,
hadoop:root:/opt/sourcecode:>ll
total 448328
-rw-r--r-- 1 root root 34523353 Dec 11 07:02 hadoop-2.8.1-src.tar.gz
hadoop:root:/opt/sourcecode:>tar -xzvf hadoop-2.8.1-src.tar.gz
hadoop:root:/opt/sourcecode:>ll hadoop-2.8.1-src
total 220
-rw-rw-r-- 1 root root 15623 May 23 2017 BUILDING.txt
drwxr-xr-x 4 root root 4096 Dec 11 15:17 dev-support
drwxr-xr-x 4 root root 4096 Dec 11 16:07 hadoop-assemblies
drwxr-xr-x 4 root root 4096 Dec 11 16:07 hadoop-build-tools
drwxrwxr-x 3 root root 4096 Dec 11 16:26 hadoop-client
drwxr-xr-x 11 root root 4096 Dec 11 16:12 hadoop-common-project
drwxr-xr-x 3 root root 4096 Dec 11 16:26 hadoop-dist
drwxr-xr-x 9 root root 4096 Dec 11 16:18 hadoop-hdfs-project
drwxr-xr-x 10 root root 4096 Dec 11 16:24 hadoop-mapreduce-project
drwxr-xr-x 4 root root 4096 Dec 11 16:07 hadoop-maven-plugins
drwxr-xr-x 3 root root 4096 Dec 11 16:26 hadoop-minicluster
drwxr-xr-x 4 root root 4096 Dec 11 16:07 hadoop-project
drwxr-xr-x 3 root root 4096 Dec 11 16:07 hadoop-project-dist
drwxr-xr-x 19 root root 4096 Dec 11 16:26 hadoop-tools
drwxr-xr-x 4 root root 4096 Dec 11 16:22 hadoop-yarn-project
-rw-rw-r-- 1 root root 99253 May 23 2017 LICENSE.txt
-rw-rw-r-- 1 root root 15915 May 23 2017 NOTICE.txt
drwxrwxr-x 2 root root 4096 Jun 2 2017 patchprocess
-rw-rw-r-- 1 root root 20477 May 28 2017 pom.xml
-rw-r--r-- 1 root root 1366 May 20 2017 README.txt
-rwxrwxr-x 1 root root 1841 May 23 2017 start-build-env.sh
# 查看Hadoop配置条件:
hadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>cat BUILDING.txt
Build instructions for Hadoop
----------------------------------------------------------------------------------
Requirements:
* Unix System
* JDK 1.7+
* Maven 3.0 or later
* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.5.0
* CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac
* Zlib devel (if compiling native code)
* openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)
* Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)
* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
----------------------------------------------------------------------------------
- JAVA安装
# 解压Java到/usr/java
hadoop:root:/home/hadoop:>cd /usr/java/
hadoop:root:/usr/java:>tar -xzvf jdk-8u45-linux-x64.gz
# 修改用户组
hadoop:root:/usr/java:>chown -R root:root jdk1.8.0_45
hadoop:root:/usr/java:>ll
total 169216
drwxr-xr-x 8 root root 4096 Apr 10 2015 jdk1.8.0_45
-rw-r--r-- 1 root root 173271626 Dec 11 07:02 jdk-8u45-linux-x64.gz
# 修改环境变量 /etc/profile
hadoop:root:/usr/java:>vi /etc/profile
export JAVA_HOME=/usr/java/jdk1.8.0_45
export PATH=$JAVA_HOME/bin:$PATH
hadoop:root:/usr/java:>source /etc/profile
hadoop:root:/usr/java:>java -version
java version "1.8.0_45"
Java(TM) SE Runtime Environment (build 1.8.0_45-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode)
- maven安装到/etc/software,配置环境变量
hadoop:root:/opt/software:>unzip apache-maven-3.3.9-bin.zip
hadoop:root:/opt/software:>vi /etc/profile
export MAVEN_HOME=/opt/software/apache-maven-3.3.9
export MAVEN_OPTS="-Xms256m -Xmx512m"
export PATH=$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH
hadoop:root:/opt/software/protobuf-2.5.0:>source /etc/profile
hadoop:root:/opt/software:>mvn -version
Apache Maven 3.3.9 (bb52d8502b132ec0a5a3f4c09453c07478323dc5; 2015-11-10T11:41:47-05:00)
Maven home: /opt/software/apache-maven-3.3.9
Java version: 1.8.0_45, vendor: Oracle Corporation
Java home: /usr/java/jdk1.8.0_45/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "2.6.32-431.el6.x86_64", arch: "amd64", family: "unix"
- protobuf安装到/etc/software,配置环境变量
hadoop:root:/opt/software:>tar -xzvf protobuf-2.5.0.tar.gz
hadoop:root:/opt/software/protobuf-2.5.0:>yum install -y gcc gcc-c++ make cmake
hadoop:root:/opt/software/protobuf-2.5.0:>make && make install
hadoop:root:/opt/software/protobuf-2.5.0:>vi /etc/profile
export PROTOC_HOME=/usr/local/protobuf
export PATH=$PROTOC_HOME/bin:$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH
hadoop:root:/opt/software/protobuf-2.5.0:>source /etc/profile
hadoop:root:/opt/software/protobuf-2.5.0:>protoc --version
libprotoc 2.5.0
- Findbugs安装到/opt/software,配置环境变量
hadoop:root:/opt/software:>unzip findbugs-1.3.9.zip
hadoop:root:/opt/software:>vi /etc/profile
export FINDBUGS_HOME=/opt/software/findbugs-1.3.9
export PATH=$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH
hadoop:root:/opt/software:>source /etc/profile
hadoop:root:/opt/software:>findbugs -version
1.3.9
- 其他依赖
yum install -y openssl openssl-devel svn ncurses-devel zlib-devel libtool
yum install -y snappy snappy-devel bzip2 bzip2-devel lzo lzo-devel lzop autoconf automake
- 编译 (PS:网速正常大概2-3小时)
hadoop:root:/opt/software:>cd /opt/sourcecode/hadoop-2.8.1-src
hadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>mvn clean package -Pdist,native -DskipTests -Dtar
安装
- 创建使用Hadoop的用户,添加sudo权限
hadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>useradd hadoop
hadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>id hadoop
uid=500(hadoop) gid=500(hadoop) groups=500(hadoop)
hadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>vi /etc/sudoers
hadoop ALL=(root) NOPASSWD:ALL
安装Java
解压Hadoop
hadoop:root:/opt/sourcecode/hadoop-2.8.1-src/hadoop-dist/target:>mv /opt/sourcecode/hadoop-2.8.1-src/hadoop-dist/target/hadoop-2.8.1.tar.gz /opt/software
hadoop:root:/opt/software:>tar -xzvf hadoop-2.8.1.tar.gz
- 配置文件
core-site.xml
hadoop:hadoop:/opt/software/hadoop:>vi /opt/software/hadoop/etc/hadoop/core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://192.168.137.5:9000</value>
</property>
</configuration>
hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>hadoop:50090</value>
</property>
<property>
<name>dfs.namenode.secondary.https-address</name>
<value>hadoop:50091</value>
</property>
</configuration>
hadoop-env.sh
# The java implementation to use.
export JAVA_HOME=/usr/java/jdk1.8.0_45
slave
hadoop:hadoop:/opt/software/hadoop/etc/hadoop:>cat slaves
hadoop
- 配置ssh信任关系
hadoop:hadoop:/home/hadoop:>ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
hadoop:hadoop:/home/hadoop:>cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
hadoop:hadoop:/home/hadoop:>chmod 0600 ~/.ssh/authorized_keys
hadoop:hadoop:/home/hadoop/.ssh:>ll
total 16
-rw------- 1 hadoop hadoop 395 Dec 14 04:46 authorized_keys
-rw------- 1 hadoop hadoop 1675 Dec 14 04:43 id_rsa
-rw-r--r-- 1 hadoop hadoop 395 Dec 14 04:43 id_rsa.pub
-rw-r--r-- 1 hadoop hadoop 1182 Dec 14 04:59 known_hosts
- 格式化
hadoop:hadoop:/opt/software/hadoop:>bin/hdfs namenode format
- 启动HDFS
hadoop:hadoop:/opt/software/hadoop:>sbin/start-dfs.sh