2、Hadoop的运行环境的搭建及hadoop目录的了解

学习目标
一、虚拟机及jdk、hadoop的安装
二、Hadoop的目录
学习内容

一、虚拟机及jdk、hadoop的安装

1、虚拟机的安装准备
1）克隆虚拟机
克隆完整的虚拟机
2）修改虚拟机的IP
vim /etc/sysconfig/network-scripts/ifcfg-ens33

TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
UUID=97ea7149-0e18-4e29-b664-0ed8aaa9a55b
DEVICE=ens33
ONBOOT=yes
IPADDR=192.168.148.133     #更改IP地址
PREFIX=24
GATEWAY=192.168.148.2
DNS1=192.168.148.2

3）修改主机名
hostnamectl set-hostname <主机名>
vim /etc /hosts

127.0.0.1   hadoop2
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.148.133  hadoop2    #添加IP及用户名

4）安装防火墙并关闭
安装 yum install iptables-services 安装过程中有选项就选y
chkconfig iptables off 重启后生效 reboot 重启
或 service iptables start 即时生效
5）创建普通用户并具有root权限

2、jdk的安装
1）安装rz命令
yum -y install lrzsz
2) 创建一个文件夹并把安装包通过rz命令上传至文件夹下
3）通过tar -xzvf <压缩包>命令解压文件
tar -xzvf <压缩包> -C /文件目录
4）通过mv <压缩包文件> <改名文件>命令更改文件名称
5）通过vim /etc/profile命令配置jdk环境变量

       fi
fi   #以下为环境变量
export JAVA_HOME=/root/usr/bag/java   #、root/文件安装路径
export PATH=$JAVA_HOME/bin:$PATH

6）通过source /etc/profile 刷新环境变量
7）通过java -version 查看jdk是否生效

[root@hadoop bag]# java -version
java version "1.8.0_161"   #说明jdk环境变量生效
Java(TM) SE Runtime Environment (build 1.8.0_161-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode)

3、hadoop的安装
1）通过tar -xzvf <压缩包>命令解压文件
tar -xzvf <压缩包> -C /文件目录
2）通过mv <压缩包文件> <改名文件>命令更改文件名称
3）通过vim /etc/profile命令配置hadoop环境变量
export HADOOP_HOME=/root/usr//bag/hadoop #hadoop的安装路径
export PATH=$HADOO{P}_{H}OME/bin:$PATH
4）通过source /etc/profile 刷新环境变量

二、Hadoop的目录结构

1、Hadoop的目录结构及说明

[root@hadoop bag]# cd hadoop
[root@hadoop hadoop]# ls
bin  include  libexec      NOTICE.txt  sbin
etc  lib      LICENSE.txt  README.txt  share

bin目录结构

[root@hadoop hadoop]# cd bin
[root@hadoop bin]# ls
container-executor  hdfs      mapred.cmd               yarn
hadoop              hdfs.cmd  oom-listener             yarn.cmd
hadoop.cmd          mapred    test-container-executor

#说明
#hdfs 存储
#hadoop 管理集群的
#yarn 管理资源调度的

etc目录结构 #etc目录下存储的都是配置文件

[root@hadoop hadoop]# ls
bin  include  libexec      NOTICE.txt  sbin
etc  lib      LICENSE.txt  README.txt  share
[root@hadoop hadoop]# cd etc
[root@hadoop etc]# ls
hadoop
[root@hadoop etc]# cd hadoop
[root@hadoop hadoop]# ls
capacity-scheduler.xml            kms-log4j.properties
configuration.xsl                 kms-site.xml
container-executor.cfg            log4j.properties
core-site.xml                     mapred-env.cmd
hadoop-env.cmd                    mapred-env.sh
hadoop-env.sh                     mapred-queues.xml.template
hadoop-metrics2.properties        mapred-site.xml
hadoop-policy.xml                 shellprofile.d
hadoop-user-functions.sh.example  ssl-client.xml.example
hdfs-site.xml                     ssl-server.xml.example
httpfs-env.sh                     user_ec_policies.xml.template
httpfs-log4j.properties           workers
httpfs-signature.secret           yarn-env.cmd
httpfs-site.xml                   yarn-env.sh
kms-acls.xml                      yarnservice-log4j.properties
kms-env.sh                        yarn-site.xml

include目录 #.h结尾的都是文件

[root@hadoop hadoop]# ls
bin  include  libexec      NOTICE.txt  sbin
etc  lib      LICENSE.txt  README.txt  share
[root@hadoop hadoop]# cd include
[root@hadoop include]# ls
hdfs.h    SerialUtils.hh  TemplateFactory.hh
Pipes.hh  StringUtils.hh

lib目录 #本地库

[root@hadoop hadoop]# ls
bin  include  libexec      NOTICE.txt  sbin
etc  lib      LICENSE.txt  README.txt  share
[root@hadoop hadoop]# cd lib
[root@hadoop lib]# ls
native
[root@hadoop lib]# cd native
[root@hadoop native]# ls
examples          libhadoop.so.1.0.0  libhdfs.so.0.0.0
libhadoop.a       libhadooputils.a    libnativetask.a
libhadooppipes.a  libhdfs.a           libnativetask.so
libhadoop.so      libhdfs.so          libnativetask.so.1.0.0

libexec 和lib基本相同
.txt文件

sbin目录 #启动、停止，包括集群等配置的文件的命令

[root@hadoop hadoop]# ls
bin  include  libexec      NOTICE.txt  sbin
etc  lib      LICENSE.txt  README.txt  share
[root@hadoop hadoop]# cd sbin
[root@hadoop sbin]# ls
distribute-exclude.sh    start-all.sh         stop-balancer.sh
FederationStateStore     start-balancer.sh    stop-dfs.cmd
hadoop-daemon.sh         start-dfs.cmd        stop-dfs.sh
hadoop-daemons.sh        start-dfs.sh         stop-secure-dns.sh
httpfs.sh                start-secure-dns.sh  stop-yarn.cmd
kms.sh                   start-yarn.cmd       stop-yarn.sh
mr-jobhistory-daemon.sh  start-yarn.sh        workers.sh
refresh-namenodes.sh     stop-all.cmd         yarn-daemon.sh
start-all.cmd            stop-all.sh          yarn-daemons.sh

share目录 #内涵说明文档及案例

[root@hadoop hadoop]# ls
bin  include  libexec      NOTICE.txt  sbin
etc  lib      LICENSE.txt  README.txt  share
[root@hadoop hadoop]# cd share
[root@hadoop share]# ls
doc  hadoop
[root@hadoop share]# cd hadoop
[root@hadoop hadoop]# ls
client  common  hdfs  mapreduce  tools  yarn
[root@hadoop hadoop]# cd mapreduce/
[root@hadoop mapreduce]# ls
hadoop-mapreduce-client-app-3.2.2.jar
hadoop-mapreduce-client-common-3.2.2.jar
hadoop-mapreduce-client-core-3.2.2.jar
hadoop-mapreduce-client-hs-3.2.2.jar
hadoop-mapreduce-client-hs-plugins-3.2.2.jar
hadoop-mapreduce-client-jobclient-3.2.2.jar
hadoop-mapreduce-client-jobclient-3.2.2-tests.jar
hadoop-mapreduce-client-nativetask-3.2.2.jar
hadoop-mapreduce-client-shuffle-3.2.2.jar
hadoop-mapreduce-client-uploader-3.2.2.jar
hadoop-mapreduce-examples-3.2.2.jar
jdiff
lib
lib-examples
sources

Hadoop主要以bin\etc\sbin为主，这几个经常用到