hadoop配置

[root@master ~]# cd /usr/local/soft
[root@master soft]# ls
data          packages  redis-6.2.6  shell01  test.txt
jdk1.8.0_171  redis     shell        show
[root@master soft]# ssh master
Last login: Mon Mar 21 16:34:46 2022 from fe80::e938:d89b:966:b6a8%ens33
[root@master ~]# ssh node1
Last login: Mon Mar 21 16:34:49 2022 from fe80::e938:d89b:966:b6a8%ens33
[root@node1 ~]# exit
登出
Connection to node1 closed.
[root@master ~]# ssh node2
Last login: Mon Mar 21 16:37:42 2022 from node1
[root@node2 ~]# exit
登出
Connection to node2 closed.

[root@master ~]# ntpdate ntp.aliyun.com
21 Mar 16:42:58 ntpdate[1914]: adjust time server 203.107.6.88 offset -0.000626 sec
[root@master ~]# date
2022年 03月 21日 星期一 16:46:26 CST
[root@master ~]# java -version
java version "1.8.0_171"
Java(TM) SE Runtime Environment (build 1.8.0_171-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.171-b11, mixed mode)

[root@master soft]# rz -E
rz waiting to receive.
[root@master soft]# ls
0??   hadoop-2.7.6.tar.gz  redis        shell01
A??   jdk1.8.0_171         redis-6.2.6  show
data  packages             shell        test.txt
[root@master soft]# mv hadoop-2.7.6.tar.gz /usr/local/soft/packages/
[root@master soft]# cd /usr/local/soft/packages/
[root@master packages]# ls
hadoop-2.7.6.tar.gz  redis-6.2.6.tar.gz
[root@master packages]# tar -zxvf hadoop-2.7.6.tar.gz -C /usr/local/soft/
[root@master packages]# cd ..
[root@master soft]# ls
0??  data          jdk1.8.0_171  redis        shell    show
A??  hadoop-2.7.6  packages      redis-6.2.6  shell01  test.txt
[root@master soft]# cd  hadoop-2.7.6
[root@master hadoop-2.7.6]# ls
bin  include  libexec      NOTICE.txt  sbin
etc  lib      LICENSE.txt  README.txt  share
root@master hadoop-2.7.6]# cd sbin/
[root@master sbin]# ls
distribute-exclude.sh    start-all.cmd        stop-balancer.sh
hadoop-daemon.sh         start-all.sh         stop-dfs.cmd
hadoop-daemons.sh        start-balancer.sh    stop-dfs.sh
hdfs-config.cmd          start-dfs.cmd        stop-secure-dns.sh
hdfs-config.sh           start-dfs.sh         stop-yarn.cmd
httpfs.sh                start-secure-dns.sh  stop-yarn.sh
kms.sh                   start-yarn.cmd       yarn-daemon.sh
mr-jobhistory-daemon.sh  start-yarn.sh        yarn-daemons.sh
refresh-namenodes.sh     stop-all.cmd
slaves.sh                stop-all.sh
[root@master sbin]# cd ..
[root@master hadoop-2.7.6]# pwd
/usr/local/soft/hadoop-2.7.6
[root@master hadoop-2.7.6]# vim /etc/profile
[root@master hadoop-2.7.6]# source /etc/profile
[root@master hadoop-2.7.6]# start-
start-all.cmd         start-pulseaudio-x11
start-all.sh          start-secure-dns.sh
start-balancer.sh     start-statd
start-dfs.cmd         start-yarn.cmd
start-dfs.sh          start-yarn.sh

[root@master hadoop-2.7.6]# cd ..
[root@master soft]# cd /etc
[root@master etc]# cd /usr/local/soft/hadoop-2.7.6/etc/hadoop/
[root@master hadoop]# pwd
/usr/local/soft/hadoop-2.7.6/etc/hadoop
[root@master hadoop]# ls
capacity-scheduler.xml      kms-env.sh
configuration.xsl           kms-log4j.properties
container-executor.cfg      kms-site.xml
core-site.xml               log4j.properties
hadoop-env.cmd              mapred-env.cmd
hadoop-env.sh               mapred-env.sh
hadoop-metrics2.properties  mapred-queues.xml.template
hadoop-metrics.properties   mapred-site.xml.template
hadoop-policy.xml           slaves
hdfs-site.xml               ssl-client.xml.example
httpfs-env.sh               ssl-server.xml.example
httpfs-log4j.properties     yarn-env.cmd
httpfs-signature.secret     yarn-env.sh
httpfs-site.xml             yarn-site.xml
kms-acls.xml

core-site.xml

  <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
    </property>

    <property>
        <name>hadoop.tmp.dir</name>
        <value>/usr/local/soft/hadoop-2.7.6/tmp</value>
    </property>

    <property>
        <name>fs.trash.interval</name>
        <value>1440</value>
    </property>

hadoop-env.sh

export JAVA_HOME=/usr/local/soft/jdk1.8.0_171

 hdfs-site.xml

 <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>

    <property>
        <name>dfs.permissions</name>
        <value>false</value>
    </property>

mapred-site.xml.template

# 1、重命名文件
cp mapred-site.xml.template mapred-site.xml
# 2、修改
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>

    <property>  
        <name>mapreduce.jobhistory.address</name>  
        <value>master:10020</value>  
    </property>  

    <property>  
        <name>mapreduce.jobhistory.webapp.address</name>  
        <value>master:19888</value>  
    </property> 

slaves

node1
node2

yarn-site.xml

 <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>master</value>
    </property>

    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>

    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>

    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>604800</value>
    </property>

[root@master soft]# cd /usr/local/soft/
[root@master soft]# scp -r hadoop-2.7.6/ node1:`pwd`
[root@master soft]# scp -r hadoop-2.7.6/ node2:`pwd`

[root@master soft]# hdfs namenode -format

[root@master soft]# start-all.sh

master:

[root@master soft]# jps
5536 NameNode
6166 Jps
5737 SecondaryNameNode
5903 ResourceManager

node1:

[root@node1 soft]# jps
3285 NodeManager
3176 DataNode
3422 Jps

node2:

[root@node2 soft]# jps
3328 NodeManager
3219 DataNode
3465 Jps

http://master:50070

 http://master:8088

 [root@master soft]# ls
0??  data          jdk1.8.0_171  redis        shell    show
A??  hadoop-2.7.6  packages      redis-6.2.6  shell01  test.txt
[root@master soft]# cd hadoop-2.7.6/
[root@master hadoop-2.7.6]# hdfs dfs -mkdir /testHDFSshell1
[root@master hadoop-2.7.6]# hdfs dfs -mkdir /testHDFSshell2
[root@master hadoop-2.7.6]# hdfs dfs -mkdir /testHDFSshell3
[root@master hadoop-2.7.6]# hdfs dfs -mkdir /testHDFSshell4