编辑 /etc/profile 文件
vim /etc/profile
添加以下内容
export HADOOP_HOME=/usr/local/hadoop/hadoop
export PATH= P A T H : PATH: PATH:HADOOP_HOME/bin:$HADOOP_HOME/sbin
四、修改配置文件
1、检查磁盘空间
首先查看磁盘挂载空间,避免hadoop的数据放在挂载空间小的目录
df -h
磁盘一共800G,home目录占了741G,故以下配置目录都会以 /home开头
2、修改配置文件
worker
hadoop003
hadoop004
hadoop005
core-site.xml
<?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>fs.defaultFS
hdfs://ns1
hadoop.tmp.dir
/home/cluster/hadoop/data/tmp
io.file.buffer.size
131072
Size of read/write buffer used in SequenceFiles
ha.zookeeper.quorum
hadoop001:2181,hadoop002:2181,hadoop003:2181
DFSZKFailoverController
hadoop.proxyuser.root.hosts
*
hadoop.proxyuser.root.groups
*
hadoop-env.sh
export HDFS_NAMENODE_OPTS=“-XX:+UseParallelGC -Xmx4g”
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_JOURNALNODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
export JAVA_HOME=/usr/java/jdk1.8
hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>dfs.namenode.name.dir
/home/cluster/hadoop/data/nn
dfs.datanode.data.dir
/home/cluster/hadoop/data/dn
dfs.journalnode.edits.dir
/home/cluster/hadoop/data/jn
dfs.nameservices
ns1
dfs.ha.namenodes.ns1
hadoop001,hadoop002
dfs.namenode.rpc-address.ns1.hadoop001
hadoop001:8020
dfs.namenode.http-address.ns1.hadoop001
hadoop001:9870
dfs.namenode.rpc-address.ns1.hadoop002
hadoop002:8020
dfs.namenode.http-address.ns1.hadoop002
hadoop002:9870
dfs.ha.automatic-failover.enabled.ns1
true
dfs.client.failover.proxy.provider.ns1
org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
dfs.permissions.enabled
false
dfs.replication
2
dfs.blocksize
HDFS blocksize of 128MB for large file-systems
dfs.namenode.handler.count
100
More NameNode server threads to handle RPCs from large number of DataNodes.
dfs.namenode.shared.edits.dir
qjournal://hadoop001:8485;hadoop002:8485;hadoop003:8485/ns1
dfs.ha.fencing.methods
sshfence
dfs.ha.fencing.ssh.private-key-files
/root/.ssh/id_rsa
mapred-site.xml
<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>mapreduce.framework.name
yarn
Execution framework set to Hadoop YARN.
mapreduce.map.memory.mb
4096
Larger resource limit for maps.
mapreduce.map.java.opts
-Xmx4096M
Larger heap-size for child jvms of maps.
mapreduce.reduce.memory.mb
4096
Larger resource limit for reduces.
mapreduce.reduce.java.opts
-Xmx4096M
Larger heap-size for child jvms of reduces.
mapreduce.task.io.sort.mb
2040
Higher memory-limit while sorting data for efficiency.
mapreduce.task.io.sort.factor
400
More streams merged at once while sorting files.
mapreduce.reduce.shuffle.parallelcopies
200
Higher number of parallel copies run by reduces to fetch outputs from very large number of maps.
mapreduce.jobhistory.address
hadoop001:10020
MapReduce JobHistory Server host:port.Default port is 10020
mapreduce.jobhistory.webapp.address
hadoop001:19888
MapReduce JobHistory Server Web UI host:port.Default port is 19888.
mapreduce.jobhistory.intermediate-done-dir
/tmp/mr-history/tmp
Directory where history files are written by MapReduce jobs.
mapreduce.jobhistory.done-dir
/tmp/mr-history/done
Directory where history files are managed by the MR JobHistory Server.
yarn-site.xml
<?xml version="1.0"?>yarn.resourcemanager.ha.enabled
true
yarn.resourcemanager.ha.automatic-failover.enabled
true
yarn.resourcemanager.ha.automatic-failover.embedded
true
yarn.resourcemanager.cluster-id
yarn-rm-cluster
yarn.resourcemanager.ha.rm-ids
rm1,rm2
yarn.resourcemanager.hostname.rm1
hadoop001
yarn.resourcemanager.hostname.rm2
hadoop002
yarn.resourcemanager.recovery.enabled
true
yarn.resourcemanager.zk.state-store.address
hadoop001:2181,hadoop002:2181,hadoop003:2181
yarn.resourcemanager.zk-address
hadoop001:2181,hadoop002:2181,hadoop003:2181
yarn.resourcemanager.address.rm1
hadoop001:8032
yarn.resourcemanager.address.rm2
hadoop002:8032
yarn.resourcemanager.scheduler.address.rm1
hadoop001:8034
yarn.resourcemanager.webapp.address.rm1
hadoop001:8088
yarn.resourcemanager.scheduler.address.rm2
hadoop002:8034
yarn.resourcemanager.webapp.address.rm2
hadoop002:8088
yarn.acl.enable
true
Enable ACLs? Defaults to false.
yarn.admin.acl
*
yarn.log-aggregation-enable
false
Configuration to enable or disable log aggregation
yarn.resourcemanager.hostname
hadoop001
host Single hostname that can be set in place of setting all yarn.resourcemanager*address resources. Results in default ports for ResourceManager components.
yarn.scheduler.minimum-allocation-mb
1024
saprk调度时一个container能够申请的最小资源,默认值为1024MB
yarn.scheduler.maximum-allocation-mb
28672
saprk调度时一个container能够申请的最大资源,默认值为8192MB
yarn.nodemanager.resource.memory-mb
28672
nodemanager能够申请的最大内存,默认值为8192MB
yarn.app.mapreduce.am.resource.mb
28672
AM能够申请的最大内存,默认值为1536MB
yarn.nodemanager.log.retain-seconds
10800
yarn.nodemanager.log-dirs
/home/cluster/yarn/log/1,/home/cluster/yarn/log/2,/home/cluster/yarn/log/3
网上学习资料一大堆,但如果学到的知识不成体系,遇到问题时只是浅尝辄止,不再深入研究,那么很难做到真正的技术提升。
一个人可以走的很快,但一群人才能走的更远!不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人,都欢迎加入我们的的圈子(技术交流、学习资源、职场吐槽、大厂内推、面试辅导),让我们一起学习成长!
ame>yarn.app.mapreduce.am.resource.mb
28672
AM能够申请的最大内存,默认值为1536MB
yarn.nodemanager.log.retain-seconds
10800
yarn.nodemanager.log-dirs
/home/cluster/yarn/log/1,/home/cluster/yarn/log/2,/home/cluster/yarn/log/3
[外链图片转存中…(img-LJW5LKHQ-1714300319959)]
[外链图片转存中…(img-PWG1J2EB-1714300319959)]
网上学习资料一大堆,但如果学到的知识不成体系,遇到问题时只是浅尝辄止,不再深入研究,那么很难做到真正的技术提升。
一个人可以走的很快,但一群人才能走的更远!不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人,都欢迎加入我们的的圈子(技术交流、学习资源、职场吐槽、大厂内推、面试辅导),让我们一起学习成长!