hadoop 配置文件-CSDN博客

本文链接：https://blog.csdn.net/u014384314/article/details/122805163

1.[GSPAgent@azurepana01 hadoop]$ cd $HADOOP_HOME/etc/hadoop

[GSPAgent@azurepana01 hadoop]$ cat core-site.xml


<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/cluster/hadoop/tmp</value>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://azurepana01:9820</value>
    </property>
    <property>
        <!-- 指定zookeeper所在的节点 -->
        <name>ha.zookeeper.quorum</name>
        <value>azurepana02:2181,azurepana03:2181,azurepana04:2181</value>
    </property>

　　<property>
　　　　<name>hadoop.proxyuser.GSPAgent.hosts</name>
　　　　<value>*</value>
　　</property>
　　<property>
　　　　<name>hadoop.proxyuser.GSPAgent.groups</name>
　　　　<value>*</value>
　　</property>

</configuration>

2.[GSPAgent@azurepana01 hadoop]$ cat yarn-site.xml

[GSPAgent@azurepana01 hadoop]$ cat yarn-site.xml
<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Site specific YARN configuration properties -->


<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>

    <property>
        <!-- 配置yarn为高可用 -->
        <name>yarn.resourcemanager.ha.enabled</name>
        <value>true</value>
    </property>
    <property>
        <!-- 集群的唯一标识 -->
        <name>yarn.resourcemanager.cluster-id</name>
        <value>mycluster</value>
    </property>
    <property>
        <!--  ResourceManager ID -->
        <name>yarn.resourcemanager.ha.rm-ids</name>
        <value>rm1,rm2</value>
    </property>
    <property>
        <!-- 指定ResourceManager 所在的节点 -->
        <name>yarn.resourcemanager.hostname.rm1</name>
        <value>azurepana01</value>
    </property>
    <property>
        <!-- 指定ResourceManager 所在的节点 -->
        <name>yarn.resourcemanager.hostname.rm2</name>
        <value>azurepana02</value>
    </property>
    <property>
        <!-- 指定ResourceManager Http监听的节点 -->
        <name>yarn.resourcemanager.webapp.address.rm1</name>
        <value>azurepana01:8088</value>
    </property>
    <property>
        <!-- 指定ResourceManager Http监听的节点 -->
        <name>yarn.resourcemanager.webapp.address.rm2</name>
        <value>azurepana02:8088</value>
    </property>
    <property>
        <!-- 指定zookeeper所在的节点 -->
      <!--  <name>yarn.resourcemanager.zk-address</name> -->
        <name>hadoop.zk.address</name>
        <value>azurepana02:2181,azurepana03:2181,azurepana04:2181</value>
    </property>

    <property>
        <!-- 启用节点的内容和CPU自动检测，最小内存为1G -->
        <name>yarn.nodemanager.resource.detect-hardware-capabilities</name>
        <value>true</value>
    </property>

<!--
<property>
     <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
     <value>0.9</value>
 </property>
-->

  <property>
        <name>yarn.scheduler.capacity.root.default.capacity</name>
        <value>100</value>
  </property>

<property>
     <name>yarn.nodemanager.resource.memory-mb</name>
     <value>21504</value>
 </property>

<property>
     <name>yarn.scheduler.minimum-allocation-mb</name>
     <value>1024</value>
 </property>
<property>
     <name>yarn.scheduler.maximum-allocation-mb</name>
     <value>21504</value>
 </property>

<property>
     <name>yarn.nodemanager.resource.cpu-vcores</name>
     <value>12</value>
 </property>
<property>
     <name>yarn.scheduler.maximum-allocation-vcores</name>
     <value>12</value>
 </property>


</configuration>

3.[GSPAgent@azurepana01 hadoop]$ cat yarn-env.sh

export JAVA_HOME=/usr/java/default

4.[GSPAgent@azurepana01 hadoop]$ cat hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->


<configuration>
    <property><!--namenode持久存储名字空间及事务日志的本地文件系统路径-->
        <name>dfs.namenode.name.dir</name>
        <value>/opt/cluster/hadoop/dfs/name</value>
     </property>
     <property><!--DataNode存放块数据的本地文件系统路径-->
         <name>dfs.datanode.data.dir</name>
         <value>/opt/cluster/hadoop/dfs/data</value>
     </property>
     <property><!--数据需要备份的数量，不能大于集群的机器数量，默认为3-->
        <name>dfs.replication</name>
        <value>2</value>
    </property>

　<property><!--必须设置为true，否则就不能通过web访问hdfs上的文件信息-->
　　　　<name>dfs.webhdfs.enabled</name>
　　　　<value>true</value>
　　</property>

    <property>
        <!--这里配置逻辑名称，可以随意写 -->
        <name>dfs.nameservices</name>
        <value>mycluster</value>
    </property>
    <property>
        <!-- 禁用权限 -->
        <name>dfs.permissions.enabled</name>
        <value>false</value>
    </property>
    <property>
        <!-- 配置namenode 的名称，多个用逗号分割  -->
        <name>dfs.ha.namenodes.mycluster</name>
        <value>nn1,nn2</value>
    </property>
    <property>
        <!-- dfs.namenode.rpc-address.[nameservice ID].[name node ID] namenode 所在服务器名称和RPC监听端口号  -->
        <name>dfs.namenode.rpc-address.mycluster.nn1</name>
        <value>azurepana01:9820</value>
    </property>
    <property>
        <!-- dfs.namenode.rpc-address.[nameservice ID].[name node ID] namenode 所在服务器名称和RPC监听端口号  -->
        <name>dfs.namenode.rpc-address.mycluster.nn2</name>
        <value>azurepana02:9820</value>
    </property>
    <property>
        <!-- dfs.namenode.http-address.[nameservice ID].[name node ID] namenode 监听的HTTP协议端口 -->
        <name>dfs.namenode.http-address.mycluster.nn1</name>
        <value>azurepana0