伪分布hadoop涉及的配置文件

wh_xia_jun

已于 2022-01-31 23:01:31 修改

阅读量2.4k

点赞数

分类专栏：软件工程文章标签： hadoop hdfs big data

于 2022-01-29 13:02:45 首次发布

本文链接：https://blog.csdn.net/wh_xia_jun/article/details/122741831

版权

软件工程专栏收录该内容

3 篇文章 0 订阅

订阅专栏

core-site.xml

<configuration>

        <property>
                        <name>fs.defaultFS</name>
                        <value>hdfs://localhost:9000</value>
                        <description>默认的HDFS端口，用于NameNode与DataNode之间到的通讯，IP为NameNode的地址</description>
        </property>


        <property>
                <name>hadoop.security.authorization</name>
                <value>false</value>
                <description></description>
         </property>
</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
 <property>
 <name>dfs.replication</name>
        <value>1</value>
</property>

<property>
   <name>dfs.namenode.name.dir</name>
   <value>/app/hdfs/namenode</value>
</property>

<property>
   <name>dfs.datanode.data.dir</name>
   <value>/app/hdfs/datanode</value>
</property>

<property>
    <name>dfs.permissions</name>
    <value>false</value>
</property>
</configuration>

mapred-site.xml

<configuration>


        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>


        <property>
                 <name>mapreduce.application.classpath</name>
                 <value>
                        $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*,
                </value>
                 <description>设置MapReduce资源调度类路径，如果不设置可能会报错</description>
        </property>`
</configuration>

yarn-site.xml


<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>

        <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
        <description>NodeManager上运行的附属服务。需配置成mapreduce_shuffle，才可运行MapReduce程序</description>
    </property>

        <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
        <description></description>
    </property>

</configuration>

hadoop-env.sh

export HADOOP JAVA_HOME=/usr/lib/jvm/java
export HADOOP_MAPRED_HOME=/app/hadoop
export HADOOP_NAMENODE_USER=root
export HADOOP_DATANODE_USER=root
export HADOOP_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

ps:5步搞定 Windows 10 docker WSL2的镜像存储位置的修改

ps2:docker-compose.yarm

version: '3'
services:
  master1:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D
    ports:
      - "9000:9000"
      - "9870:9870"
      - "8088:8088"
 
  master2:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D

    
  worker1:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D

  worker2:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D

  worker3:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D