大数据开发之windows安装hadoop教程

茅草屋的屋

已于 2022-12-16 15:15:49 修改

阅读量2k

点赞数 3

分类专栏：大数据开发教程文章标签： hadoop 大数据

于 2022-12-15 16:59:46 首次发布

本文链接：https://blog.csdn.net/yjjhk/article/details/128331352

版权

大数据开发教程专栏收录该内容

3 篇文章 0 订阅

订阅专栏

第一步安装JDK

第二步安装Hadoop

下载hadoop地址：http://archive.apache.org/dist/hadoop/common/hadoop-3.2.2/ 2. 下载hadoop-winutils https://github.com/cdarlint/winutils

将红色方框里面的文件复制到hadoop-3.2.2\bin目录中，hadoop主要基于linux编写，这个winutil.exe主要用于模拟linux下的目录环境。因此hadoop放在windows下运行的时候，需要这个辅助程序才能运行。

第三步配置hadoop

配置环境变量：
新增系统变量 HADOOP_HONE

编辑系统变量中的Path
进入hadoop安装文件夹下的etc/hadoop文件夹，修改以下文件内容：

（1）core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->


<configuration>
    <property>
        <name>fs.default.name</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>

（2）hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->


<configuration>
    <!-- 这个参数设置为1，因为是单机版hadoop -->
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
	 <property>
        <name>dfs.namenode.name.dir</name>
          <!-- 需要先创建此目录 -->
        <value>file:/G:/hadoop/data/dfs/namenode</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
         <!-- 需要先创建此目录 -->
        <value>file:/G:/hadoop/data/dfs/datanode</value>
    </property>
	 <property>
        <name>dfs.http.address</name>
        <value>0.0.0.0:50070</value>
    </property>
	  <property>
        <name>dfs.permissions</name>
         <!-- 以便在网页中可以创建、上传文件 -->
        <value>false</value>
    </property>
</configuration>

（3）mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
   <property>
      <name>mapreduce.framework.name</name>
      <value>yarn</value>
    </property>

</configuration>

（4）yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>
  <property>
     <name>yarn.nodemanager.aux-services</name>
     <value>mapreduce_shuffle</value>
   </property>
  <property>
     <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
     <value>org.apache.hadoop.mapred.ShuffleHandler</value>
   </property>
</configuration>