Windows Hadoop 搭建

Hadoop Windows 环境

安装

https://github.com/MuhammadBilalYar/Hadoop-On-Window/wiki/Step-by-step-Hadoop-2.8.0-installation-on-Window-10

Hadoop 2.8.0 下载地址

http://www-eu.apache.org/dist/hadoop/common/hadoop-2.8.0/hadoop-2.8.0.tar.gz

Hadoop Configuration 下载地址

https://github.com/MuhammadBilalYar/HADOOP-INSTALLATION-ON-WINDOW-10/blob/master/Hadoop%20Configuration.zip

解压Hadoope 2.8.0

---

<configuration>
   <property>
       <name>fs.defaultFS</name>
       <value>hdfs://localhost:9000</value>
   </property>
</configuration>

---

<configuration>
   <property>
       <name>dfs.replication</name>
       <value>1</value>
   </property>
   <property>
       <name>dfs.namenode.name.dir</name>
       <value>D:\Local\hadoop-2.8.0\data\namenode</value>
   </property>
   <property>
       <name>dfs.datanode.data.dir</name>
       <value>D:\Local\hadoop-2.8.0\data\datanode</value>
   </property>
</configuration>

---

<configuration>
   <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
   </property>
   <property>
        <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>  
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
   </property>
</configuration>

hdfs namenode –format

hdfs namenode –format

启动

start-all.cmd

http://localhost:8088

#

http://localhost:50070

IDEA 配置

https://blog.csdn.net/u010171031/article/details/53024516

代码

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class Example1 {
    public Example1() {
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        FileSystem hdfs = FileSystem.get(conf);
        Path src = new Path("1.txt");
        Path dst = new Path("hdfs://localhost:9000/");
        hdfs.copyFromLocalFile(src, dst);
        System.out.println("Upload to " + conf.get("fs.default.name"));
        FileStatus[] files = hdfs.listStatus(dst);
        FileStatus[] var6 = files;
        int var7 = files.length;

        for(int var8 = 0; var8 < var7; ++var8) {
            FileStatus file = var6[var8];
            System.out.println(file.getPath());
        }

    }
}

错误

Exception in thread "main" java.lang.IllegalArgumentException: Wrong FS: hdfs://localhost:9000/, expected: file:///

解决方法

将hdfs-site.xml 和 core-site.xml 拷贝到工程src下面