hadoop 开发环境搭建 （windows \ eclipse）

搭建开发环境所需文件已经上传到 hadoop开发环境搭建所需文件

1. 将hadoop-2.6.2.tar.gz 解压到 D 盘
2. 配置环境变量(系统变量) HADOOP_HOME = D:\hadoop-2.6.2
3. 配置环境变量(系统变量)  HADOOP_USER_NAME = (服务器hadoop的用户名)
4. 将hadoop-eclipse-plugin-2.2.0.jar 放到 eclipse  plugins文件夹下 
5. 打开eclipse   window ->> show view ->> Map/Reduce locations ->> edit hadoop locations  
6. Map/Reduce Master  -  host:hadoop所在服务器IP   port:50020
7. DFS Master         -  host:hadoop所在服务器IP   port:9000
8. User name :服务器hadoop的用户名
9. socks proxy  默认即可
10. 将winutils.exe文件放到D:\hadoop-2.6.2\bin目录下

hadoop hdfs demo

hdfs相关jar包  maven 配置

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>2.6.2</version>
    </dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>2.6.2</version>
</dependency>

JAVA代码

package listDemo;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;


public class Demo {

    public void WriteFile(String hdfs) throws IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(hdfs), conf);
        FSDataOutputStream hdfsOutStream = fs.create(new Path(hdfs));
        hdfsOutStream.writeChars("hello");
        hdfsOutStream.close();
        fs.close();
    }

    public void ReadFile(String hdfs) throws IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(hdfs), conf);
        FSDataInputStream hdfsInStream = fs.open(new Path(hdfs));

        byte[] ioBuffer = new byte[1024];
        int readLen = hdfsInStream.read(ioBuffer);
        while (readLen != -1) {
            System.out.write(ioBuffer, 0, readLen);
            readLen = hdfsInStream.read(ioBuffer);
        }
        hdfsInStream.close();
        fs.close();
    }

    // 在指定位置新建一个文件，并写入字符
    public static void WriteToHDFS(String file, String words) throws IOException, URISyntaxException {
        Configuration conf = new Configuration();
        conf.set("fs.default.name","192.168.30.178:9000");         //hadoop所在服务器IP
        conf.set("mapred.job.tracker","192.168.30.178:9001");    //hadoop所在服务器IP
        FileSystem fs = FileSystem.get(URI.create(file), conf);
        Path path = new Path(file);
        FSDataOutputStream out = fs.create(path); // 创建文件

        // 两个方法都用于文件写入，好像一般多使用后者
        /*out.writeBytes(words);*/
        out.write(words.getBytes("UTF-8"));

        out.close();
        // 如果是要从输入流中写入，或是从一个文件写到另一个文件（此时用输入流打开已有内容的文件）
        // 可以使用如下IOUtils.copyBytes方法。
        // FSDataInputStream in = fs.open(new Path(args[0]));
        // IOUtils.copyBytes(in, out, 4096, true) //4096为一次复制块大小，true表示复制完成后关闭流
    }

    public static void main(String[] args) throws IOException {
        String hdfs = "hdfs://192.168.30.178:9000/test/test.txt";     //hadoop所在服务器IP
        Demo t = new Demo();
        String words = "\n    test xxxxxxxxx";  
        try {
            WriteToHDFS(hdfs, words);
        } catch (URISyntaxException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }  
/*        t.WriteFile(hdfs);
        t.ReadFile(hdfs);*/
    }
}