Hadoop(05) HDFS Java 接口

FileSystem API 简单操作文件

package hadoop.hdfs;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.Before;
import org.junit.Test;

public class HDFSTest {
    private static FileSystem fileSystem = null;
    private final static int buffSize    = 4096;
    private final static boolean close   = true;

    @Before
    public void before() throws Exception {
        // 获取通用文件系统
        fileSystem = FileSystem.get(new URI("hdfs://Huaqing:9000"),
                new Configuration(), "root");
    }

    @Test
    public void _put() throws Exception {
        // "/test.c" 代表HDFS / 下的test.c 文件
        InputStream in = fileSystem.open(new Path("/test.c"));
        OutputStream out = new FileOutputStream(new File("D:/test.c"));

        // 原始拷贝文件
        // int readBytes = 0;
        // byte[] buffer = new byte[4096];
        // try {
        //     while (-1 != (readBytes = in.read(buffer))) {
        //         out.write(buffer);
        //     }
        // } finally {
        //     out.close();
        //     in.close();
        // }

        // hadoop 提供的拷贝工具
        IOUtils.copyBytes(in, out, buffSize, close);
    }

    @Test
    public void _get() throws Exception {
        boolean overwrite = true;
        InputStream in = new BufferedInputStream(
                new FileInputStream(new File("D:/pom.xml")));
        OutputStream out = fileSystem.create(new Path("/pom.xml"), overwrite);
        IOUtils.copyBytes(in, out, buffSize, close);
    }

    @Test
    public void _rm() throws Exception {
        boolean recursive = true; // 递归
        boolean result = fileSystem.delete(new Path("/del_dir"), recursive);
        System.out.println("rm -rf del_dir success " + result);
    }

    @Test
    public void _mkdir() throws Exception {
        boolean result = fileSystem.mkdirs(new Path("/del_dir"));
        System.out.println("mkdir success " + result);
    }

    @Test
    public void _ls() throws Exception {
        FileStatus[] lses = fileSystem.listStatus(new Path("/"));
        for (FileStatus fileStatus : lses) {
            System.out.println(fileStatus);
        }
    }

}

• 在windows异常
  java.lang.IllegalArgumentException: java.net.UnknownHostException: master
  解决：在window下的 C:\Windows\System32\drivers\etc\hosts 配置与hdfs的关系映射。如：192.168.1.103 master