java操作hadoop hdfs，实现文件上传下载demo

本文主要参考了Hadoop HDFS文件系统通过java FileSystem 实现上传下载等，并实际的做了一下验证。代码与引用的文章差别不大，现列出来作为备忘。

import java.io.*;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.Test;

public class HdfsDemo {

    private static String SOURCE_PATH = "C:\\settings.jar";
    private static String DEST_PATH = "/hbase/upload/settings.jar";
    private static String MASTER_URI = "hdfs://192.168.209.129:9000";

    /**
     * 测试上传文件
     * 如果下述代码抛出异常：
     * org.apache.hadoop.security.AccessControlException: Permission denied: user=xxx, access=WRITE。
     * 则表明上传文件的用户没有权限。解决方式如下：
     * 1.配置环境变量HADOOP_USER_NAME， 并设置值为有访问hadoop集群有权限的用户，比如：hadoop。源码调试发现hadoop会首先从环境变量中去查找上传文件的用户名。查找不到会议pc的名字作为用户名。
     * 2.赋予hdfs上的文件夹/hbase/upload/读写权限。 命令是：hadoop fs -chmod 777  /hbase/upload。这样，可以保证上传文件到该目录成功，但上传到其他目录依然无权限。
     *
     * @throws Exception
     * @author lihong10 2017年12月9日 上午10:42:30
     */
    @Test
    public void testUpload() throws Exception {

        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI(MASTER_URI), conf);
        InputStream in = new FileInputStream(SOURCE_PATH);
        OutputStream out = fs.create(new Path(DEST_PATH), new Progressable() {
            @Override
            public void progress() {
                System.out.println("上传完一个设定缓存区大小容量的文件！");
            }
        });
        IOUtils.copyBytes(in, out, conf);

      /*byte[] buffer = new byte[1024];
        int len = 0;
		while((len=in.read(buffer))>0){
			out.write(buffer, 0, len);
		} 
		out.flush();
		in.close();
		out.close();*/
    }


    /**
     * 测试下载文件
     * @throws Exception
     */
    @Test
    public void testDownload() throws Exception {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI(MASTER_URI), conf);
//		InputStream in = new FileInputStream("e:\\hadoop.avi");
//		OutputStream out = fs.create(new Path("/demo/b.avi"));
        InputStream in = fs.open(new Path(DEST_PATH));
        OutputStream out = new FileOutputStream("E:\\XXXX.jar");
        IOUtils.copyBytes(in, out, conf);

      /*byte[] buffer = new byte[1024];
		int len = 0;
		while((len=in.read(buffer))>0){
			out.write(buffer, 0, len);
		} 
		out.flush();
		in.close();
		out.close();*/
    }

    /**
     * 复制文件到远程文件系统，也可以看做是上传
     * @param conf
     * @param uri
     * @param local
     * @param remote
     * @throws IOException
     */
    public static void copyFile(Configuration conf, String uri, String local, String remote) throws IOException {
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        fs.copyFromLocalFile(new Path(local), new Path(remote));
        System.out.println("copy from: " + local + " to " + remote);
        fs.close();
    }

    @Test
    public void testCopyFile() throws IOException {
        // 使用命令：hadoop fs -ls  /hbase/upload， 可以查看复制到/hbase/upload目录下的文件
        copyFile(new Configuration(), MASTER_URI, SOURCE_PATH, "/hbase/upload/settings01.jar");
    }

    /**
     * 在hdfs文件系统下创建目录
     * @param conf
     * @param uri
     * @param remoteFile
     * @throws IOException
     */
    public static void makeDir(Configuration conf, String uri, String remoteFile) throws IOException {
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        Path path = new Path(remoteFile);

        fs.mkdirs(path);
        System.out.println("创建文件夹" + remoteFile);
    }

    @Test
    public void testMakeDir() throws IOException {
        //可以通过：hadoop fs -ls  /hbase/upload。   查看创建的目录subdir01
        makeDir(new Configuration(), MASTER_URI, "/hbase/upload/subdir01");
    }


    /**
     * 删除文件
     * @param conf
     * @param uri
     * @param filePath
     * @throws IOException
     */
    public static void delete(Configuration conf, String uri, String filePath) throws IOException {
        Path path = new Path(filePath);
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        fs.deleteOnExit(path);
        System.out.println("Delete: " + filePath);
        fs.close();
    }

    @Test
    public void testDelete() throws IOException {
        delete(new Configuration(), MASTER_URI, "/hbase/upload/settings01.jar");
    }


    /**
     * 查看文件内容
     * @param conf
     * @param uri
     * @param remoteFile
     * @throws IOException
     */
    public static void cat(Configuration conf, String uri, String remoteFile) throws IOException {
        Path path = new Path(remoteFile);
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        FSDataInputStream fsdis = null;
        System.out.println("cat: " + remoteFile);
        try {
            fsdis = fs.open(path);
            IOUtils.copyBytes(fsdis, System.out, 4096, false);
        } finally {
            IOUtils.closeStream(fsdis);
            fs.close();
        }
    }

    @Test
    public void testCat() throws IOException {
        //hbase/hbase.version 这个文件已经存在
        cat(new Configuration(), MASTER_URI, "/hbase/hbase.version");
    }

    /**
     * 查看目录下面的文件
     * @param conf
     * @param uri
     * @param folder
     * @throws IOException
     */
    public static void ls(Configuration conf, String uri, String folder) throws IOException {
        Path path = new Path(folder);
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        FileStatus[] list = fs.listStatus(path);
        System.out.println("ls: " + folder);
        System.out.println("==========================================================");
        for (FileStatus f : list) {
            System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDirectory(), f.getLen());
        }
        System.out.println("==========================================================");
        fs.close();
    }

    @Test
    public void testLs() throws IOException {
        ls(new Configuration(), MASTER_URI, "/hbase/upload");
    }

}