Java API 操作HDFS文件大集合

public class HdfsTest02 {

    static Configuration conf=new Configuration();


//创建文件并写入
   @Test
    public void create() throws IOException, URISyntaxException {
       FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.201:8020"),conf);
       FSDataOutputStream outputStream = fs.create(new Path("/user/new12/afdsd.txt"));
       outputStream.write("hello hadoop".getBytes());
       outputStream.flush();
       outputStream.close();
   }

    /**
     * 查看HDFS文件的内容
     * @throws Exception
     */
    @Test
    public void cat() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.201:8020"),conf);
        FSDataInputStream inputStream = fs.open(new Path("/user/new12/a.txt"));
        IOUtils.copyBytes(inputStream,System.out,1024);
        inputStream.close();

    }

    /**
     * 删除文件
     * @throws Exception
     */
    @Test
    public void delete() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.201:8020"),conf);
        fs.delete(new Path("/user/new12/a.txt"),true);
    }

    /**
     * 文件重命名
     * @throws Exception
     */
    @Test
    public void rename() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.201:8020"),conf);
        Path oldPath = new Path("/user/new12/a.txt");
        Path newPath = new Path("/user/new12/b.txt");
        fs.rename(oldPath,newPath);
    }

    /**
     * 上传本地小文件到HDFS
     * @throws Exception
     */
    @Test
    public void copyFromLocalFile() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.201:8020"),conf);
        Path localPath = new Path("F:\\sl");
        //ls为小文件的一个集合
        Path hdfsPath = new Path("/user/new12");

        fs.copyFromLocalFile(localPath,hdfsPath);
    }

    /**
     * 上传本地大文件且需要进度条
     * @throws Exception
     */
    @Test
    public void copyFromLocalFileWithProgress() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.201:8020"),conf);
        InputStream in = new BufferedInputStream(new FileInputStream(new File("C:\\Users\\Administrator\\Desktop\\3HDFS讲义.doc")));
        FSDataOutputStream outputStream = fs.create(new Path("/user/new12/asd.doc"),
                new Progressable() {
                    @Override
                    public void progress() {
                        System.out.print(".");   // 带进度提醒
                    }
                });
        IOUtils.copyBytes(in,outputStream,4096);
    }


    /**
     * 下载HDFS文件到本地
     * @throws Exception
     */
    @Test
    public void copyToLocalFile() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.201:8020"),conf);
        Path localPath = new Path("F:\\sl");
        Path hdfsPath = new Path("/user/new12/asd.doc");
        fs.copyToLocalFile(hdfsPath,localPath);
    }


}