JAVA客户端操作HDFS的常用API

测试类：

package com._51doit.test;

import com._51doit.test.utils.FsUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Test;

import java.io.*;
import java.net.URI;

public class Test01 {

    public static void main(String[] args) throws Exception {

        Configuration config = new Configuration();
        FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:9000"), config,"root");
        boolean flag = fs.mkdirs(new Path("/aaa/bbb"));

        fs.close();if(flag){
            System.out.println("创建成功");
        }else{
            System.out.println("创建失败");
        }


    }

    //下载 download
    @Test
    public void download() throws Exception {
        FileSystem fs = FsUtil.getFs();
        Path path1 = new Path("/a.txt");
        Path path2 = new Path("d://");
        fs.copyToLocalFile(path1,path2);
        fs.close();
    }

    //上传 upload
    @Test
    public void upload()throws Exception{
        FileSystem fs = FsUtil.getFs();
        Path localPath = new Path("d://MachineCode.xml");
        Path hdfsPath = new Path("/aaa/bbb");
        fs.copyFromLocalFile(localPath,hdfsPath);
        fs.close();

    }

    /**
     * 移动   rename
     * 注意：目标文件夹必须要存在才能移动
     * @throws Exception
     */
    @Test
    public void move() throws Exception{
        FileSystem fs = FsUtil.getFs();
        Path sourcePath = new Path("/MachineCode.xml");
        Path targetPath = new Path("/bbb/a.txt");

        //判断文件夹是否存在,不存在则创建
        Path dir = new Path("/bbb");
        if(!fs.exists(dir)){
            fs.mkdirs(dir);
        }

        fs.rename(sourcePath,targetPath);
        fs.close();
    }

    //删除文件夹
    @Test
    public void delete() throws Exception {
        FileSystem fs = FsUtil.getFs();
        Path path = new Path("/bbb");
        fs.delete(path,true);
        fs.close();
    }

    //读取文件 open
    @Test
    public void readFile() throws Exception {
        FileSystem fs = FsUtil.getFs();
        Path filePath = new Path("/MachineCode.xml");
        FSDataInputStream fsis = fs.open(filePath);
        BufferedReader br = new BufferedReader(new InputStreamReader(fsis));
        String line;
        while ((line=br.readLine())!=null){
            System.out.println(line);
        }
        fsis.close();
        br.close();
        fs.close();
    }


    //追加数据
    @Test
    public void append() throws Exception {
        FileSystem fs = FsUtil.getFs();
        Path filePath = new Path("/MachineCode.xml");
        FSDataOutputStream fsos = fs.append(filePath);
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fsos));
        bw.write("===============追加的内容3\n");
        bw.write("===============追加的内容4\n");
        //fsos.close();  此处不需要关闭，不然报错java.nio.channels.ClosedChannelException
        bw.close();
        fs.close();
    }

    //遍历目录(只查看文件，不包含文件夹)
    @Test
    public void catFile() throws Exception {
        FileSystem fs = FsUtil.getFs();
        Path path = new Path("/");
        RemoteIterator<LocatedFileStatus> fileIterator = fs.listFiles(path, true);//true ：递归遍历
        while (fileIterator.hasNext()){
            LocatedFileStatus file = fileIterator.next();
            // 文件的路径
            Path filePath = file.getPath();
            // 从路径中获取文件名
            String name = filePath.getName();
            //获取文件的大小
            long len = file.getLen();
            short replication = file.getReplication(); // 副本数
            long blockSize = file.getBlockSize();  //切块大小
            // 获取数据块信息
            BlockLocation[] locations = file.getBlockLocations();
            for (BlockLocation blockLocation : locations) {
                System.out.println("文件块信息："+blockLocation);
                // 获取数据块所在的机器
               //String[] hosts = blockLocation.getHosts();
                String[] names = blockLocation.getNames() ;
                for (String name2 : names) {
                    //192.168.133.101:50010
                    System.out.println("文件块所在机器："+name2);
                }
            }
            System.out.println("文件路径："+filePath +"  文件名："+name+"  文件大小："+len+"  副本个数: "+replication+"  块大小: "+blockSize);
        }
        fs.close();
    }

    //遍历目录(包括文件夹)
    @Test
    public void catDir() throws Exception {
        FileSystem fs = FsUtil.getFs();
        Path path = new Path("/");
        FileStatus[] fileStatus = fs.listStatus(path);
        digui(fileStatus);
        fs.close();
    }

    private void recursionCatDir(Path path) throws Exception {
        FileSystem fs = FsUtil.getFs();
        FileStatus[] fileStatus = fs.listStatus(path);
        digui(fileStatus);
        fs.close();
    }

    private void digui(FileStatus[] fileStatus) throws Exception {
        for (FileStatus status : fileStatus)
            if (status.isDirectory()) {  // 文件夹
                String name = status.getPath().getName();
                System.out.println("文件夹："+name+" ,路径: "+status.getPath());
                //递归读取文件夹
                recursionCatDir(status.getPath());
            } else { // 文件
                Path filePath = status.getPath();
                String name = filePath.getName();//文件名
                long size = status.getLen(); //文件大小
                long blockSize = status.getBlockSize(); // 切块大小
                short repCount = status.getReplication();// 副本个数
                System.out.println("文件名：" + name+" ,路径: "+status.getPath() + " ,文件大小：" + size + "B ,切块大小：" + blockSize + "B ,副本个数：" + repCount);
            }
    }

    //设置配置文件方式
    public void configDemo(){
        // 在classPath路径下添加 hdfs-site.xml 等配置文件会自动读取
    }
}

工具类：

package com._51doit.test.utils;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;

import java.net.URI;

public class FsUtil {

    public static FileSystem getFs() throws Exception {
        Configuration conf = new Configuration();
        return FileSystem.newInstance(new URI("hdfs://linux01:9000"),conf,"root");
    }

}