hdfsAPI

package test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.example.GroupReadSupport;

import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class HdfsAPI {
    // HDFS文件系统服务器的地址以及端口
    // hdfs://172.20.131.25:8020/data_source
    public static final String HDFS_PATH = "hdfs://node2:8020";
    //配置文件
    Configuration conf = new Configuration();
    //通过这种方式设置java客户端身份
    FileSystem fs = FileSystem.get(new URI(HDFS_PATH), conf, "root");

    //抛出最大异常
    public HdfsAPI() throws Exception {
    }

    public static void main(String[] args) throws Exception {
        HdfsAPI hdfsAPI = new HdfsAPI();
        Path inPath = new Path("/user/hbase");

//        hdfsAPI.getDirectoryFromHdfs(inPath);
//        hdfsAPI.parquetReader(inPath);
    }

    public void cat(Path filePath) throws Exception {
        // 读取文件
        FSDataInputStream in = fs.open(filePath);

        // 将文件内容输出到控制台上，第三个参数表示输出多少字节的内容
        IOUtils.copyBytes(in, System.out, 1024*1024*100);
        System.out.println();
        in.close();
    }

    public void mkdir(String dirPath)throws Exception{
        // 需要传递一个Path对象
        boolean bool = fs.mkdirs(new Path(dirPath));
        if (bool) {
            System.out.println("创建成功");
        }else {
            System.out.println("创建失败");
        }
    }

//    public void delete(Path dirPath)throws Exception{
//        // 第二个参数指定是否要递归删除，false=否，true=是
//        boolean bool = fs.delete(dirPath, true);
//        if (bool) {
//            System.out.println("删除成功");
//        }else {
//            System.out.println("删除失败");
//        }
//    }

    public void getDirectoryFromHdfs(Path direPath) throws IOException {
        FileStatus[] filelist = fs.listStatus(direPath);
//        List<Map<String, Object>> mapList = new ArrayList<>();
        for (FileStatus fileStatus : filelist) {
//            Map<String, Object> map = new HashMap<>();
//            map.put("Name", fileStatus.getPath().getName());
//            map.put("Size", fileStatus.getLen());
//            map.put("Type", (fileStatus.isDirectory() ? "dir" : "file"));
//            map.put("Path", fileStatus.getPath());
//            mapList.add(map);
            System.out.println("_________" + direPath + "目录下所有文件______________");
            System.out.println("文件名字:" + fileStatus.getPath().getName());
            System.out.println("文件类型:" + (fileStatus.isDirectory() ? "dir" : "file"));
            System.out.println("文件路径："+fileStatus.getPath());
            System.out.println("块的大小："+fileStatus.getBlockSize());
            System.out.println("文件所有者："+fileStatus.getOwner()+":"+fileStatus.getGroup());
            System.out.println("文件权限："+fileStatus.getPermission());
            System.out.println("文件长度："+fileStatus.getLen());
            System.out.println("备份数："+fileStatus.getReplication());
            System.out.println("修改时间："+fileStatus.getModificationTime());
        }
        fs.close();
    }

    public void copyToLocalFile() throws Exception{
        Path src = new Path("/data_source/");
        Path dst = new Path("D:\\test");
        fs.copyToLocalFile(false, src, dst, true);
           }

    public void parquetReader(Path inPath) throws Exception{
        // 读取文件
        FileStatus fileStatus = fs.getFileStatus(inPath);

        GroupReadSupport readSupport = new GroupReadSupport();
        ParquetReader reader = new ParquetReader(fileStatus.getPath(),readSupport);
        Group line=null;
        while((line= (Group) reader.read())!=null){
            System.out.println(line.toString());

        }

        System.out.println("读取结束");

    }

}