[Hadoop] 递归的拷贝本地文件到HDFS文件系统中

递归的拷贝本地文件到hdfs文件系统中

//递归的拷贝本地文件到hdfs文件系统中
    @Test
    public static  void copyFileToLocal(String filename) throws URISyntaxException, IOException, InterruptedException {

        File f = new File(filename);
        File[] files = f.listFiles();
        for(File sf:files) {
            if (sf.isDirectory()) {
                copyFileToLocal(sf.getAbsolutePath());
            } else if (sf.isFile()) {
                FileInputStream in = new FileInputStream(sf);
                //获取文件去掉盘符的路径
                String path = sf.getAbsolutePath().split(":")[1].replace("\\","/");

                FSDataOutputStream out = fs.create(new Path(path));
                IOUtils.copyBytes(in,out,configuration);
                in.close();
                out.close();
            }
        }
    }
    //递归拷贝文件测试
    public static void main(String[] args) throws InterruptedException, IOException, URISyntaxException {

        configuration = new Configuration();

        fs = FileSystem.get(new URI("hdfs://hadoop101:9000"),configuration,"root");
        //拷贝的文件位置
        copyFileToLocal("D:\\A");
    }
}

本地目录结构



hdfs目录结构

hdfs不支持tree命令，可见文件是完整的拷贝过去了

其他常用api操作

public class HdfsDemo1 {
    static FileSystem fs;
    static Configuration configuration;
    @Before
    public void getConnection() throws URISyntaxException, IOException, InterruptedException {
        //获取文件系统配置
        System.out.println("执行api配置参数..");
        configuration = new Configuration();

        fs = FileSystem.get(new URI("hdfs://hadoop101:9000"),configuration,"root");
    }
    //文件上传
    @Test
    public void testCopyFromLocalFile() throws IOException, URISyntaxException, InterruptedException {

        //2.上传文件
        fs.copyFromLocalFile(new Path("d:/jdk api 1.8_google.CHM"),new Path("/jdkapi"));
        fs.close();
    }

    //文件下载
    @Test
    public void testCopyToLocalFile() throws URISyntaxException, IOException, InterruptedException {

        fs.copyToLocalFile(true,new Path("/jdkapi"),new Path("d:/"),true);
        fs.close();
    }

    //文件详情递归查看 recursive=True表示递归
    @Test
    public void testListFiles() throws URISyntaxException, IOException, InterruptedException {

        //获取文件详情
        RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"),true);

        while(files.hasNext()){
            LocatedFileStatus status = files.next();

            //文件大小
            System.out.println(status.getLen());
            //文件权限
            System.out.println(status.getPermission());
            //获取当前文件存储到的文件块
            BlockLocation[] blockLocations = status.getBlockLocations();

            for(BlockLocation location : blockLocations){
                //获取块存储在那一台主机上
                String[] hosts = location.getHosts();
                for(String host:hosts){
                    System.out.println(host);
                }
            }
        }
            fs.close();
    }
   }