Hadoop HDFS文件系统通过java FileSystem 实现上传下载等

package linlintest;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class HdfsFileSystem {
    
     public static void main(String[] args) throws Exception  {
         //String uri="hdfs://LL-167:8020/";   //hdfs 地址
         // String remote="hdfs://LL-167:8020/lin/in1/1.txt"; //hdfs上的路径
         String uri="hdfs://192.168.0.151:8020/";   //hdfs 地址
         String local="C:/Users/Administrator/Desktop/a.txt";  //本地路径
         String remote="hdfs://192.168.0.151:8020/Workspace/houlinlin";
         Configuration conf = new Configuration();
         
         //cat(conf ,uri,"hdfs://LL-167:8020/lin/in1/1.txt");
         //download(conf ,uri,remote,local);
        //  delete(conf ,uri,"hdfs://192.168.0.173:8020/Workspace/houlinlin");
         // markDir(conf ,uri,"hdfs://192.168.0.151:8020/Workspace/houlinlin/file/apply/2014/8a8380c7459dd8b90145a1fafb500235");
        
        //  checkDir(uri,"d:/file");
        // getFile(conf ,uri,"","hdfs://192.168.0.151:8020/Workspace/houlinlin/a.txt");
         copyFile(conf,uri,"C:/Users/Administrator/Desktop/8a8380c745d05d370145d06719aa3c89.txt","hdfs://192.168.0.151:8020/Workspace/houlinlin/file/apply/2014/8a8380c7459dd8b90145a1fafb500235");
       //   ls(conf ,"hdfs://fulonghadoop","hdfs://fulonghadoop/");
    }
    /**
     * 上传文件
     * @param conf
     * @param local
     * @param remote
     * @throws IOException
     */
    public static void copyFile(Configuration conf , String uri , String local, String remote) throws IOException {
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        fs.copyFromLocalFile(new Path(local), new Path(remote));
        System.out.println("copy from: " + local + " to " + remote);
        fs.close();
    }
    
    /**
     * 获取hdfs上文件流
     * @param conf
     * @param uri
     * @param local
     * @param remote
     * @throws IOException
     */
    public static  void getFileStream(Configuration conf , String uri , String local, String remote) throws IOException{
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        Path path= new Path(remote);
        FSDataInputStream in = fs.open(path);//获取文件流
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/b.txt");//输出流
        int ch = 0;
        while((ch=in.read()) != -1){  
            fos.write(ch);  
        }  
        System.out.println("-----");
        in.close(); 
        fos.close();  
    }
    
    /**
     * 创建文件夹
     * @param conf
     * @param uri
     * @param remoteFile
     * @throws IOException
     */
    public static void markDir(Configuration conf , String uri , String remoteFile ) throws IOException{
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        Path path = new Path(remoteFile);
        
        fs.mkdirs(path);
        System.out.println("创建文件夹"+remoteFile);
         
    }
    /**
     * 查看文件
     * @param conf
     * @param uri
     * @param remoteFile
     * @throws IOException
     */
     public static void cat(Configuration conf , String uri ,String remoteFile) throws IOException {
            Path path = new Path(remoteFile);
            FileSystem fs = FileSystem.get(URI.create(uri), conf);
            FSDataInputStream fsdis = null;
            System.out.println("cat: " + remoteFile);
            try {
                fsdis = fs.open(path);
                IOUtils.copyBytes(fsdis, System.out, 4096, false);
            } finally {
                IOUtils.closeStream(fsdis);
                fs.close();
            }
    }
     /**
      * 下载 hdfs上的文件
      * @param conf
      * @param uri
      * @param remote
      * @param local
      * @throws IOException
      */
     public static void download(Configuration conf , String uri ,String remote, String local) throws IOException {
            Path path = new Path(remote);
            FileSystem fs = FileSystem.get(URI.create(uri), conf);
            fs.copyToLocalFile(path, new Path(local));
            System.out.println("download: from" + remote + " to " + local);
            fs.close();
    }
     /**
      * 删除文件或者文件夹
      * @param conf
      * @param uri
      * @param filePath
      * @throws IOException
      */
     public static void delete(Configuration conf , String uri,String filePath) throws IOException {
            Path path = new Path(filePath);
            FileSystem fs = FileSystem.get(URI.create(uri), conf);
            fs.deleteOnExit(path);
            System.out.println("Delete: " + filePath);
            fs.close();
    }
     /**
      * 查看目录下面的文件
      * @param conf
      * @param uri
      * @param folder
      * @throws IOException
      */
     public static  void ls(Configuration conf , String uri , String folder) throws IOException {
            Path path = new Path(folder);
            FileSystem fs = FileSystem.get(URI.create(uri), conf);
            FileStatus[] list = fs.listStatus(path);
            System.out.println("ls: " + folder);
            System.out.println("==========================================================");
            for (FileStatus f : list) {
                System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(),f.isDirectory() , f.getLen());
            }
            System.out
                    .println("==========================================================");
            fs.close();
    }

        /**
         * 
         * @param parentName 绝对路径地址
         * @throws Exception
         */
        public static void checkDir(String uri,String parentName)    throws Exception{
            //D:\file
            Configuration conf = new Configuration();
            File file = new File(parentName);
            boolean flag = true;
            while (flag)    {
                //查出parentName下的所有文件
                File[] fileNames = file.listFiles(new FileFilter());
                if(fileNames != null)    {
                    for (int i = 0; i < fileNames.length; i++) {
                        File f = fileNames[i];
                        //System.out.println("parent directory:"+f.getParent()+",file name:"+f.getName());
                        System.out.println("parent directory:"+f.getParent().replace("\\", "/").substring(2)+",file name:"+f.getName());
                        String remoteFolrd= "hdfs://192.168.0.173:8020/Workspace/houlinlin"+f.getParent().replace("\\", "/").substring(2);
                        markDir(conf ,uri,remoteFolrd);
                         copyFile(conf ,uri,f.getParent()+"\\"+f.getName(),remoteFolrd);
                    }                
                }
                //查出parentName下的所有目录
                File[] directories = file.listFiles(new DirectortyFilter());
                if(directories != null)    {
                    for (int i = 0; i < directories.length; i++) {
                        File dir = directories[i];
                        //绝对路径
                        String path =  dir.getAbsolutePath();
                        //递归
                        checkDir(uri,path);
                    }
                }
                flag = false;
            }
        }
     
}

 

转载于:https://www.cnblogs.com/onetwo/p/5590864.html