Hadoop常用的API方法（六）

下面代码：封装了,实现创建目录，创建文件，增，删，改文件，获取节点，上传文件，下载文件，以流的方式上传视频

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;

public class HdfsDemo {


    //创建新文件
    public static void CreateFile(String url,String path){
        try {
            //该类的对象封转了客户端或者服务器的配置。
            Configuration config = new Configuration();
            URI uri = new URI("hdfs://192.168.52.140:9000");
            //该类的对象是一个文件系统对象，可以用该对象的一些方法来对文件进行操作。
            FileSystem fs = FileSystem.get(uri, config);
            //目标路径
            Path p = new Path("hdfs://192.168.52.140:9000/dirs");
            fs.create(p, new Progressable() {

                @Override
                public void progress() {
                    System.out.print(">>");
                }
            });

            System.out.println("create dir successful!!!");

        } catch (URISyntaxException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    //判断文件是否存在于Hdfs
    public static boolean IfFileExits(String FileName)throws Exception 
    {
         Configuration conf = new Configuration();
          FileSystem fs = FileSystem.get(URI.create(FileName),conf);
          Path path = new Path(FileName);
          boolean isExists = fs.exists(path);
          return isExists;
    }

    //删除文件
    public static void delete(String url,String filePath) throws IOException{
        try {
            Configuration conf = new Configuration();
            URI uri = new URI(url);
            FileSystem fs = FileSystem.get(uri,conf);
            Path path = new Path(filePath);
            boolean isok = fs.deleteOnExit(path);
            if(isok){
                System.out.println("delete ok!");
            }else{
                System.out.println("delete failure");
            }
            fs.close();
        } catch (IllegalArgumentException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

    //上传本地文件
    public static void uploadFile(String url,String src,String dst) throws IOException{
        try {
            Configuration conf = new Configuration();
            URI uri = new URI(url);
            FileSystem fs = FileSystem.get(uri,conf);
            Path srcPath = new Path(src); //原路径
            Path dstPath = new Path(dst); //目标路径
            //调用文件系统的文件复制函数,前面参数是指是否删除原文件，true为删除，默认为false
            fs.copyFromLocalFile(false,srcPath, dstPath);
            //打印文件路径
            System.out.println("Upload to "+conf.get("fs.default.name"));
            System.out.println("------------list files------------"+"\n");
            FileStatus [] fileStatus = fs.listStatus(dstPath);
            for (FileStatus file : fileStatus) 
            {
                System.out.println(file.getPath());
            }
            fs.close();
        } catch (IllegalArgumentException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

    //从hdfs下载文件
    public static void downFile(String url,String src,String dst) throws IOException{
        try {
            Configuration conf = new Configuration();
            URI uri = new URI(url);
            FileSystem fs = FileSystem.get(uri,conf);
            Path srcPath = new Path(src); //原路径
            Path dstPath = new Path(dst); //目标路径
            //调用文件系统的文件复制函数,前面参数是指是否删除原文件，true为删除，默认为false
            fs.copyToLocalFile(false,srcPath, dstPath);
            //打印文件路径
            System.out.println("Upload to "+conf.get("fs.default.name"));
            System.out.println("------------list files------------"+"\n");
            FileStatus [] fileStatus = fs.listStatus(dstPath);
            for (FileStatus file : fileStatus) 
            {
                System.out.println(file.getPath());
            }
            fs.close();
        } catch (IllegalArgumentException e) {
        } catch (URISyntaxException e) {
        }
    }


    //文件重命名
    public static void rename(String url,String oldName,String newName) throws IOException{
        try {
            Configuration conf = new Configuration();
            URI uri = new URI(url);
            FileSystem fs = FileSystem.get(uri,conf);
            Path oldPath = new Path(oldName);
            Path newPath = new Path(newName);
            boolean isok = fs.rename(oldPath, newPath);
            if(isok){
                System.out.println("rename ok!");
            }else{
                System.out.println("rename failure");
            }
            fs.close();
        } catch (IllegalArgumentException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

    //创建目录
    public static void mkdir(String url,String path) throws IOException{
        try {
            Configuration conf = new Configuration();
            URI uri = new URI(url);
            FileSystem fs = FileSystem.get(uri,conf);
            Path srcPath = new Path(path);
            boolean isok = fs.mkdirs(srcPath);
            if(isok){
                System.out.println("create dir ok!");
            }else{
                System.out.println("create dir failure");
            }
            fs.close();
        } catch (IllegalArgumentException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }
    //上传视频
public static void shangchaungshiping(){
    Configuration conf=new Configuration();
    URI uri=new URI("hdfs://192.168.61.128:9000");
    FileSystem hdfs=FileSystem.get(uri,conf);
    FileSystem local=FileSystem.getLocal(conf);//获取windows下的文件管理系统
    Path inputDir=new Path("F:\\1\\love.mkv");
    Path hdfsFile=new Path("/vdio");
    hdfs.mkdirs(hdfsFile);
    FSDataOutputStream out;
    FileStatus[] inputFiles=local.listStatus(inputDir);
    //通过OutputStream.write()来将视频文件循环写入HDFS下的指定目录
    for(int i=0;i<inputFiles.length;i++){
        System.out.println("*******"+inputFiles[i].getPath().getName());
        FSDataInputStream in=local.open(inputFiles[i].getPath());
        out=hdfs.create(new Path("/vdio/"+inputFiles[i].getPath().getName()));
        byte buffer[]=new byte[256];
        int bytesRead=0;
        while((bytesRead=in.read(buffer))>0){
            out.write(buffer,0,bytesRead);
        }
        out.close();
        in.close();
        File file=new File(inputFiles[i].getPath().toString());
        file.delete();
    }

}

    //读取文件的内容
    public static void readFile(String url,String filePath) throws IOException{
        try {
            Configuration conf = new Configuration();
            URI uri = new URI(url);
            FileSystem fs = FileSystem.get(uri,conf);
            Path srcPath = new Path(filePath);
            InputStream in = null;
            try {
                in = fs.open(srcPath);
                IOUtils.copyBytes(in, System.out, 4096, false); //复制到标准输出流
            } finally {
                IOUtils.closeStream(in);
            }
        } catch (IllegalArgumentException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

    public static void GetLocation(String url,String FileName)throws Exception//查看想看的文件存放在哪几个节点当中
    {
        Configuration conf = new Configuration();
        URI uri = new URI(url);
          FileSystem fs = FileSystem.get(uri,conf);
          Path path = new Path(FileName);
          FileStatus fileStatus = fs.getFileStatus(path);

          BlockLocation[] blkLocations = fs.getFileBlockLocations(fileStatus , 0, fileStatus.getLen());

          int blkCount = blkLocations.length;
          for (int i=0; i < blkCount; i++) {
            String[] hosts = blkLocations[i].getHosts();
            System.out.print(blkLocations[i].getHosts());
            System.out.print(hosts[i]);
            // Do something with the block hosts
          }

    }
  //显示Hdfs一组路径的文件信息
    public static void ListStatus(String args[]) throws Exception 
    {

        String uri = args[0];
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(uri),conf);
        Path[] paths = new Path[args.length];
                for(int i=0;i<paths.length;i++)
                {
                    paths[i] = new Path(args[i]);
                }
        FileStatus[] status =fs.listStatus(paths);
        Path[] listedPaths = FileUtil.stat2Paths(status);
        for(Path p: listedPaths){
            System.out.println(p);
        }

    }

    public static void main(String[] args) throws Exception {
        String url="hdfs://192.168.52.140:9000";
        //CreateFile("hdfs://192.168.52.140:9000","hdfs://192.168.52.140:9000/dirs2");
        //delete("hdfs://192.168.52.140:9000","hdfs://192.168.52.140:9000/dirs");
        //rename(url,"hdfs://192.168.52.140:9000/dir","hdfs://192.168.52.140:9000/newdir");
    //  mkdir(url,"hdfs://192.168.52.140:9000/mydir");
        String relativelyPath =System.getProperty("user.dir")+"/src/log4j.properties";
        String downPath =System.getProperty("user.dir")+"/";
        //uploadFile(url,relativelyPath,"hdfs://192.168.52.140:9000/mydir");

        //readFile(url,"hdfs://192.168.52.140:9000/mydir/log4j.properties");
        //downFile(url,"hdfs://192.168.52.140:9000/mydir/log4j.properties","D:/");
        //System.out.println(IfFileExits("hdfs://192.168.52.140:9000/mydir/log4j.properties"));

        GetLocation(url,"hdfs://192.168.52.140:9000/mydir/log4j.properties");

    }

}