HDFS API

package com.snsoft.hadoop.hdfs;

 

import java.io.FileNotFoundException;

import java.io.IOException;

import java.io.InputStream;

 

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.BlockLocation;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.FileUtil;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.fs.PathFilter;

import org.apache.hadoop.io.IOUtils;

 

public class Upload {

 

/**

 * @param args

 * @throws FileNotFoundException 

 */

public static void main(String[] args) throws Exception {

// TODO Auto-generated method stub

/*上传文件到HDFS

String localsrc="C:/2013公司年会搞笑舞蹈视频 高清.mp4";

        String hdfsdst="/user/Administrator/input";

        

        uploadLocalFile2HDFS(localsrc,hdfsdst);

        

        */

//删除HDFS上的文件

//delete("/user/Administrator/test");

//读取HDFS上的文件

//readFile("/user/Administrator/input/file1.txt");

//重命名文件

//rename("/user/Administrator/input/file5.txt", "/user/Administrator/input/file1.txt");

//创建文件，并写入内容

//byte[] contents =  "hello world 世界你好\n".getBytes();

        //createFile("/user/hadoop/test1/d.txt",contents);

//创建目录

mkdir("test");                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    

//查看文件是否存在

//exist("/user/Administrator/input/file5.txt");

//查看文件的最后修改时间

//lasttime("/user/Administrator/input/file1.txt");

//查看某个目录下的所有文件

//all("/user/Administrator");

//查询文件在集群中的位置

//weizhi("/user/Administrator/input/file1.txt");

//

}

/**

 * 上传文件到HDFS

 * @param s  上传的文件路径

 * @param d  HDFS上的路径

 * @throws IOException

 */

   public static void uploadLocalFile2HDFS(String s, String d) throws IOException

   {

       Configuration config = new Configuration();

       FileSystem hdfs = FileSystem.get(config);

       

       Path src = new Path(s);

       Path dst = new Path(d);

       

       hdfs.copyFromLocalFile(src, dst);

       System.out.println("上传完成");

       hdfs.close();

   }

   /**

    * 删除HDFS上的文件

    * @param filePath HDFS上文件的路径

    * @throws IOException

    */

   public static void delete(String filePath) throws IOException{

        Configuration conf = new Configuration();

        FileSystem fs = FileSystem.get(conf);

        Path path = new Path(filePath);

        boolean isok = fs.deleteOnExit(path);

        if(isok){

            System.out.println("delete ok!");

        }else{

            System.out.println("delete failure");

        }

        fs.close();

    }

   /**

    * 读取HDFS上文件的内容

    * @param filePath HDFS上文件的路径

    * @throws IOException

    */

   public static void readFile(String filePath) throws IOException{

        Configuration conf = new Configuration();

        FileSystem fs = FileSystem.get(conf);

        Path srcPath = new Path(filePath);

        InputStream in = null;

        try {

            in = fs.open(srcPath);

            IOUtils.copyBytes(in, System.out, 4096, false); //复制到标准输出流

        } finally {

            IOUtils.closeStream(in);

        }

    }

   /**

    * 文件重命名

    * @param oldName 原文件名

    * @param newName 新文件名

    * @throws IOException

    */

   public static void rename(String oldName,String newName) throws IOException{

        Configuration conf = new Configuration();

        FileSystem fs = FileSystem.get(conf);

        Path oldPath = new Path(oldName);

        Path newPath = new Path(newName);

        boolean isok = fs.rename(oldPath, newPath);

        if(isok){

            System.out.println("rename ok!");

        }else{

            System.out.println("rename failure");

        }

        fs.close();

    }

   /**

    * 创建新文件

    * @param dst 目标路径

    * @param contents 文件内容

    * @throws IOException

    */

   public static void createFile(String dst , byte[] contents) throws IOException{

        Configuration conf = new Configuration();

        FileSystem fs = FileSystem.get(conf);

        Path dstPath = new Path(dst); //目标路径

        //打开一个输出流

        FSDataOutputStream outputStream = fs.create(dstPath);

        outputStream.write(contents);

        outputStream.close();

        fs.close();

        System.out.println("文件创建成功！");

    }

   /**

    * 创建文件目录

    * @param path 目录的路径

    * @throws IOException

    */

   public static void mkdir(String path) throws IOException{

        Configuration conf = new Configuration();

        FileSystem fs = FileSystem.get(conf);

        Path srcPath = new Path(path);

        boolean isok = fs.mkdirs(srcPath);

        if(isok){

            System.out.println("create dir ok!");

        }else{

            System.out.println("create dir failure");

        }

        fs.close();

    }

   /**

    * 查看文件是否存在

    * @param path 文件的完整路径

    * @throws IOException

    */

   public static void exist(String path) throws IOException{

   Configuration conf=new Configuration();

        FileSystem hdfs=FileSystem.get(conf);

   Path findf=new Path(path);

        boolean isExists=hdfs.exists(findf);

        if(isExists)

        {

         System.out.println("文件已存在");

        }

        else{

         System.out.println("文件不存在");

        }

   }

   /**

    * 文件的修改时间

    * @param path 文件路径

    * @throws IOException

    */

   public static void lasttime(String path) throws IOException{

   Configuration conf=new Configuration();

        FileSystem hdfs=FileSystem.get(conf);

        Path fpath =new Path(path);

        FileStatus fileStatus=hdfs.getFileStatus(fpath);

        long modiTime=fileStatus.getModificationTime();

        System.out.println("file1.txt的修改时间是"+modiTime);

   }

   /**

    * 查看某个目录下的所有文件

    * @param path

    * @throws IOException

    */

   public static void all(String path) throws IOException{

   Configuration conf=new Configuration();

        FileSystem hdfs=FileSystem.get(conf);        

        Path listf =new Path(path);        

        FileStatus stats[]=hdfs.listStatus(listf);

        for(int i = 0; i < stats.length; ++i)

        {

         System.out.println(stats[i].getPath().toString());

        }

        hdfs.close();

   }

   /**

    * 查找某个文件在集群中的位置

    * @param path 文件路径

    * @throws IOException

    */

   public static void weizhi(String path) throws IOException{

   Configuration conf=new Configuration();

        FileSystem hdfs=FileSystem.get(conf);

        Path fpath=new Path(path);

        FileStatus filestatus = hdfs.getFileStatus(fpath);

        BlockLocation[] blkLocations = hdfs.getFileBlockLocations(filestatus, 0, filestatus.getLen());

        int blockLen = blkLocations.length;

        for(int i=0;i<blockLen;i++){

            String[] hosts = blkLocations[i].getHosts();

            System.out.println("block_"+i+"_location:"+hosts[0]);

        }

   }

   

   

   

   

}