java 操作 hdfs

package com.wisedu.hadoop.hdfs;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * Created by heavy_li on 2016/11/19.
 */
public class HdfsFileDemo {

     private Configuration conf ;
     private FileSystem fileSystem;

     public HdfsFileDemo(){
         //加载配置文件，默认会加载CLASSPATH下的core-site.xml
         conf = new Configuration();
         //也可以自己手动加载配置文件  如下
         //conf.addResource("core-site.xml");
         log(conf.get("fs.defaultFS"));

         try {
             //用超级管理员账号 不会存在权限问题
             fileSystem = FileSystem.get(new URI(conf.get("fs.defaultFS")),conf,conf.get("hadoop.loginName"));
         } catch (IOException e) {
             e.printStackTrace();
         } catch (InterruptedException e) {
             e.printStackTrace();
         } catch (URISyntaxException e) {
             e.printStackTrace();
         }
     }

    /**
     * 显示目录下的所有文件 对应ls命令
     * @param fileUri
     */
    public void showDir(String fileUri,PathFilter pathFilter){
        try {
            FileStatus[] fileStatuses = null;
            if(pathFilter==null){
                fileStatuses = fileSystem.listStatus(new Path(fileUri));
            }else{
                fileStatuses = fileSystem.listStatus(new Path(fileUri),pathFilter);
            }
            if(fileStatuses!=null&&fileStatuses.length!=0){
                for(FileStatus fs : fileStatuses){
                    log(fs.getOwner().concat("  ").concat(fs.getGroup()).concat("  ").concat(fs.getPath().toString()).concat("  ").concat(fs.getLen()+"  "));
                }
            }else{
                log("该目录下没有任何文件！");
            }
        } catch (IOException e) {
            if(e instanceof FileNotFoundException){
                log("目录不存在！");
            }else {
                e.printStackTrace();
            }
        }
    }

    /**
     * 读取文件，对应 cat命令  get命令 可以将字节从新生成文件
     * @param fileUri
     */
    public void readFile(String fileUri){
        Path path = new Path(fileUri);
        try {
            if(fileSystem.exists(path)){
                FSDataInputStream is = fileSystem.open(path);
                FileStatus status = fileSystem.getFileStatus(path);
                byte[] buffer = new byte[Integer.parseInt(String.valueOf(status.getLen()))];
                is.readFully(0, buffer);
                is.close();
                log(new String(buffer));
            }else{
                log("该文件不存在！");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 创建目录，可以创建多级目录  对应命令mkdir
     * @param uri
     */
    public void mkdir(String uri){
        Path path = new Path(uri);
        try {
            fileSystem.create(path);
            log(uri.concat(" created!"));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 删除目录或者文件 对应 rm -r 命令
     * @param uri
     */
    public void deleteDirOrFile(String uri){
        Path path = new Path(uri);
        try {
            //  true 相当于rm -r
            fileSystem.delete(path,true);
            log(uri.concat(" deleted!"));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 将本地文件上传到hdfs 对应命令: copyFromLocal
     * @param localPath
     * @param remoteUri  目录不存在的话会自动创建
     */
    public void copyFromLocal(String localPath,String remoteUri){
        Path src = new Path(localPath);
        Path dst = new Path(remoteUri);
        try {
            fileSystem.copyFromLocalFile(src, dst);
            log("over!");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 以字节形式写入hdfs系统
     * @param data
     * @param uri
     */
    public void writeFile(byte[] data,String uri){
        Path path = new Path(uri);
        try {
            FSDataOutputStream out = fileSystem.create(path);
            out.write(data);
            log("over!");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 直接写入字符
     * @param s
     * @param uri
     */
    public void writeFileStr(String s ,String uri){
        Path path = new Path(uri);
        try {
            FSDataOutputStream out = fileSystem.create(path);
            out.writeUTF(s);
            log("over!");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 查找文件块在哪个服务器上
     * @param uri
     */
    public void getFileBlockLoaction(String uri){
        Path path = new Path(uri);

        FileStatus status = null;
        try {
            status = fileSystem.getFileStatus(path);
            BlockLocation[] locations = fileSystem.getFileBlockLocations(status, 0, status.getLen());

            int length = locations.length;
            for(int i=0;i<length;i++){
                String[] hosts = locations[i].getHosts();
                for(String s:hosts) {
                    log("block_" + i + "_location:" + s);
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 查看各节点的状态
     */
    public void getDataNodeStatus(){
        DistributedFileSystem dfs = (DistributedFileSystem)fileSystem;
        DatanodeInfo[] dataNodeStats = null;
        try {
            dataNodeStats = dfs.getDataNodeStats();
        } catch (IOException e) {
            e.printStackTrace();
        }
        if(dataNodeStats!=null) {
            int i=1;
            for (DatanodeInfo node : dataNodeStats) {
                log("DataNode " + i++ + " :" .concat(node.getDatanodeReport()));
            }
        }
    }



    public void close(){
        if(fileSystem!=null){
            try {
                fileSystem.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

     private void log(String s){
         System.out.println(s);
     }

     public static void main(String[] args){
         HdfsFileDemo hdfsFileSystem = new HdfsFileDemo();
         PathFilter pathFilter = new PathFilter() {
             public boolean accept(Path path) {
                 if(path.getName().toLowerCase().endsWith(".txt")){
                     return true;
                 }else {
                     return false;
                 }
             }
         };
      //   hdfsFileSystem.showDir("/",pathFilter);
     //    hdfsFileSystem.readFile("/a/hello.txt");
       //  hdfsFileSystem.mkdir("/aa/b");
         //hdfsFileSystem.deleteDirOrFile("/aa");
         //hdfsFileSystem.deleteDirOrFile("/empty.txt");
      //   hdfsFileSystem.copyFromLocal("F:\\my picture\\Camera\\IMG_20130914_105631.jpg","/cc/dctest1.jpg");
         //hdfsFileSystem.writeFileStr("I am from java! \n","/cc/a.txt");
         //hdfsFileSystem.getFileBlockLoaction("/test1.txt");
         hdfsFileSystem.getDataNodeStatus();
         hdfsFileSystem.close();
     }

}