Hadoop学习三(java api 对hdfs常用操作)

上文测试通过后，对于其他的一些java api操作hdfs进行了编写和测试，如下:

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;

/**
 * Created by lixintang on 2017/4/18.
 */
public class HDFSUtils {

    private static final String FS_URI = "hdfs://192.168.56.101:9000";

    public HDFSUtils(){
        System.setProperty("hadoop.home.dir", "D:\\soft\\dev_soft\\database\\hadoop\\hadoop-2.7.1");
    }

    public Configuration getConfiguration(){
        Configuration conf = new Configuration();
        return conf;
    }

    public FileSystem getFileSystem() throws Exception{
        FileSystem fs = null;
        try {
             fs = FileSystem.get(URI.create(FS_URI),getConfiguration());
            if(fs == null) throw new Exception("fileSystem is null !");
        } catch (IOException e) {
            e.printStackTrace();
            throw new Exception("getFileSystem fail !" + e.getMessage());
        }
        return fs;
    }

    /**
     * 查看目录及文件（ls）
     * @param folder
     * @return
     * @throws Exception
     */
    public String ls(String folder) throws Exception{
        FileSystem fs = getFileSystem();
        StringBuffer sb = new StringBuffer();
        FileStatus[] list = fs.listStatus(new Path(folder));
        System.err.println("查看文件及目录结果.....size :" + list == null ? null: list.length);
        for (FileStatus f : list) {
            System.err.printf("name: %s, folder: %s, size: %d\n", f.getPath(),
                    f.isDir(), f.getLen());
            sb.append("name:").append(f.getPath()).append(",folder:").append(f.isDir()).append(",size:" + f.getLen()).append("\r\n");
        }
        fs.close();
        return sb.toString();
    }

    /**
     * 创建目录
     * @param folder
     * @param isDelete 是否是删除,true 删除目录，false 创建目录
     * @throws Exception
     */
    public void mkdir(String folder, boolean isDelete) throws Exception{
        Path path = new Path(folder);
        FileSystem fs = getFileSystem();
        if(isDelete) {
            System.err.println("删除目录开始...." + folder);
            fs.deleteOnExit(path);
        }else {
            if(!fs.exists(path)) {
                System.err.println("创建目录开始...." + folder);
                fs.mkdirs(path);
            }else
                throw new Exception("path exists !");
        }
        fs.close();
    }

    /**
     * 文件重命名
     * @param src  源文件
     * @param dst  目标文件
     * @throws Exception
     */
    public void rename(String src, String dst) throws Exception{
        Path pathSrc = new Path(src);
        Path pathDst = new Path(dst);
        System.err.printf("文件重命名开始.....src:%s,dst:%s",src, dst);
        FileSystem fs = null;
        try {
            fs = getFileSystem();
            fs.rename(pathSrc, pathDst);
        } catch (Exception e){
            e.printStackTrace();
        } finally {
            if(fs != null)
                fs.close();
        }
    }

    /**
     * 复制本地文件到hdfs
     * @param local
     * @param remote
     * @throws Exception
     */
    public void uploadFile(String local, String remote) throws IOException {
        FileSystem fs = null;
        try {
            fs = getFileSystem();
            fs.copyFromLocalFile(new Path(local), new Path(remote));
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if(fs != null)
                fs.close();
        }
    }

    /**
     * 下载文件到本地
     * @param remote
     * @param local
     * @throws Exception
     */
    public void download(String remote, String local) throws Exception {
        FileSystem fs = null;
        try {
            fs = getFileSystem();
            fs.copyToLocalFile(new Path(remote), new Path(local));
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if(fs != null)
                fs.close();
        }
        
    }

    /**
     * 查看文件
     * @param remoteFile
     * @return
     * @throws Exception
     */
    public String catFile(String remoteFile) throws Exception{
        OutputStream out = new ByteArrayOutputStream();
        InputStream in = null;
        String str = null;
        FileSystem fs = null;
        try {
            fs = getFileSystem();
            in = fs.open(new Path(remoteFile));
            IOUtils.copyBytes(in, out, 4096, false);
            str = out.toString();
        } finally {
            if(in != null)
                IOUtils.closeStream(in);
            if(fs != null)
                fs.close();
        }
        return str;
    }

}

测试：测试了几个方法，没有全部测试，如果有问题，请自行修改调整，呵呵

/**
 * Created by lixintang on 2017/4/17.
 */
public class HDFSTest {

    public static void main(String[] args) throws Exception{
        HDFSUtils utils = new HDFSUtils();
        //查看跟目录
        utils.ls("/");
        //创建目录
        utils.mkdir("/test", false);
        //查看创建结果
        utils.ls("/");
        //删除目录
        utils.mkdir("/test", true);
        //查看删除后的目录
        utils.ls("/");
        //重命名文件
        utils.rename("/newrt", "/rt.jar");
        //查看重命名后结果
        utils.ls("/");
    }
}