HDFS JavaAPI 学习

写给自己:晚上的时候,在看着这个HDFS的JavaAPI.于是也顺手写一写.不过,这并不是想说的话.小侄女还不到一岁.因为肚子不舒服,在一直哭.然后,心思也跟着碎了.一个人长大真不容易.希望我的小侄女找些长大,少经历一些痛苦.嘿嘿,也许在十多年后,我就能开始教她敲代码了.带她走入程序的世界.

希望在这之前,学习一下Maven的使用.maven安装配置就不说了.安装完成后比较重要的一个事情就是修改远程仓库的地址.修改成阿里云的.这样,下载jar包的时候就快了.这些网上都能搜索到.配置完成后,在eclipse配置一下(修改成自己的配置).然后可以开始了.

package managemen;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class Demo01 {

    Configuration configuration=null;
    FileSystem fileSystem=null;
    @Before
    public void init() throws IOException, InterruptedException, URISyntaxException{
        configuration=new Configuration();
        fileSystem=FileSystem.get(new URI("hdfs://hadoop:9000"),configuration,"root");

    }

    @Test
    public void testMkdir() throws IllegalArgumentException, IOException{
        System.out.println(fileSystem.mkdirs(new Path("/Date")));
        fileSystem.close();
    }

    @Test
    public void testCopyFromLocal() throws Exception{
        Path path=new Path("F:/h.txt");
        Path dfs=new Path("/Date");
        fileSystem.copyFromLocalFile(path, dfs);
        fileSystem.close();
    }

    @Test
    public void testCopytoLocal () throws Exception{
        Path src=new Path("/data/test.txt");
        Path dst = new Path("F:/");
        fileSystem.copyToLocalFile(src, dst);
        fileSystem.close();
    }

    @Test
    public void testRemove() throws Exception{
        fileSystem.delete(new Path("/data"),false);
        fileSystem.close();
    }

    @Test
    public void testReName() throws IllegalArgumentException, IOException{
        fileSystem.rename(new Path("/Data"),new Path("/Datas"));
        fileSystem.close();
    }

    @Test
    public void testList() throws FileNotFoundException, IllegalArgumentException, IOException{
        RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/Data"), true);
        while (listFiles.hasNext()) {
            LocatedFileStatus fileStatus = listFiles.next();
            System.out.println(fileStatus.getPath().getName());
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getPermission());
            System.out.println(fileStatus.getReplication());
            System.out.println(fileStatus.getLen());
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            for (BlockLocation blockLocation : blockLocations) {
                System.out.println(blockLocation.getLength()+blockLocation.getOffset());
                String[] hosts = blockLocation.getHosts();
                for (String string : hosts) {
                    System.out.println(string);
                }

            }

        }

    }

    @Test
    public void testListStatus() throws Exception {
        FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
        String flag = " ";
        for (FileStatus status : listStatus) {
            if (status.isDirectory()) {
                flag = "目录";
            } else {
                flag = "文件";
            }
            System.out.println(flag + "\t" + status.getPath().getName());
        }
        fileSystem.close();
    }

    @Test
    public void testDownloadToLocal() throws IllegalArgumentException, IOException{
                //上传一个文件。
                //先获取HDFS 的一个输出流
                //再获取本地文件的输出流
        FSDataInputStream open = fileSystem.open(new Path("/data"));
        FileOutputStream fileOutputStream=new FileOutputStream(new File("D:/date.txt"));
        org.apache.commons.io.IOUtils.copy(open, fileOutputStream);
    }

    @Test
    public void testCat() throws IllegalArgumentException, IOException{
        FSDataInputStream in = fileSystem.open(new Path("/Mytest/jdk-7u71-linux-x64"));
        FileStatus[] listStatus = fileSystem.listStatus(new Path("/Mytest/jdk-7u71-linux-x64"));
        BlockLocation[] fileBlockLocations = fileSystem.getFileBlockLocations(
                listStatus[0], 0L, listStatus[0].getLen());
        long length=fileBlockLocations[0].getLength();
        long offset=fileBlockLocations[0].getOffset();
        System.out.println(length);
        System.out.println(offset);
        org.apache.hadoop.io.IOUtils.copyBytes(in, System.out, (int)length);
        byte[] b=new byte[4096];
        FileOutputStream os = new FileOutputStream(new File("d:/"));
        while(in.read(offset,b,0,4096)!=-1){
            os.write(b);
            offset+=4096;
            if (offset > length)
                return;
        }
        os.flush();
        os.close();
        in.close();
        }

}