利用JUnit实现对hadoop中javaAPI的测试

package gorilla.test;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.junit.Before;
import org.junit.Test;

public class HDFSTest02 {
    FileSystem fSystem;
    /**
     * 连接hdfs
     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Before
    public void init() throws IOException, InterruptedException, URISyntaxException {
        Configuration configuration = new Configuration();
        fSystem = FileSystem.get(new URI("hdfs://192.168.15.134:9000"), configuration, "hadoop");
        /*
        Configuration conf = new Configuration();
        // 1.设置用户名
        System.setProperty("HADOOP_USER_NAME", "hadoop");
        // 设置要使用的文件系统是hdfs 地址为...
        conf.set("fs.defaultFS", "hdfs://192.168.15.134:9000");
        fs = FileSystem.get(conf);
        */
    }
    @Test
    public void test() {
        System.out.println("test");
    }
    
    /**
     * 将本地文件上传
     * @throws IllegalArgumentException
     * @throws IOException
     */
    @Test
    public void addFileToHdfs() throws IllegalArgumentException, IOException {
        /*
         *  比如/eee 而根目录下不存在eee这个文件夹，就会认为是要将文件传到/ 下面的eee文件里
         *  如果eee是一个文件夹就会在eee文件夹下使用原名字上传
         *  如果是/eee/ff.md就知道是要上传到/eee这个文件夹下的ff.md这个文件中
         */
        fSystem.copyFromLocalFile(new Path("C:/Users/Administrator/Desktop/input/block.rar"), new Path("/user/block.rar"));
    }
    
    /**
     * 将hdfs文件下载到本地
     * @throws IllegalArgumentException
     * @throws IOException
     */
    @Test
    public void addFileFromHdfs() throws IllegalArgumentException, IOException {
        fSystem.copyToLocalFile(false, new Path("/source.txt"), new Path("G:/HDFSDownload"), true);
    }
    
    // 删除一个参数的方法以及弃用,两个参数的第二个参数表示是否要使用递归操作,如果删除的是文件夹，只要不是空文件夹，就必须设置成true允许递归操作才能删除
    @Test
    public void delete() throws IllegalArgumentException, IOException {
        fSystem.delete(new Path(""), true);
    }
    // 查看目录下的文件信息
    @Test
    public void testListFiles() throws FileNotFoundException, IllegalArgumentException, IOException {
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user"), true);
        while (listFiles.hasNext()) {
            LocatedFileStatus next = listFiles.next();
            System.out.println(next);
            System.out.println(next.getBlockSize());
            System.out.println(next.getPath().getName());
            BlockLocation[] blockLocations = next.getBlockLocations();
            System.out.println("开始展示block块");
            for (BlockLocation blockLocation : blockLocations) {
                System.out.println(blockLocation);
                System.out.println("---------");
                // offset为0是因为是第一个block
                String[] hosts = blockLocation.getHosts();
                for (String string : hosts) {
                    System.out.println(string);
                }
            }
        }
    }
    
    // 向HDFS上传文件
    @Test
    public void uploadByStream() throws IllegalArgumentException, IOException {
        // input 把数据读入
        FileInputStream fis = new FileInputStream("G:/HDFSDownload/source.txt");
        // output 把输入进来的数据写到某个地方去
        FSDataOutputStream hdfsOutputStream = fSystem.create(new Path("/gmj/source.txt"),true);
        IOUtils.copyBytes(fis, hdfsOutputStream, 1024);
    }
    
    // 从HDFS下载文件或文件夹
    @Test
    public void downloadByStream() throws IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/gmj/source.txt"));
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/source2.txt");
        org.apache.commons.io.IOUtils.copy(inputStream, fos);
    }
    
    @Test
    public void downloadByStreamUseSeek() throws IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/user/jdk-8u131-windows-x64.exe"));
        inputStream.seek(1024*1024*128);
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/outseek1.exe");
        org.apache.commons.io.IOUtils.copy(inputStream, fos);
    }
    
    @Test
    public void testListAll() throws FileNotFoundException, IllegalArgumentException, IOException {
        FileStatus[] listStatus = fSystem.listStatus(new Path("/"));
        for (FileStatus fileStatus : listStatus) {
            System.out.println(fileStatus.getPath().getName());
        }
    }
    
    // 读取一个文件大小介于129M~256M，切块中的第二个block块的内容
    @Test
    public void readSecondBlock() throws FileNotFoundException, IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/user/jdk-8u131-windows-x64.exe"));
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user/jdk-8u131-windows-x64.exe"), false);
        
        LocatedFileStatus next = listFiles.next();
        BlockLocation[] blockLocations = next.getBlockLocations();
        for (int j = 0; j < blockLocations.length; j++) {
            System.out.println(blockLocations[j].getOffset());
            if (j == 1) {
                inputStream.seek(blockLocations[j].getOffset());
                break;
            }
        }
        
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/outseek3.exe");
        org.apache.commons.io.IOUtils.copy(inputStream, fos);
    }
    
    // 读取一个文件大于256M且切块中的第二个block块的内容
    @Test
    public void readSecondBlockAgain() throws FileNotFoundException, IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/user/block.rar"));
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user/block.rar"), false);

        LocatedFileStatus next = listFiles.next();
        BlockLocation[] blockLocations = next.getBlockLocations();
        for (int j = 0; j < blockLocations.length; j++) {
            System.out.println(blockLocations[j].getOffset());
            if (j == 1) {
                long length = blockLocations[j].getLength();
                FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/bigblock2.rar");
                org.apache.commons.io.IOUtils.copyLarge(inputStream, fos, blockLocations[j].getOffset(), length);
                break;
            }
        }
    }
    
    // 将一个超过两个block块的大文件以每一个文件如block0，block1， 下载下来
    @Test
    public void readSecondBlockAgainToo() throws FileNotFoundException, IllegalArgumentException, IOException {
        
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user/block.rar"), false);

        LocatedFileStatus next = listFiles.next();
        BlockLocation[] blockLocations = next.getBlockLocations();
        for (int j = 0; j < blockLocations.length; j++) {
            FSDataInputStream inputStream = fSystem.open(new Path("/user/block.rar"));
            System.out.println(blockLocations[j].getOffset());
            System.out.println(blockLocations.length);
            FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/bigdata" + j + ".rar");
            org.apache.commons.io.IOUtils.copyLarge(inputStream, fos, blockLocations[j].getOffset(), blockLocations[j].getLength());
            
        }
    }   
}