API操作HDFS

通过API操作HDFS

package HdfsAPI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;

import static org.apache.hadoop.fs.FileSystem.*;

public class HdfsAPI {
    //实例化hadoop配置文件
    //后推前  ctrl+alt+v
    //去除多余的包 ctrl+calt+o
    //提示 ctrl+alt+o
    //@Test junit test
    FileSystem fs;
    Configuration conf;
    @Test
    public void initHdfsToString() throws IOException{

        conf = new Configuration();

        //get filesystem
        fs= get(conf);

        //print filesystem
        System.out.println(fs.toString());

    }
    @Before
    public void initHdfs() throws URISyntaxException, IOException, InterruptedException {

        //实例化配置
        conf=new Configuration();
        //获取文件系统
        fs=get(new URI("hdfs://bigdata111:9000"), conf, "root");
        //System.out.println("before running");
    }
    @After
    public void closeHDFs() throws IOException{
        fs.close();
        //System.out.println("after running");
    }
    @Test
    /**
     * upload
     */
    public void putFileHdfs() throws URISyntaxException, IOException, InterruptedException {
//        //1.实例化配置文件
//        Configuration conf = new Configuration();
        //设置参数(dfs.replication为配置文件中的冗余name和value)
        conf.set("dfs.replication","2");
        //2.获取文件系统
        FileSystem fs = get(new URI("hdfs://bigdata111:9000"), conf, "root");

        //3.定义文件输入输出的路径
        Path in = new Path("D:/text/z.txt");
        Path out = new Path("hdfs://bigdata111:9000/");

        //4.上传
        fs.copyFromLocalFile(in,out);

//        //5.关闭文件系统
//        fs.close();

        System.out.println("upload success");
    }
    /**
     * 下载
     */
    @Test
    public void getFileHdfs() throws URISyntaxException, IOException, InterruptedException {
//        //实例化配置
//        Configuration conf=new Configuration();
//        //获取文件系统
//        FileSystem fs=get(new URI("hdfs://bigdata111:9000"), conf, "root");
        //定义下载地址，hdfs
        Path src = new Path("/english.txt");
        //下载到win的路径
        Path out = new Path("D:\\text\\test.txt");

        //download
        //The first parameter---delSrc-->whether delete source file
        //The second parameter---hdfs path
        //The third parameter---win path
        //The final parameter---useRawLocalFileSystem-->whether verification file completeness
        fs.copyToLocalFile(false,src,out,true);

//        fs.close();

        System.out.println("download success");
    }
    @Test
    /**
     * Create catalogue
     */
    public void mkidrHdfsFile() throws URISyntaxException, IOException, InterruptedException {
//        //实例化配置
//        Configuration conf=new Configuration();
//        //获取文件系统
//        FileSystem fs=get(new URI("hdfs://bigdata111:9000"), conf, "root");
        //Hdfs path
        Path path = new Path("/0310");
        //Create catalogue
        fs.mkdirs(path);

//        fs.close();

        System.out.println("Create success");

    }
    @Test
    /**
     * delete catalouge
     */
    public void rmHdfsFile() throws IOException {
        //delete hdfs path
        Path path = new Path("/0310");
        //delete
        //delete file false ,catalogue true(是否递归)
        fs.delete(path,false);
        System.out.println("delete success");
    }
    @Test
    /**
     * rename
     */
    public void reName() throws IOException {
        //The first parameter-->hdfs name
        fs.rename(new Path("/qqq.txt"), new Path("/aaa.txt"));
        System.out.println("reName success");
    }
    @Test
    /**
     * check catalogue or file detailed information
     */
    public void readFileHdfs() throws IOException {
        //attain catalogue's all file iterator
        RemoteIterator<LocatedFileStatus> listfiles = fs.listFiles(new Path("/"), true);
        while (listfiles.hasNext()){
            //get every file object
            LocatedFileStatus file = listfiles.next();
            System.out.println("fileName:"+file.getPath().getName()+
                    "\n"+"Blocksize:"+file.getBlockSize()+"\n"+"filesize:"+file.getLen());

            //every block size
            BlockLocation[] blockLocations = file.getBlockLocations();
            for (BlockLocation block:blockLocations){
                //偏移量
                System.out.println("Offset:"+block.getOffset());
                String[] hosts = block.getHosts();
                for(String s:hosts){
                    System.out.println(hosts);
                }
            }

            System.out.println("=======================");
        }
    }
    @Test
    /**
     * judge whether is file
     */
    public void isFile() throws IOException {
        //attain catalogue all path's object
        FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
        for (FileStatus s:fileStatuses){
            if(s.isFile()){
                System.out.println(s.getPath().getName()+" is file");
            }else{
                System.out.println(s.getPath().getName()+" is directory");
            }
        }
    }


    /**
     * IO流方式上传
     */
    @Test
    public void putFileToHDFSIO() throws IOException{

        FileInputStream fis = new FileInputStream(new File("d:/text/qqq.txt"));

        //2.输出路径
        //注意：不能/plus,记得后边写个名 比如：:/text/qqq.txt
        Path writePath = new Path("hdfs://bigdata111:9000/qqq.txt");
        FSDataOutputStream fos = fs.create(writePath);

        //3.流对接
        //InputStream in    输入
        //OutputStream out  输出
        //int buffSize      缓冲区
        //boolean close     是否关闭流
        try {
            IOUtils.copyBytes(fis,fos,4 * 1024,false);
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            IOUtils.closeStream(fos);
            IOUtils.closeStream(fis);
            fs.close();

            System.out.println("upload success");
        }
    }


    @Test
    /**
     * download
     */
    public void getFileHdfsIO() throws IOException {
        //Create input stream
        FSDataInputStream fis=fs.open(new Path("/person"));
        //Create otput stream
        FileOutputStream fos=new FileOutputStream(new File("D:/text/person.txt"));
        //Create mutual flow
        //set true auto close flow
        IOUtils.copyBytes(fis,fos,4 * 1024,true);
        System.out.println("download success");

    }
    @Test
    /**
     * Location reading the fist file block
     */
    public void readFileBloack1() throws IOException {
        //Create input stream
        FSDataInputStream fis=fs.open(new Path("/z.txt"));
        //Create otput stream
        FileOutputStream fos=new FileOutputStream(new File("D:/text/x.txt"));
        //create mutual flow
        byte[] buff = new byte[1024];
        for (int i=0;i<1;i++){
            fis.read(buff);
            fos.write(buff);
        }
        IOUtils.closeStream(fis);
        IOUtils.closeStream(fos);
        fs.close();
    }
    @Test
    /**
     * location reading the second file block
     */
    public void readFileBloack2() throws IOException {
        //Create input stream
        FSDataInputStream fis=fs.open(new Path("/z.txt"));
        //Create otput stream
        FileOutputStream fos=new FileOutputStream(new File("D:/text/x.txt"));
        //location offset
        fis.seek(128*1024*1024);
        ///create mutual stream
        IOUtils.copyBytes(fis,fos,1024,true);

        fs.close();
    }
}

合并文件

在window命令窗口中执行，type A2 » A1 然后更改后缀为rar即可。