idea连接Hadoop操作指令

配置文件

创建文件

迭代删除

获取当前目录下 所有文件的对象

查看

写入

上传和下载

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Before;
import org.junit.Test;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.URI;


/**
 *  RPC 网络编程
 *  1.连接
 */
public class HadoopAPI {
    FileSystem fs;
    @Before
    public void init() throws Exception{
        //hadoop配置文件 自动获取haddop-hdfs的配置文件
        Configuration conf = new Configuration();
        conf.set("dfs.replication", "1");
        URI uri = new URI("hdfs://master:9000");
        fs = FileSystem.get(uri, conf);
    }
    @Test
    public void mkdir() throws Exception{
        fs.mkdirs(new Path("/mk"));
    }
    @Test
    public void delete()throws Exception{
        // true:迭代删除
        fs.delete(new Path("/data"),true);
    }
    @Test
    public void listStatus()throws Exception{
        // 获取当前目录下 所有文件的对象
        FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
        for (FileStatus fileStatus : fileStatuses) {
            System.out.println(fileStatus.getLen());
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getPath());
            System.out.println(fileStatus.getReplication());
        }
    }

    @Test
    public void getFileStatus()throws Exception{
        FileStatus fileStatus = fs.getFileStatus(new Path("/student.txt"));
        System.out.println(fileStatus);
    }

    //查看
    @Test
    public void load()throws Exception{
        FSDataInputStream open = fs.open(new Path("/student.txt"));
        BufferedReader br = new BufferedReader(new InputStreamReader(open));
        String line;
        while ((line=br.readLine())!=null){
            System.out.println(line);
        }
        br.close();
        open.close();
    }

    @Test
    public void create()throws Exception{
        FSDataOutputStream fsDataOutputStream = fs.create(new Path("/test.txt"));
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream));
        bw.write("你好！");
        bw.newLine();
        bw.write("世界！");
        bw.newLine();
        bw.close();
        fsDataOutputStream.close();
    }

    //通过代码复制 实现上传和下载
    //上传
    @Test
    public void copyFromLocalFile() throws Exception{
        Path hdfs = new Path("/");
        Path local = new Path("E:\ideaFile\shujia\bd13\data\students.txt");
        fs.copyFromLocalFile(local,hdfs);
    }
    //下载
    @Test
    public void copyToLocalFile()throws Exception{
        Path path = new Path("/students.txt");
        Path local = new Path("E:\ideaFile\shujia\bd13\data");
        fs.copyToLocalFile(false,path,local,true);
    }

}