Hadoop HDFS 基本API使用

一叶龙洲

于 2023-03-15 10:28:35 发布

阅读量115

点赞数

分类专栏： # Hadoop 文章标签： hadoop hdfs 大数据

本文链接：https://blog.csdn.net/LXdragon1991/article/details/129546935

版权

Hadoop 专栏收录该内容

2 篇文章 0 订阅

订阅专栏

该文章展示了如何在Java中使用Hadoop进行文件操作，包括添加Hadoop相关依赖，配置core-site.yaml和hdfs-site.yaml，以及进行连接、创建目录、上传文件、检查数据块位置和读取文件内容的测试。

摘要由CSDN通过智能技术生成

1、添加依赖

<!-- hadoop -->
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>2.6.5</version>
    </dependency>
      <dependency>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-hdfs</artifactId>
          <version>2.6.5</version>
          <scope>test</scope>
      </dependency>

    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.13.2</version>
      <scope>compile</scope>
    </dependency>

2、添加resource

添加core-site.yaml

添加hdfs-site.yaml

3、测试

public class HdfsTest {
    private Configuration configuration = null;
    private FileSystem fs = null;

    @Before
    public void connect() throws Exception {
        configuration = new Configuration(true);
//        fs = FileSystem.get(configuration);
        fs = FileSystem.get(URI.create("hdfs://node01:9000"), configuration, "root");
    }

    @Test
    public void mkdir() throws IOException {
        Path path = new Path("/msb01");
        if (fs.exists(path)) {
            fs.delete(path, true);
        }
        fs.mkdirs(path);
    }

    @Test
    public void upload() throws IOException {
        BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(new File("./data/hello.txt")));
        Path outfile = new Path("/msb01/out.txt");
        FSDataOutputStream outputStream = fs.create(outfile);
        IOUtils.copyBytes(inputStream, outputStream, configuration, true);
    }

    @Test
    public void blocks() throws IOException {
        Path file = new Path("/user/root/data.txt");
        FileStatus fileStatus = fs.getFileStatus(file);
        BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
        for (BlockLocation b : fileBlockLocations) {
            System.out.println(b);
        }

        FSDataInputStream in = fs.open(file);
        in.seek(1048576);

        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
        System.out.println((char)in.readByte());
    }


    @After
    public void close() throws IOException {
        fs.close();
    }
}