使用java对HDFS文件进行读写

HDFS读文件

HDFS写文件

代码实例

先建maven工程

配置pom.xml,陪、配完静等依赖包下载完毕

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>cn.kgc.kbll</groupId>
  <artifactId>2021319-hdfs</artifactId>
  <version>1.0-SNAPSHOT</version>

  <name>2021319-hdfs</name>
  <!-- FIXME change it to the project's website -->
  <url>http://www.example.com</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <maven.compiler.source>1.8</maven.compiler.source>
    <maven.compiler.target>1.8</maven.compiler.target>
  </properties>

  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.12</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>2.6.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client</artifactId>
      <version>2.6.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <version>2.6.0</version>
    </dependency>
  </dependencies>


</project>

java代码

package cn.kgc.kbll.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * @Author ZhangPeng
 * @Date 2021/3/19
 * @Description
 */
public class TestHDFS {
    FileSystem fs;

    //初始化fs
    public void initFS(String fsPath,String user){
        Configuration conf = new Configuration();
        try {
            fs = FileSystem.get(new URI(fsPath),conf,user);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

    //创建文件夹
    public void createDir(String path){
        try {
            fs.mkdirs(new Path(path));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //上传文件
    public void put(String src,String dest){
        try {
            fs.copyFromLocalFile(new Path(src),new Path(dest));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //下载文件
    public void get(String src,String dest){
        try {
            fs.copyToLocalFile(new Path(src),new Path(dest));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //删除文件
    public void delete(String path){
        try {
            fs.delete(new Path(path));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //创建文件并写入内容
    public void createFile(String path,String content){
        try {
            FSDataOutputStream os = fs.create(new Path(path));
            os.write(content.getBytes());
            os.flush();
            fs.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //读文件显示在控制台
    public void readFile(String path) {
        try {
            FSDataInputStream is = fs.open(new Path(path));
            IOUtils.copyBytes(is,System.out,1024);
            fs.close();
        }catch (IOException e){
            e.printStackTrace();
        }
    }

    //关闭hadoop文件系统
    public void close(){
        try {
            fs.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {
        TestHDFS ts = new TestHDFS();
        ts.initFS(args[0],args[1]);
        int argc = args.length;
        switch (args[argc - 1]){
            case "c":
                ts.createDir(args[argc-2]);
                break;
            case "p":
                ts.put(args[argc-3],args[argc-2]);
                break;
            case "g":
                ts.get(args[argc-3],args[argc-2]);
                break;
            case "d":
                ts.delete(args[argc-2]);
                break;
            case "w":
                ts.createFile(args[argc-3],args[argc-2]);
                break;
            case "r":
                ts.readFile(args[argc-2]);
                break;
            default:
                break;
        }
        ts.close();


    }
}

Linux虚拟机运行jar包

1.将上述java代码打成jar包传达虚拟机上
2.执行如下命令运行：

hadoop jar _2021319hdfs.jar cn.kgc.kbll.test.TestHDFS hdfs://192.168.56.12:9000 root /test/java/hadoop/b.txt r

_2021319hdfs.jar：为要运行的jar包
cn.kgc.kbll.test.TestHDFS：主类的全类名
后面全为要传递的参数