JAVA实现HDFS文件的读写

1. 启动hdfs: start-all.sh

2. 创建upload.txt文件：hadoop fs -touchz /upload.txt

3. 在本地创建test.txt文件并将下面文字写入：

   The Apache™ Hadoop® project develops open-source software for reliable, scalable, distributed computing.

   The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using simple programming models. It is designed to scale up from single servers to thousands of machines, each offering local computation and storage. Rather than rely on hardware to deliver high-availability, the library itself is designed to detect and handle failures at the application layer, so delivering a highly-available service on top of a cluster of computers, each of which may be prone to failures

4. 

5.  编写Java代码：

package com.hadoop.RWFile;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * Created with Intellij IDEA
 *
 * @Auther:zhangsong
 * @Date:2022-04-22-17:02
 * @Description:hdfs练手java程序
 */
public class RWriteFile {

    //作业的配置信息类，任何作用的配置信息必须通过Configuration传递，因为通过Configuration可以实现在多个mapper和多个reducer任务之间共享信息
    private static Configuration configuration;
    static {
        try {
            //初始化cofiguration
            configuration = new Configuration();
            configuration.set("fs.default.name","hdfs://192.168.37.128:9000");
        }catch (Exception e){
            e.printStackTrace();
        }
    }
    /*
    * 方法说明：读取文件内容并存入list集合中
    * 参数说明：path(需要读取的文件路径)
    * */
    public static List<String> readFile(String path)  {
        //1.list对象存取读取到的文件内容
        List<String> list = new ArrayList<String>();
        try {
            //2.文件读取流
            FileReader fr = new FileReader(new File(path));
            //使用BufferedReader进行缓冲
            BufferedReader br = new BufferedReader(fr);
            //3.开辟字符串存储空间，用与接收读取的文件数据
            String line ;
            //4.读到没有行可以读以后结束
            while ((line = br.readLine()) != null){
                //循环写入ArrayList
                list.add(line);
            }
            //4.关闭资源
            fr.close();
            br.close();
        }catch (Exception e){
            e.printStackTrace();
        }
        //返回list
        return list;
    }
    /*
    * 方法说明：写文件到hdfs
    * 参数说明：hdfsPath(hdfs文件系统需要接收数据的文件路径，upload.txt的路径，这里我输入：hdfs:/upload.txt)
    *         list readFile()方法的返回值，也就是需要写入的数据
    * */
    public static void writeFile(String hdfsPath,List<String> list){
        try {

            //1,获得hdfs文件路径
            FileSystem fileSystem = FileSystem.get(configuration);
            //2.hdfs文件输出流
            FSDataOutputStream fsDataOutputStream = fileSystem.append(new Path(hdfsPath));
            //3.遍历list集合并写入文件
            for(String s : list){
                fsDataOutputStream.writeBytes(s);
            }
            //4.关闭资源
            fileSystem.close();
            fsDataOutputStream.close();
            System.out.println("文件写入成功");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {
        List<String> strings = RWriteFile.readFile("C:\\Users\\lenovo\\Desktop\\1.txt");
        RWriteFile.writeFile("hdfs:/upload.txt",strings);
    }
}

5、运行程序

6、在NameNode上查看写入的信息即可。