java操作HDFS，上传文件，删除文件，下载文件，解析文件数据

首先在pom文件中引入hadoop相关依赖

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.1.3</version>
        </dependency>

1:上传文件

        // 数据库获取的数据
        List<HadoopEntity> list = hadoopMapper.queryParam();

        JSONArray array = JSONArray.parseArray(JSON.toJSONString(list));

        String str = array.toString();

        InputStream in = new ByteArrayInputStream(str.getBytes("UTF-8"));
        Configuration conf = new Configuration();
        conf.addResource(new Path("D:/Hadoop/hadoop-2.7.7/etc/hadoop/core-site.xml"));
        // 2种FileSystem都可以，使用哪一种随意

        //FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "root");

        FileSystem fs = FileSystem.get(conf);
        //String hdfs = "D:/Hadoop/hadoop-2.7.7/input/1.txt";
        String hdfs = "hdfs://localhost:9000/input/param.txt";
        FSDataOutputStream fo = fs.create(new Path(hdfs));
        int len = 0;
        byte[] b = new byte[1024];
        while ((len = in.read(b)) > -1) {
            fo.write(b, 0, len);
        }
        in.close();
        fo.close();

2:删除文件

try {
            Configuration conf = new Configuration();
            conf.addResource(new Path("D:/Hadoop/hadoop-2.7.7/etc/hadoop/core-site.xml"));
            FileSystem fs = FileSystem.get(conf);
            // 要删除的文件路径
            Boolean deleteResult = fs.delete(new Path("hdfs://localhost:9000/input/1.txt"), true);
            System.out.println(deleteResult);
        } catch (Exception e) {
            log.error("删除文件失败" + e.getMessage());
        }

3：下载和解析文件

public static void main(String[] args) {
        try {
            Configuration conf = new Configuration();
            conf.addResource(new Path("D:/Hadoop/hadoop-2.7.7/etc/hadoop/core-site.xml"));
            FileSystem fs = FileSystem.get(conf);
            InputStream in = fs.open(new Path("hdfs://localhost:9000/input/param.txt"));
            byte[] buff = new byte[2048];
            int hasRead;
            StringBuffer sbf = new StringBuffer();
            while ((hasRead = in.read(buff)) > 0) {
                sbf.append(new String(buff, 0, hasRead, "GBK"));
            }
            HadoopEntity hadoopEntity = new HadoopEntity();
            JSONArray array = JSONArray.parseArray(String.valueOf(sbf));
            List<HadoopEntity> list = JSONObject.parseArray(array.toJSONString(), HadoopEntity.class);
            System.out.println(list);
            // 下载到本地
//        OutputStream out = new FileOutputStream("D:/param.txt");
//        IOUtils.copyBytes(in, out, 4096, true);
        } catch (Exception e) {
            System.out.println(e.getMessage());
            log.error("下载文件失败" + e.getMessage());
        }
//        try {
//        hdfsReadFile("hdfs://localhost:9000/input/param.txt","D:/1111/hw.txt");
//        } catch (Exception e) {
//            log.error("下载文件失败" + e.getMessage());
//        }

    }

    public static void hdfsReadFile(String hdfsFile, String fileName) throws Exception {
        // 创建配置对象实例
        Configuration cfg = new Configuration();
        //设置操作的文件系统是HDFS，并且指定HDFS操作地址
        cfg.addResource(new Path("D:/Hadoop/hadoop-2.7.7/etc/hadoop/core-site.xml"));
        //cfg.set("fs.defaultFS",hdfsUrl);
        //创建FileSystem 对象实例
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://localhost:9000"), cfg, "root");
        //FileSystem fileSystem = FileSystem.get(cfg);
        FSDataInputStream fsdiStream = null;
        FileOutputStream fileOutputStream = null;
        try {
            if (!fileSystem.exists(new Path(hdfsFile))) {
                System.out.println("111");
            }
            //读取操作，从hdfs指定的文件读出数据对象
            fsdiStream = fileSystem.open(new Path(hdfsFile));
            fileOutputStream = new FileOutputStream(fileName);
            byte[] buffer = new byte[2048];
            int count = fsdiStream.read(buffer, 0, 2048);
            while (count > 0) {
                fileOutputStream.write(buffer, 0, count);
                count = fsdiStream.read(buffer, 0, 2048);

            }

        } catch (IllegalArgumentException e) {
            System.out.println(e.getMessage());
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            fsdiStream.close();
            fileOutputStream.close();
        }
    }