首先在pom文件中引入hadoop相关依赖
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.1.3</version>
</dependency>
1:上传文件
// 数据库获取的数据
List<HadoopEntity> list = hadoopMapper.queryParam();
JSONArray array = JSONArray.parseArray(JSON.toJSONString(list));
String str = array.toString();
InputStream in = new ByteArrayInputStream(str.getBytes("UTF-8"));
Configuration conf = new Configuration();
conf.addResource(new Path("D:/Hadoop/hadoop-2.7.7/etc/hadoop/core-site.xml"));
// 2种FileSystem都可以,使用哪一种随意
//FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "root");
FileSystem fs = FileSystem.get(conf);
//String hdfs = "D:/Hadoop/hadoop-2.7.7/input/1.txt";
String hdfs = "hdfs://localhost:9000/input/param.txt";
FSDataOutputStream fo = fs.create(new Path(hdfs));
int len = 0;
byte[] b = new byte[1024];
while ((len = in.read(b)) > -1) {
fo.write(b, 0, len);
}
in.close();
fo.close();
2:删除文件
try {
Configuration conf = new Configuration();
conf.addResource(new Path("D:/Hadoop/hadoop-2.7.7/etc/hadoop/core-site.xml"));
FileSystem fs = FileSystem.get(conf);
// 要删除的文件路径
Boolean deleteResult = fs.delete(new Path("hdfs://localhost:9000/input/1.txt"), true);
System.out.println(deleteResult);
} catch (Exception e) {
log.error("删除文件失败" + e.getMessage());
}
3:下载和解析文件
public static void main(String[] args) {
try {
Configuration conf = new Configuration();
conf.addResource(new Path("D:/Hadoop/hadoop-2.7.7/etc/hadoop/core-site.xml"));
FileSystem fs = FileSystem.get(conf);
InputStream in = fs.open(new Path("hdfs://localhost:9000/input/param.txt"));
byte[] buff = new byte[2048];
int hasRead;
StringBuffer sbf = new StringBuffer();
while ((hasRead = in.read(buff)) > 0) {
sbf.append(new String(buff, 0, hasRead, "GBK"));
}
HadoopEntity hadoopEntity = new HadoopEntity();
JSONArray array = JSONArray.parseArray(String.valueOf(sbf));
List<HadoopEntity> list = JSONObject.parseArray(array.toJSONString(), HadoopEntity.class);
System.out.println(list);
// 下载到本地
// OutputStream out = new FileOutputStream("D:/param.txt");
// IOUtils.copyBytes(in, out, 4096, true);
} catch (Exception e) {
System.out.println(e.getMessage());
log.error("下载文件失败" + e.getMessage());
}
// try {
// hdfsReadFile("hdfs://localhost:9000/input/param.txt","D:/1111/hw.txt");
// } catch (Exception e) {
// log.error("下载文件失败" + e.getMessage());
// }
}
public static void hdfsReadFile(String hdfsFile, String fileName) throws Exception {
// 创建配置对象实例
Configuration cfg = new Configuration();
//设置操作的文件系统是HDFS,并且指定HDFS操作地址
cfg.addResource(new Path("D:/Hadoop/hadoop-2.7.7/etc/hadoop/core-site.xml"));
//cfg.set("fs.defaultFS",hdfsUrl);
//创建FileSystem 对象实例
FileSystem fileSystem = FileSystem.get(new URI("hdfs://localhost:9000"), cfg, "root");
//FileSystem fileSystem = FileSystem.get(cfg);
FSDataInputStream fsdiStream = null;
FileOutputStream fileOutputStream = null;
try {
if (!fileSystem.exists(new Path(hdfsFile))) {
System.out.println("111");
}
//读取操作,从hdfs指定的文件读出数据对象
fsdiStream = fileSystem.open(new Path(hdfsFile));
fileOutputStream = new FileOutputStream(fileName);
byte[] buffer = new byte[2048];
int count = fsdiStream.read(buffer, 0, 2048);
while (count > 0) {
fileOutputStream.write(buffer, 0, count);
count = fsdiStream.read(buffer, 0, 2048);
}
} catch (IllegalArgumentException e) {
System.out.println(e.getMessage());
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
fsdiStream.close();
fileOutputStream.close();
}
}