一、客户端环境准备
1.相关包
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.6.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.5</version>
</dependency>
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.8</version>
<scope>system</scope>
<systemPath>C:/Program Files/Java/jdk1.8.0_152/lib/tools.jar</systemPath>
</dependency>
二、基础操作代码
1.创建连接
public FileSystem createNewFileSystem(){
try {
Configuration conf = new Configuration();
URI uri = new URI("hdfs://ip:port");
String scheme = uri.getScheme();
String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
conf.setBoolean(disableCacheName, true);
System.setProperty("hadoop.home.dir", "D:\\hadoop-3.1.1");
FileSystem fs = FileSystem.newInstance(uri,conf , "用户名");
return fs;
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
}
}
2.文件上传
FSDataOutputStream out = null;
InputStream is = null;
try {
//hdfs路径格式 /权限文件夹/系统名称/当天日期/文件名称
String hdfsPath = "/"+fileFolder+"/"+resourceSystem+"/"+DateUtils.getDate()+"/"+fileName;
FileSystem fs = createNewFileSystem();
out = fs.create(new Path(hdfsPath));
is = handler.getInputStream();
byte[] b = new byte[is.available()];
int read = 0;
while((read = is.read(b)) > 0){
out.write(b, 0, read);
}
//保存数据入库
return JSONArray.toJSON(AjaxResult.success(uuid)).toString();
} catch (Exception e) {
e.printStackTrace();
return JSONArray.toJSON(AjaxResult.error("文件上传失败")).toString();
} finally {
try {
if (is != null) {
is.close();
}
if (out != null) {
out.close();
}
if(fs != null){
fs.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
3.文件下载
1.本地下载
try{
FileSystem fs = createNewFileSystem();
fs.copyToLocalFile(false, new Path("/hello.txt"), new Path("d:\\hello.txt"), true);
}catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally {
if(fs != null){
fs.close();
}
}
2.返回byte数组
FSDataInputStream hdfsInStream = null;
byte ioBuffer[] = null;
FileSystem fs = createNewFileSystem();
//查询文件是否存在,
FileResource fileResource = iFileResourceService.selectFileResourceByFileid(fileid);
if(null!= fileResource) {
Path path = new Path(fileResource.getResourceFilePath());
try {
hdfsInStream = fs.open(path);
ioBuffer = new byte[hdfsInStream.available()];
IOUtils.read(hdfsInStream, ioBuffer);
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
if (hdfsInStream!= null) {
hdfsInStream.close();
}
if(fs != null{
fs.close();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return ioBuffer;
}else {
return null;
}
}
4.文件删除
待添加