1、pom依赖
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.1.1.3.1.4.0-315</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>3.1.1.3.1.4.0-315</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.1.1.3.1.4.0-315</version>
</dependency>
2、HDFS配置类
@Configuration
public class HdfsConfig {
@Value("${xxxx.xxxx.xxxx.xxxx}")
private String defaultHdfsUri;
@Value("${xxxx.xxxx.xxxx.xxxx}")
private String nameservices;
@Value("${xxxx.xxxx.xxxx.xxxx}")
private String nn1;
@Value("${xxxx.xxxx.xxxx.xxxx}")
private String nn2;
@Bean
public HdfsUtil getHdfsService() {
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
conf.set("fs.defaultFS", defaultHdfsUri);
conf.set("dfs.nameservices", nameservices);
conf.set("dfs.ha.namenodes."+nameservices, "nn1,nn2");
conf.set("dfs.namenode.rpc-address."+nameservices+".nn1", nn1);
conf.set("dfs.namenode.rpc-address."+nameservices+".nn2", nn2);
conf.set("dfs.client.failover.proxy.provider."+nameservices, "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
return new HdfsUtil(conf, defaultHdfsUri);
}
}
3、HDFS工具类
public class HdfsUtil {
Zlogger logger = ZlogFactory.getZlogger(HdfsUtil.class);
private Configuration conf = null;
/**
* 默认的HDFS路径,比如:hdfs://192.168.197.130:9000
*/
private String defaultHdfsUri;
//通过这种方式设置java客户端身份
// System.setProperty("HADOOP_USER_NAME", "hdfs");
// FileSystem fs = FileSystem.get(conf);
//或者使用下面的方式设置客户端身份
//FileSystem fs = FileSystem.get(new URI("hdfs://master:9000"),conf,"root");
public HdfsUtil(Configuration conf, String defaultHdfsUri) {
this.conf = conf;
this.defaultHdfsUri = defaultHdfsUri;
}
/**
* 获取HDFS文件系统
*
* @return org.apache.hadoop.fs.FileSystem
*/
private FileSystem getFileSystem() throws IOException, URISyntaxException, InterruptedException {
return FileSystem.get(new URI(defaultHdfsUri), conf, "hdfs");
}
/**
* 创建HDFS目录
*
* @param path HDFS的相对目录路径,比如:/testDir
* @return boolean 是否创建成功
* @author adminstrator
* @since 1.0.0
*/
public boolean mkdir(String path) {
//如果目录已经存在,则直接返回
if (checkExists(path)) {
return true;
} else {
FileSystem fileSystem = null;
try {
fileSystem = getFileSystem();
//最终的HDFS文件目录
String hdfsPath = generateHdfsPath(path);
//创建目录
return fileSystem.mkdirs(new Path(hdfsPath));
} catch (Exception e) {
logger.error(MessageFormat.format("创建HDFS目录失败,path:{0}", path), e);
return false;
} finally {
close(fileSystem);
}
}
}
/**
* 上传文件至HDFS
*
* @param srcFile 本地文件路径,比如:D:/test.txt
* @param dstPath HDFS的相对目录路径,比如:/testDir
* @author adminstrator
* @since 1.0.0
*/
public void uploadFileToHdfs(String srcFile, String dstPath) {
this.uploadFileToHdfs(false, true, srcFile, dstPath);
}
/**
* 上传文件至HDFS
*
* @param delSrc 是否删除本地文件
* @param overwrite 是否覆盖HDFS上面的文件
* @param srcFile 本地文件路径,比如:D:/test.txt
* @param dstPath HDFS的相对目录路径,比如:/testDir
* @author adminstrator
* @since 1.0.0
*/
public void uploadFileToHdfs(boolean delSrc, boolean overwrite, String srcFile, String dstPath) {
//源文件路径
Path localSrcPath = new Path(srcFile);
//目标文件路径
Path hdfsDstPath = new Path(generateHdfsPath(dstPath));
FileSystem fileSystem = null;
try {
fileSystem = getFileSystem();
fileSystem.copyFromLocalFile(delSrc, overwrite, localSrcPath, hdfsDstPath);
} catch (Exception e) {
logger.error(MessageFormat.format("上传文件至HDFS失败,srcFile:{0},dstPath:{1}", srcFile, dstPath), e);
} finally {
close(fileSystem);
}
}
/**
* 判断文件或者目录是否在HDFS上面存在
*
* @param path HDFS的相对目录路径,比如:/testDir、/testDir/a.txt
* @return boolean
* @author adminstrator
* @since 1.0.0
*/
public boolean checkExists(String path) {
FileSystem fileSystem = null;
try {
fileSystem = getFileSystem();
//最终的HDFS文件目录
String hdfsPath = generateHdfsPath(path);
//创建目录
return fileSystem.exists(new Path(hdfsPath));
} catch (Exception e) {
logger.error(MessageFormat.format("'判断文件或者目录是否在HDFS上面存在'失败,path:{0}", path), e);
return false;
} finally {
close(fileSystem);
}
}
/**
* HDFS创建文件
* @param path
* @param file
* @throws Exception
*/
public void createFile(String path, MultipartFile file) throws Exception {
if (StringUtils.isEmpty(path) || null == file.getBytes()) {
return;
}
String fileName = file.getOriginalFilename();
String newFileName = UUID.randomUUID().toString().replaceAll("-","")+ "_" + fileName;
FileSystem fs = getFileSystem();
// 上传时默认当前目录,后面自动拼接文件的目录
Path newPath = new Path(path + "/" + newFileName);
// 打开一个输出流
FSDataOutputStream outputStream = fs.create(newPath);
byte[] bytes = file.getBytes();
// todo
outputStream.write(bytes);
outputStream.close();
fs.close();
}
/**
* 获取HDFS上面的某个路径下面的所有文件或目录(不包含子目录)信息
*
* @param path HDFS的相对目录路径,比如:/testDir
* @return java.util.List<java.util.Map < java.lang.String, java.lang.Object>>
* @author adminstrator
* @since 1.0.0
*/
public List<Map<String, Object>> listFiles(String path, PathFilter pathFilter) {
//返回数据
List<Map<String, Object>> result = new ArrayList<>();
//如果目录已经存在,则继续操作
if (checkExists(path)) {
FileSystem fileSystem = null;
try {
fileSystem = getFileSystem();
//最终的HDFS文件目录
String hdfsPath = generateHdfsPath(path);
FileStatus[] statuses;
//根据Path过滤器查询
if (pathFilter != null) {
statuses = fileSystem.listStatus(new Path(hdfsPath), pathFilter);
} else {
statuses = fileSystem.listStatus(new Path(hdfsPath));
}
if (statuses != null) {
for (FileStatus status : statuses) {
//每个文件的属性
Map<String, Object> fileMap = new HashMap<>(2);
fileMap.put("path", status.getPath().toString());
fileMap.put("isDir", status.isDirectory());
fileMap.put("fileStatus", status.toString());
result.add(fileMap);
}
}
} catch (Exception e) {
logger.error(MessageFormat.format("获取HDFS上面的某个路径下面的所有文件失败,path:{0}", path), e);
} finally {
close(fileSystem);
}
}
return result;
}
/**
* 读取HDFS文件内容
* @param path
* @return
* @throws Exception
*/
public String readFile(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!checkExists(path)) {
return null;
}
FileSystem fs = getFileSystem();
// 目标路径
Path srcPath = new Path(path);
FSDataInputStream inputStream = null;
try {
inputStream = fs.open(srcPath);
// 防止中文乱码
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
String lineTxt = "";
StringBuffer sb = new StringBuffer();
while ((lineTxt = reader.readLine()) != null) {
sb.append(lineTxt);
}
return sb.toString();
} finally {
inputStream.close();
fs.close();
}
}
public String readFileToPrint(String path, int readline) throws Exception {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!checkExists(path)) {
return null;
}
FileSystem fs = getFileSystem();
// 目标路径
Path srcPath = new Path(path);
FSDataInputStream inputStream = null;
try {
inputStream = fs.open(srcPath);
// 防止中文乱码
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
String lineTxt = "";
StringBuffer sb = new StringBuffer();
for (int i = 0; i < readline; i++) {
lineTxt = reader.readLine();
sb.append(lineTxt);
}
System.out.println(sb.toString());
return sb.toString();
} finally {
inputStream.close();
fs.close();
}
}
/**
* 从HDFS下载文件至本地
*
* @param srcFile HDFS的相对目录路径,比如:/testDir/a.txt
* @param dstFile 下载之后本地文件路径(如果本地文件目录不存在,则会自动创建),比如:D:/test.txt
* @author adminstrator
* @since 1.0.0
*/
public void downloadFileFromHdfs(String srcFile, String dstFile) {
//HDFS文件路径
Path hdfsSrcPath = new Path(generateHdfsPath(srcFile));
//下载之后本地文件路径
Path localDstPath = new Path(dstFile);
FileSystem fileSystem = null;
try {
fileSystem = getFileSystem();
fileSystem.copyToLocalFile(hdfsSrcPath, localDstPath);
} catch (Exception e) {
logger.error(MessageFormat.format("从HDFS下载文件至本地失败,srcFile:{0},dstFile:{1}", srcFile, dstFile), e);
} finally {
close(fileSystem);
}
}
/**
* 打开HDFS上面的文件并返回 InputStream
*
* @param path HDFS的相对目录路径,比如:/testDir/c.txt
* @return FSDataInputStream
* @author adminstrator
* @since 1.0.0
*/
public FSDataInputStream open(String path) {
//HDFS文件路径
Path hdfsPath = new Path(generateHdfsPath(path));
FileSystem fileSystem = null;
try {
fileSystem = getFileSystem();
return fileSystem.open(hdfsPath);
} catch (Exception e) {
logger.error(MessageFormat.format("打开HDFS上面的文件失败,path:{0}", path), e);
}
return null;
}
/**
* 打开HDFS上面的文件并返回byte数组,方便Web端下载文件
* <p>new ResponseEntity<byte[]>(byte数组, headers, HttpStatus.CREATED);</p>
* <p>或者:new ResponseEntity<byte[]>(FileUtils.readFileToByteArray(templateFile), headers, HttpStatus.CREATED);</p>
*
* @param path HDFS的相对目录路径,比如:/testDir/b.txt
* @return FSDataInputStream
* @author adminstrator
* @since 1.0.0
*/
public byte[] openWithBytes(String path) {
//HDFS文件路径
Path hdfsPath = new Path(generateHdfsPath(path));
FileSystem fileSystem = null;
FSDataInputStream inputStream = null;
try {
fileSystem = getFileSystem();
inputStream = fileSystem.open(hdfsPath);
return IOUtils.toByteArray(inputStream);
} catch (Exception e) {
logger.error(MessageFormat.format("打开HDFS上面的文件失败,path:{0}", path), e);
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
// ignore
}
}
}
return null;
}
/**
* 打开HDFS上面的文件并返回String字符串
*
* @param path HDFS的相对目录路径,比如:/testDir/b.txt
* @return FSDataInputStream
* @author adminstrator
* @since 1.0.0
*/
public String openWithString(String path) {
//HDFS文件路径
Path hdfsPath = new Path(generateHdfsPath(path));
FileSystem fileSystem = null;
FSDataInputStream inputStream = null;
try {
fileSystem = getFileSystem();
inputStream = fileSystem.open(hdfsPath);
return IOUtils.toString(inputStream, Charset.forName("UTF-8"));
} catch (Exception e) {
logger.error(MessageFormat.format("打开HDFS上面的文件失败,path:{0}", path), e);
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
// ignore
}
}
}
return null;
}
/**
* 打开HDFS上面的文件并转换为Java对象(需要HDFS上门的文件内容为JSON字符串)
*
* @param path HDFS的相对目录路径,比如:/testDir/c.txt
* @return FSDataInputStream
* @author adminstrator
* @since 1.0.0
*/
public <T extends Object> T openWithObject(String path, Class<T> clazz) {
//1、获得文件的json字符串
String jsonStr = this.openWithString(path);
//2、使用com.alibaba.fastjson.JSON将json字符串转化为Java对象并返回
return JSON.parseObject(jsonStr, clazz);
}
/**
* 重命名
*
* @param srcFile 重命名之前的HDFS的相对目录路径,比如:/testDir/b.txt
* @param dstFile 重命名之后的HDFS的相对目录路径,比如:/testDir/b_new.txt
* @author adminstrator
* @since 1.0.0
*/
public boolean rename(String srcFile, String dstFile) {
//HDFS文件路径
Path srcFilePath = new Path(generateHdfsPath(srcFile));
//下载之后本地文件路径
Path dstFilePath = new Path(dstFile);
FileSystem fileSystem = null;
try {
fileSystem = getFileSystem();
return fileSystem.rename(srcFilePath, dstFilePath);
} catch (Exception e) {
logger.error(MessageFormat.format("重命名失败,srcFile:{0},dstFile:{1}", srcFile, dstFile), e);
} finally {
close(fileSystem);
}
return false;
}
/**
* 删除HDFS文件或目录
*
* @param path HDFS的相对目录路径,比如:/testDir/c.txt
* @return boolean
* @author adminstrator
* @since 1.0.0
*/
public boolean delete(String path) {
//HDFS文件路径
Path hdfsPath = new Path(generateHdfsPath(path));
FileSystem fileSystem = null;
try {
fileSystem = getFileSystem();
return fileSystem.delete(hdfsPath, true);
} catch (Exception e) {
logger.error(MessageFormat.format("删除HDFS文件或目录失败,path:{0}", path), e);
} finally {
close(fileSystem);
}
return false;
}
/**
* 获取某个文件在HDFS集群的位置
*
* @param path HDFS的相对目录路径,比如:/testDir/a.txt
* @return org.apache.hadoop.fs.BlockLocation[]
* @author adminstrator
* @since 1.0.0
*/
public BlockLocation[] getFileBlockLocations(String path) {
//HDFS文件路径
Path hdfsPath = new Path(generateHdfsPath(path));
FileSystem fileSystem = null;
try {
fileSystem = getFileSystem();
FileStatus fileStatus = fileSystem.getFileStatus(hdfsPath);
return fileSystem.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
} catch (Exception e) {
logger.error(MessageFormat.format("获取某个文件在HDFS集群的位置失败,path:{0}", path), e);
} finally {
close(fileSystem);
}
return null;
}
/**
* 将相对路径转化为HDFS文件路径
*
* @param dstPath 相对路径,比如:/data
* @return java.lang.String
* @author adminstrator
* @since 1.0.0
*/
private String generateHdfsPath(String dstPath) {
String hdfsPath = defaultHdfsUri;
if (dstPath.startsWith("/")) {
hdfsPath += dstPath;
} else {
hdfsPath = hdfsPath + "/" + dstPath;
}
return hdfsPath;
}
/**
* close方法
*/
private void close(FileSystem fileSystem) {
if (fileSystem != null) {
try {
fileSystem.close();
} catch (IOException e) {
logger.error(e.getMessage());
}
}
}
/**
* HDFS文件复制
* @param sourcePath
* @param targetPath
* @throws Exception
*/
public void copyFile(String sourcePath, String targetPath) throws Exception {
if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) {
return;
}
FileSystem fs = getFileSystem();
// 原始文件路径
Path oldPath = new Path(sourcePath);
// 目标路径
Path newPath = new Path(targetPath);
FSDataInputStream inputStream = null;
FSDataOutputStream outputStream = null;
try {
inputStream = fs.open(oldPath);
outputStream = fs.create(newPath);
//Files.copy(inputStream,outputStream);
IOUtils.copy(inputStream, outputStream);
} finally {
inputStream.close();
outputStream.close();
fs.close();
}
}
}
需要某项具体功能嗲用或重写工具类中的方法即可。