ManagerHdfsUtil.class
package cn.getech.data.manager.utils;
import cn.getech.data.development.constant.DataDevelopmentBizExceptionEnum;
import cn.getech.data.intelligence.common.exception.RRException;
import cn.getech.data.manager.config.ManagerDataConfig;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.log4j.Logger;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
/**
* hdfs工具类
*/
public class ManagerHdfsUtil {
private static Logger logger = Logger.getLogger(ManagerHdfsUtil.class);
public DistributedFileSystem dfs;
public Configuration conf;
public ManagerHdfsUtil(ManagerDataConfig managerDataConfig) throws Exception {
// Configuration conf=new Configuration(false);
conf = new Configuration(false);
String ns = managerDataConfig.getNamespace();
String[] nameNodesAddr = managerDataConfig.getNamenodestr().split(",");
String[] nameNodes = {"nn1", "nn2"};
conf.set("fs.defaultFS", "hdfs://" + ns);
conf.set("dfs.nameservices", ns);
conf.set("dfs.ha.namenodes." + ns, nameNodes[0] + "," + nameNodes[1]);
conf.set("dfs.namenode.rpc-address." + ns + "." + nameNodes[0], nameNodesAddr[0]);
conf.set("dfs.namenode.rpc-address." + ns + "." + nameNodes[1], nameNodesAddr[1]);
conf.set("dfs.client.failover.proxy.provider." + ns, "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
conf.set("dfs.client.use.datanode.hostname", "true");
conf.set("dfs.socket.timeout", "6000000");
//路径权限
conf.set("fs.permissions.umask-mode","000");
String hdfsRPCUrl = "hdfs://" + ns + ":" + managerDataConfig.getNameport();
dfs = new DistributedFileSystem();
dfs.initialize(URI.create(hdfsRPCUrl), conf);
}
/**
* 获取输出流
*
* @param dst 目标路径
* @return 成功还是失败
*/
public FSDataOutputStream writeFile( String dst) {
Path dstPath = new Path(dst);
try {
FSDataOutputStream outputStream;
if (dfs.exists(dstPath)) {
dfs.delete(dstPath, true);
}
if (!dfs.exists(dstPath.getParent())) {
dfs.mkdirs(dstPath.getParent());
}
//Init output stream
outputStream = dfs.create(dstPath);
return outputStream;
} catch (IOException ie) {
logger.error(ie.getMessage());
throw new RRException("获取流失败");
}
}
/**
* 上传方法
*
* @param fileContent 内容
* @param dst 目标路径
* @return 成功还是失败
*/
public boolean writeFile(byte[] fileContent, String dst) {
Path dstPath = new Path(dst);
try {
if (dfs.exists(dstPath)) {
dfs.delete(dstPath, true);
}
if (!dfs.exists(dstPath.getParent())) {
dfs.mkdirs(dstPath.getParent());
}
//Init output stream
FSDataOutputStream outputStream = dfs.create(dstPath);
//Cassical output stream usage
outputStream.write(fileContent);
outputStream.close();
} catch (IOException ie) {
logger.error(ie.getMessage());
return false;
}
return true;
}
/**
* 上传方法
*
* @param fileContent 内容
* @param dst 目标路径
* @param partitions 分区路径
* @param writeType 追加方式 2:追加 1:覆盖
* @return 成功还是失败
*/
public boolean writeFile(byte[] fileContent, String dst, String partitions, String fileName, Integer writeType) {
String rodownDataDst = dst + "/" + fileName + "&&&&" + UUID.randomUUID();
if (StringUtils.isNotEmpty(partitions)) {
rodownDataDst = dst + "/" + partitions + fileName + "&&&&" + UUID.randomUUID();
dst = dst + "/" + partitions;
}
Path dstPath = new Path(dst);
try {
if (Objects.equals(2, writeType)) {
//追加
} else {
//覆盖
if (dfs.exists(dstPath)) {
dfs.delete(dstPath, true);
}
}
dstPath = new Path(rodownDataDst);
if (!dfs.exists(dstPath.getParent())) {
//创建777的目录,防止其他用户访问hive的数据有权限问题
dfs.mkdirs(dstPath.getParent(),new FsPermission(FsAction.ALL,FsAction.ALL,FsAction.ALL));
}
//Init output stream
FSDataOutputStream outputStream = dfs.create(dstPath);
//Cassical output stream usage
outputStream.write(fileContent);
outputStream.close();
} catch (IOException ie) {
logger.error(ie.getMessage());
return false;
}
return true;
}
/**
* @param hdfsUrl 绝对路径
* @return
* @throws IOException
*/
//TODO 改为返回流格式
public BufferedInputStream readFile(String hdfsUrl) {
FSDataInputStream fis = null;
// Configuration conf = getConf();
Path path = new Path(hdfsUrl);
BufferedInputStream bis = null;
try {
FileSystem fs = FileSystem.get(URI.create(hdfsUrl), conf);
if (fs.exists(path)) {
fis = fs.open(path);
bis = new BufferedInputStream(fis);
// fis = fs.open(path);
// byte[] buffer = new byte[fis.available()];
// fis.read(buffer);
// fis.close();
// // 清空response
// response.reset();
// // 设置response的Header
// response.addHeader("Content-Disposition", "attachment;filename=" + primaryFileName);
response.addHeader("Content-Length", "" + file.length());
// OutputStream toClient = new BufferedOutputStream(response.getOutputStream());
// response.setContentType("application/octet-stream");
// toClient.write(buffer);
// toClient.flush();
// toClient.close();
} else {
throw new RRException(DataDevelopmentBizExceptionEnum.MODEL_NOT_FOUND_IN_HDFS.getMessage());
}
} catch (IOException | IllegalArgumentException e) {
logger.error(e.getMessage());
}
return bis;
// finally {
// try{
// outputStream.flush();
// outputStream.close();
// fis.close();
// }catch (IOException e){
// e.printStackTrace ();
// }
// }
}
/**
* 获取路径下所有文件的路径进而获取table名字
*
* @param dir
* @throws IOException
*/
public List<String> dbTableNameList(String dir) throws IOException {
// Configuration conf = getConf();
FileSystem fs = FileSystem.get(conf);
FileStatus[] stats = null;
List<String> tableNameList = new ArrayList<>();
//判断路径是否存在
if (fs.exists(new Path(dir))) {
stats = fs.listStatus(new Path(dir));
for (int i = 0; i < stats.length; i++) {
if (stats[i].isDirectory()) {
tableNameList.add(stats[i].getPath().getName());
} else {
System.out.println(stats[i].getPath().toString());
}
}
fs.close();
}
return tableNameList;
}
public boolean delete(String dst) {
Path dstPath = new Path(dst);
try {
if (dfs.exists(dstPath)) {
dfs.delete(dstPath, true);
}
} catch (IOException ie) {
logger.error(ie.getMessage());
return false;
}
return true;
}
public boolean mkdir(String dst) {
Path dstPath = new Path(dst);
try {
if (!dfs.exists(dstPath)) {
dfs.mkdirs(dstPath, FsPermission.getDefault());
}
} catch (IOException ie) {
logger.error(ie.getMessage());
return false;
}
return true;
}
public boolean rename(String source, String target) {
Path sourcePath = new Path(source);
Path targetPath = new Path(target);
try {
if (dfs.exists(sourcePath)) {
dfs.rename(sourcePath, targetPath);
}
} catch (IOException ie) {
logger.error(ie.getMessage());
return false;
}
return true;
}
public void close() {
try {
dfs.close();
} catch (IOException e) {
logger.error(e.getMessage());
}
}
public String readFile2(String hdfsUrl, HttpServletResponse response) {
FSDataInputStream fis = null;
Path path = new Path(hdfsUrl);
BufferedReader bis = null;
String str = "";
try {
FileSystem fs = FileSystem.get(URI.create(hdfsUrl), conf);
if (fs.exists(path)) {
fis = fs.open(path);
InputStreamReader isr = new InputStreamReader(fis, "utf-8");
bis = new BufferedReader(isr);
String line;
while ((line = bis.readLine()) != null) {
str = str + line;
}
} else {
throw new RRException(DataDevelopmentBizExceptionEnum.MODEL_NOT_FOUND_IN_HDFS.getMessage());
}
} catch (UnsupportedEncodingException ex) {
ex.printStackTrace();
} catch (IOException ex) {
ex.printStackTrace();
} catch (IllegalArgumentException e) {
logger.error(e.getMessage());
}
return str;
}
public Long getTableOrDbSize(String dir) throws IOException {
FileSystem fs = FileSystem.get(conf);
Long tableSize ;
//判断路径是否存在
if (fs.exists(new Path(dir))) {
FileStatus stats = fs.getFileStatus(new Path(dir));
tableSize = fs.getContentSummary(stats.getPath()).getLength();
fs.close();
return tableSize;
}
return null;
}
}
配置属性:
bdp-manager:
namenodestr: bigdata-test-1:8020,bigdata-test-2:8020
namespace: ns
nameport: 8020