用java读取hdfs的文件
直接撸代码:
package com.nature.base.util;
import com.nature.component.process.vo.DebugDataResponse;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import java.io.*;
import java.net.URI;
import java.util.*;
/**
* 操作hdfs文件的工具类
*/
public class HdfsUtils {
private static Logger logger = LoggerUtil.getLogger();
/**
* 读取hdfs指定路径的内容
*/
public static String gethdfsData(String hdfsPath){
String result = "";
if(com.nature.third.utils.StringUtils.isNotEmpty(hdfsPath)){
Path path = new Path(hdfsPath);
Configuration configuration = new Configuration();
FSDataInputStream fsDataInputStream = null;
FileSystem fileSystem = null;
BufferedReader br = null;
// 定义一个字符串用来存储文件内容
try {
fileSystem = path.getFileSystem(configuration);
fsDataInputStream = fileSystem.open(path);
br = new BufferedReader(new InputStreamReader(fsDataInputStream));
String str2;
while ((str2 = br.readLine()) != null) {
// 遍历抓取到的每一行并将其存储到result里面
result += str2 + "\n";
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if(br!=null){
try {
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(fsDataInputStream!=null){
try {
fsDataInputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(fileSystem!=null){
try {
fileSystem.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
logger.debug("文件内容:" + result);
}
return result;
}
}
核心方法:
Path path = new Path("hdfs://192.168.1.111:9000/test/1234log");
Configuration configuration = new Configuration();
FileSystem fileSystem = path.getFileSystem(configuration);
FSDataInputStream fsDataInputStream = fileSystem.open(path);
注意我加的那些编码格式,可以防止中文乱码!!
这种方式即可以读取hdfs上面的文件也可以读取本地文件