Java实现获取HDFS子目录数量_Java实现读取HDFS目录

LOG.info("检查是否有当前日期的目录");

fsUtil.checkDirExists(new File(configUtil.readConfig("download.file.path") + currentDay));

FSDataInputStream in = null;

LOG.info("生成以JobID命名的目录");

String fileName = configUtil.readConfig("download.file.path") + currentDay + "/" + jobID + ".csv";

File file = new File(fileName);

List> dataList = new ArrayList<>();

InputStreamReader inputStreamReader = null;

try {

String dataPath = configUtil.readConfig("offLine.data.path") + currentDay + "/" + jobID + ".csv/";

LOG.info(dataPath);

Path path = new Path(dataPath);

Configuration configuration = dfsUtil.getHadoopConf();

FileSystem hdfs = FileSystem.get(configuration);

BufferedReader br = null;

if (hdfs.exists(path) && hdfs.isDirectory(path)) {

FileStatus[] srcFileStatus = hdfs.listStatus(path);

for (FileStatus fs : srcFileStatus) {

if (fs.getLen() > 0) {

String line;

in = hdfs.open(fs.getPath());

inputStreamReader = new InputStreamReader(in, "UTF-8");

br = new BufferedReader(inputStreamReader);

while ((line = br.readLine()) != null) {

dataList.add(Lists.newArrayList(new String(line.getBytes(Constant.CODE_FORMAT))));

}

}

}

}

} catch (Exception error) {

error.getMessage();

error.printStackTrace();

} finally {

org.apache.hadoop.io.IOUtils.closeStream(in);

}