1.读取单个文件
[java] view plain copy
- Date date = DateUtil.getSpecifiedDayBefore();
- String yesterday = DateUtil.dateToStr(date, "yyyy-MM-dd");
- String path = "hdfs://ip:9000/output_log/output_log_click" + yesterday;
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(URI.create(path), conf);
- FSDataInputStream hdfsInStream = fs.open(new Path(path));
- InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8");
- BufferedReader br = new BufferedReader(isr);
- String line;
- // int k = 0;
- while ((line = br.readLine()) != null) {
- System.out.println(line);
- }
2.读取文件夹
[java] view plain copy
- Date date = DateUtil.getSpecifiedDayBefore();
- String yesterday = DateUtil.dateToStr(date, "yyyy-MM-dd");
- String path = "hdfs://ip:9000/output_log/output_log_click" + yesterday;
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(URI.create(path), conf);
- FileStatus[] status = fs.listStatus(new Path(path));
- for (FileStatus file : status) {
- if (!file.getPath().getName().startsWith("newsMap")) {
- continue;
- }
- FSDataInputStream hdfsInStream = fs.open(file.getPath());
- InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8");
- BufferedReader br = new BufferedReader(isr);
- String line;
- // int k = 0;
- while ((line = br.readLine()) != null) {
- System.out.println(line);
- }
- }