需求:将log4j产生的日志文件上传到hdfs集群上,清理超过24小时的文件,保留24小时以内的文件
1.产生日志
public class MoreLog {
public static void main(String[] args) throws Exception {
Logger logger = LogManager.getLogger(MoreLog.class);
System.out.println("开始打印日志了----");
while(true){
logger.info("----------------------------------------------------");
Thread.sleep(1);
}
}
}
2.收集产生的日志文件并上传到hdfs集群上
public class ClollactionTask extends TimerTask {
@Override
public void run() {
try {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm");
String datetime = format.format(new Date());
File logDir = new File("d:/testlog/");
File[] listFiles = logDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.startsWith("test.log.");
}
});
for (File file : listFiles) {
FileUtils.moveFileToDirectory(file, new File("d:/waitUpLoad"), true);
}
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop01:9000"), new Configuration(), "root");
Path detPath = new Path("/log/"+datetime.substring(0, 10));
boolean exists = fs.exists(detPath);
if(!exists){
fs.mkdirs(detPath);
}
File backDir = new File("d:/backDir/"+datetime);
boolean exists2 = backDir.exists();
if(!exists2){
backDir.mkdirs();
}
String hostName = InetAddress.getLocalHost().getHostName();
File file = new File("d:/waitUpLoad");
File[] listFile = file.listFiles();
for (File f : listFile) {
fs.copyFromLocalFile(new Path(f.getPath()), new Path(detPath,hostName+"_"+f.getName()+"_"+System.currentTimeMillis()));
FileUtils.moveFileToDirectory(f, backDir, true);
}
fs.close();
} catch (Exception e) {
e.getStackTrace();
}
}
}
3.清理超过24小时的文件
public class CleanTask extends TimerTask {
@Override
public void run() {
try {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm");
Date nowdate = new Date();
File file = new File("D:/backDir");
File[] listFiles = file.listFiles();
for (File dir : listFiles) {
String name = dir.getName();
Date dirdate = format.parse(name);
if(nowdate.getTime() - dirdate.getTime() > 24*60*60*1000) {
FileUtils.deleteDirectory(dir);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
4.启动日志采集、上传以及清理过期文件
public class StartUp {
public static void main(String[] args) {
Timer timer = new Timer();
timer.schedule(new ClollactionTask(), 0, 2*60*1000);
timer.schedule(new CleanTask(),0, 2*60*60*1000);
}
}