日志收集步骤:
1:从日志目录里面查看哪需要上传的(.1 .2 …..)
2:把需要上传的文件移动到待上传目录
3:上传到hdfs上(/log/2018-8-20/xxx.log)
4:移动到备份目录
第一步,模拟系统产生日志文件,在这里用一个永真的循环打印。
public static void main(String[] args) throws Exception {
Logger logger = LogManager.getLogger(MoreLog.class);
System.out.println("开始打印日志了----");
while(true){
logger.info("-------------------------------------------------");
Thread.sleep(1);
}
}
第二步,使用定时器进行调度
public static void main(String[] args) {
Timer timer = new Timer();
timer.schedule(new CollectionTask(), 0, 2*60*1000);
}
第三步,重写TimerTask的run方法,重点
package cn.pengpeng.day03.log;
import java.io.File;
import java.io.FilenameFilter;
import java.net.InetAddress;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimerTask;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class CollectionTask extends TimerTask {
@Override
public void run() {
try {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm");
String datename = format.format(new Date());
// TODO Auto-generated method stub
File file = new File("d:/testlog/");
File[] listFiles = file.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.startsWith("test.log.");
}
});
// 将文件移动到待上传目录
for (File fi : listFiles) {
FileUtils.moveFileToDirectory(fi, new File("d:/waitUpload"), true);
}
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop-master:9000"), new Configuration(), "root");
Path dst = new Path("/log/"+datename.substring(0, 10));
boolean exists = fs.exists(dst);
if(!exists) {
fs.mkdirs(dst);
}
File backDir = new File("d:/backDir/"+datename);
boolean exists2 = backDir.exists();
if(!exists2) {
backDir.mkdirs();
}
String hostName = InetAddress.getLocalHost().getHostName();
File file2 = new File("d:/waitUpload");
File[] list = file2.listFiles();
for (File f : list) {
fs.copyFromLocalFile(new Path(f.getPath()), new Path(dst,hostName+"-"+System.currentTimeMillis()));
FileUtils.moveFileToDirectory(f, backDir, true);
}
fs.close();
} catch (Exception e) {
// TODO: handle exception
e.getStackTrace();
}
}
}