在本地打印日志 , 对日志文件进行分析 , 把需要上传的日志文件移动到待上传目录 , 将待上传的日志文件逐个上传到HDFS , 并移动到备份目录 , 定期清理过期备份日志 .
1.首先生产日志文件:
-
/**
-
* 生产日志
-
* @author Administrator
-
*
-
*/
-
public
class MoreLog {
-
public static void main(String[] args) throws Exception {
-
Logger logger = LogManager.getLogger(MoreLog.class);
-
System.out.println(
"开始打印日志了");
-
while(
true){
-
logger.info(
"张俊杰zhangjunjie张俊杰zhangjunjie张俊杰zhangjunjie");
-
Thread.sleep(
1);
-
}
-
}
-
}
2.启动日志采集和日志清理
-
/**
-
* 启动日志采集
-
* 和日志清理
-
* @author Administrator
-
*
-
*/
-
-
public
class StartUp {
-
public static void main(String[] args) {
-
Timer timer =
new Timer();
-
//上传
-
timer.schedule(
new ClollactionTask(),
0,
2*
60*
1000l);
-
-
//清理文件
-
timer.schedule(
new ClearTask(),
0,
24*
60*
1000);
-
}
-
}
3.日志采集并上传HDFS
-
import java.io.File;
-
import java.io.FilenameFilter;
-
import java.net.InetAddress;
-
import java.net.URI;
-
import java.text.SimpleDateFormat;
-
import java.util.Date;
-
import java.util.TimerTask;
-
-
import org.apache.commons.io.FileUtils;
-
import org.apache.hadoop.conf.Configuration;
-
import org.apache.hadoop.fs.FileSystem;
-
import org.apache.hadoop.fs.Path;
-
-
/**
-
* 日志采集的:
-
* 1.从日志目录里面查看哪些需要上传(1.2.3.....)
-
* 2.把需要上传的文件移动到待上传文件
-
* 3.上传到hdfs上(/log/2018-8-20/xxx.log)
-
* 4.将上传完成的日志文件移动到备份目录
-
* @author Administrator
-
*
-
*/
-
-
public
class ClollactionTask extends TimerTask{
-
-
@Override
-
public void run() {
-
System.out.println(
"开始执行任务");
-
try {
-
SimpleDateFormat format =
new SimpleDateFormat(
"yyyy-MM-dd-HH-mm");
-
String datetime = format.format(
new Date());
-
-
// 1:查看哪些文件是需要上传的
-
File logDir =
new File(
"e:/testlog/");
-
File[] listFiles = logDir.listFiles(
new FilenameFilter() {
-
// FileNameFilter 是过哪些文件能够获取的
-
@Override
-
public boolean accept(File dir, String name) {
-
return name.startsWith(
"test.log.");
-
}
-
});
-
-
// 2:将文件移动到待上传目录
-
for (File file : listFiles) {
-
// file.renameTo(new File(""));
-
FileUtils.moveFileToDirectory(file,
new File(
"e:/waitUpLoad"),
true);
-
}
-
-
//3:将待上传的文件逐个上传到hdfs上，并移动到备份目录
-
FileSystem fs = FileSystem.get(
new URI(
"hdfs://hadoop01:9000"),
new Configuration(),
"root");
-
Path detPath =
new Path(
"/log/"+datetime.substring(
0,
10));
-
//判断待上传的目录是否存在，不存在则创建一个（hdfs上的文件夹）
-
boolean exists = fs.exists(detPath);
-
if(!exists){
-
fs.mkdirs(detPath);
-
}
-
-
//判断备份目录是否存在
-
File backDir =
new File(
"e:/backDir/"+datetime);
-
boolean exists2 = backDir.exists();
-
if(!exists2){
-
backDir.mkdirs();
-
}
-
-
//得到上传的是哪一个服务上的日志文件
-
String hostName = InetAddress.getLocalHost().getHostName();
-
-
//4:遍历待上传的目录
-
File file =
new File(
"e:/waitUpLoad");
-
File[] listFile = file.listFiles();
-
for (File f : listFile) {
-
//上传到hdfs上
-
fs.copyFromLocalFile(
new Path(f.getPath()),
new Path(detPath,hostName+
"_"+f.getName()+
"_"+System.currentTimeMillis()));
-
-
//cp到备份目录
-
FileUtils.moveFileToDirectory(f, backDir,
true);
-
}
-
fs.close();
-
-
}
catch (Exception e) {
-
e.getStackTrace();
-
}
-
-
}
-
-
}
4.清理过期日志
-
import java.io.File;
-
import java.text.SimpleDateFormat;
-
import java.util.Date;
-
import java.util.TimerTask;
-
-
import org.apache.commons.io.FileUtils;
-
/**
-
* 清理过期日志
-
* @author Administrator
-
*
-
*/
-
public
class ClearTask extends TimerTask {
-
-
@Override
-
public void run() {
-
try {
-
SimpleDateFormat format =
new SimpleDateFormat(
"yyyy-MM-dd-HH-mm");
-
Date nowdate =
new Date();
-
-
File file =
new File(
"E:/backDir");
-
File[] listFiles = file.listFiles();
-
-
for (File dir : listFiles) {
-
String name = dir.getName();
-
Date dirdate = format.parse(name);
-
if(nowdate.getTime() - dirdate.getTime() >
24*
60*
60*
1000) {
-
FileUtils.deleteDirectory(dir);
-
}
-
}
-
}
catch (Exception e) {
-
e.printStackTrace();
-
}
-
}
-
}
附:log4j配置文件
-
log4j.rootLogger =ALL,systemOut,logRollingFile
-
#log4j.rootLogger =ALL,systemOut,logFile,logDailyFile,logRollingFile,logMail,logDB
-
#\u8F93\u51FA\u5230\u63A7\u5236\u53F0
-
log4j.appender.systemOut = org.apache.log4j.ConsoleAppender
-
log4j.appender.systemOut.layout = org.apache.log4j.PatternLayout
-
log4j.appender.systemOut.layout.ConversionPattern = [%-
5p][%-
22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
-
log4j.appender.systemOut.Threshold = DEBUG
-
log4j.appender.systemOut.ImmediateFlush = TRUE
-
log4j.appender.systemOut.Target = System.out
-
-
#\u8F93\u51FA\u5230\u6587\u4EF6
-
log4j.appender.logFile = org.apache.log4j.FileAppender
-
log4j.appender.logFile.layout = org.apache.log4j.PatternLayout
-
log4j.appender.logFile.layout.ConversionPattern = [%-
5p][%-
22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
-
log4j.appender.logFile.Threshold = WARN
-
log4j.appender.logFile.ImmediateFlush = TRUE
-
log4j.appender.logFile.Append = TRUE
-
log4j.appender.logFile.File = ../Struts2/WebRoot/
log/File/log4j_Struts.log
-
log4j.appender.logFile.Encoding = UTF-
8
-
-
#\u6309DatePattern\u8F93\u51FA\u5230\u6587\u4EF6
-
log4j.appender.logDailyFile = org.apache.log4j.DailyRollingFileAppender
-
log4j.appender.logDailyFile.layout = org.apache.log4j.PatternLayout
-
log4j.appender.logDailyFile.layout.ConversionPattern = [%-
5p][%-
22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
-
log4j.appender.logDailyFile.Threshold = DEBUG
-
log4j.appender.logDailyFile.ImmediateFlush = TRUE
-
log4j.appender.logDailyFile.Append = TRUE
-
log4j.appender.logDailyFile.File = ../Struts2/WebRoot/
log/DailyFile/log4j_Struts
-
log4j.appender.logDailyFile.DatePattern =
'.'yyyy-MM-dd-HH-mm
'.log'
-
log4j.appender.logDailyFile.Encoding = UTF-
8
-
-
#\u8BBE\u5B9A\u6587\u4EF6\u5927\u5C0F\u8F93\u51FA\u5230\u6587\u4EF6
-
log4j.appender.logRollingFile = org.apache.log4j.RollingFileAppender
-
log4j.appender.logRollingFile.layout = org.apache.log4j.PatternLayout
-
log4j.appender.logRollingFile.layout.ConversionPattern = [%-
5p][%-
22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
-
log4j.appender.logRollingFile.Threshold = DEBUG
-
log4j.appender.logRollingFile.ImmediateFlush = TRUE
-
log4j.appender.logRollingFile.Append = TRUE
-
log4j.appender.logRollingFile.File = e:
/testlog/test.log
-
log4j.appender.logRollingFile.MaxFileSize =
1MB
-
log4j.appender.logRollingFile.MaxBackupIndex =
50
-
log4j.appender.logRollingFile.Encoding = UTF-
8
-
-
#\u7528Email\u53D1\u9001\u65E5\u5FD7
-
log4j.appender.logMail = org.apache.log4j.net.SMTPAppender
-
log4j.appender.logMail.layout = org.apache.log4j.HTMLLayout
-
log4j.appender.logMail.layout.LocationInfo = TRUE
-
log4j.appender.logMail.layout.Title = Struts2 Mail LogFile
-
log4j.appender.logMail.Threshold = DEBUG
-
log4j.appender.logMail.SMTPDebug = FALSE
-
log4j.appender.logMail.SMTPHost = SMTP.
163.com
-
log4j.appender.logMail.From = xly300
0@163.com
-
log4j.appender.logMail.To = xly300
0@gmail.com
-
#log4j.appender.logMail.Cc = xly3000@gmail.com
-
#log4j.appender.logMail.Bcc = xly3000@gmail.com
-
log4j.appender.logMail.SMTPUsername = xly300
0
-
log4j.appender.logMail.SMTPPassword =
1234567
-
log4j.appender.logMail.Subject = Log4j Log Messages
-
#log4j.appender.logMail.BufferSize = 1024
-
#log4j.appender.logMail.SMTPAuth = TRUE
-
-
#\u5C06\u65E5\u5FD7\u767B\u5F55\u5230MySQL\u6570\u636E\u5E93
-
log4j.appender.logDB = org.apache.log4j.jdbc.JDBCAppender
-
log4j.appender.logDB.layout = org.apache.log4j.PatternLayout
-
log4j.appender.logDB.Driver = com.mysql.jdbc.Driver
-
log4j.appender.logDB.URL = jdbc:mysql:
//
127.0.
0.
1:
3306/xly
-
log4j.appender.logDB.User = root
-
log4j.appender.logDB.Password =
123456
-
log4j.appender.logDB.Sql = INSERT INTOT_log4j(project_name,create_date,level,category,file_name,thread_name,line,all_category,message)
values(
'Struts2',
'%d{yyyy-MM-ddHH:mm:ss}',
'%p',
'%c',
'%F',
'%t',
'%L',
'%l',
'%m')
1473
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
