1.添加Maven依赖
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.8.3</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
2.while循环生成日志文件
Logger logger = LogManager.getLogger(CreateLog.class);
while(true) {
logger.info("这是info");
Thread.sleep(2);
}
3.使用Timer构建一个定时上传,备份的任务
//Task类需继承TimeTask方法,30ms后运行,每2s运行一次
new Timer().schedule(new Task(), 30,2000);
4.编写任务类
public class Task extends TimerTask {
@Override
public void run() {
try {
//时间以拼接文件名防止重名,构造目录分类日志文件
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH");
String date = format.format(new Date());
File logdir = new File("d:/testlog");
File[] listFiles = logdir.listFiles(new FilenameFilter() {
// FileNameFilter是过滤哪些文件能够获取的
@Override
public boolean accept(File dir, String name) {
// test.log.1 等等,日志满1M生成一份日志文件
return name.startsWith("test.log.");
}
});
for (File file : listFiles) {
// file.renameTo(new File(""));
//apache.common的FileUtils类
FileUtils.moveFileToDirectory(file, new File("d:/waitUpLoad"), true);
}
//连接Hdfs服务器
FileSystem fs=FileSystem.get(new URI("hdfs://jiqun01:9000"), new Configuration(), "root");
Path path = new Path("/log/"+date.substring(0, 10));
boolean exists = fs.exists(path);
if(!exists) {
fs.mkdirs(path);
}
//得到上传的是哪一个服务器上的日志文件
String hostName = InetAddress.getLocalHost().getHostName();
File fileWaitUpload=new File("d:/waitUpLoad");
File[] list = fileWaitUpload.listFiles();
for (File f : list) {
//上传文件到服务器,路径为/date,名称为主机名+文件名+毫秒数
fs.copyFromLocalFile(new Path(f.getPath()), new Path(path,hostName+"_"+f.getName()+"_"+System.currentTimeMillis()));
//cp到备份目录
FileUtils.moveFileToDirectory(f, new File("d:/backDir/"+date.substring(0, 10)), true);
}
fs.close();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
5.构建删除时间超过一天的备份日志的任务
public class DeleteBackLog {
public static void main(String[] args) {
Timer timer = new Timer();
timer.schedule(new TimerTask() {
//匿名内部类
@Override
public void run() {
File file = new File("d:/backDir");
File[] listFiles = file.listFiles();
for (File file2 : listFiles) {
if (file2.isDirectory()) {
File[] listFiles2 = file2.listFiles();
for (File file3 : listFiles2) {
//得到最后修改时间即创建时间
long lastModified = file3.lastModified();
long currentTimeMillis = System.currentTimeMillis();
//得到天数,1000ms=1s,1day=86400000ms
long minutes = (currentTimeMillis - lastModified) / 86400000;
if((int) minutes>1) {
boolean delete = file3.delete();
System.out.println(file3.getName()+"存在时间:"+(int) minutes+"分钟"+((delete==true) ?"已经":"没有")+"删除");
}
}
} else {
long lastModified = file2.lastModified();
long currentTimeMillis = System.currentTimeMillis();
long minutes = (currentTimeMillis - lastModified) / 60000;
if((int) minutes>1) {
boolean delete = file2.delete();
System.out.println(file2.getName()+"存在时间:"+(int) minutes+""+(delete==true) != null?"已经":"没有"+"删除");
}
}
}
}
}, 0, 3600000);
}
7.补充
Hosts文件,此处Hadoop集群构建在虚拟机上
#127.0.0.1 localhost
192.168.59.138 jiqun01
192.168.59.139 jiqun02
192.168.59.140 jiqun03
log4j配置文件
log4j.appender.logRollingFile.Append = TRUE
log4j.appender.logRollingFile.File = d:/testlog/test.log
log4j.appender.logRollingFile.MaxFileSize = 1MB
log4j.appender.logRollingFile.MaxBackupIndex = 50
log4j.appender.logRollingFile.Encoding = UTF-8
8.运行效果截图
启动hadoop
备份文件
hdfs查看上传的日志
定期删除文件,此处将时常改为一分钟