将系统产生的日志收集到HDFS上

import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.util.Date;

public class LogGenerator {
    public static void main(String[] args) throws InterruptedException {
        Logger logger = LogManager.getLogger("LogGenerator");
        int i = 0;
        while(true){
            logger.info("----" + new Date().toString() + "----");
            i++;
            Thread.sleep(500);
            if (i > 1000000){
                break;
            }
        }
    }
}

maven打包java项目
进入项目根目录ab
mvn clean package -DskipTests


在生产环境上执行java程序


java -Djava.ext.dirs=../software com.lhjava.LogGenerator


编写一个shell:upload.sh
修改upload.sh的执行权限
chmod u+x upload.sh
=================================================
#!/bin/bash


source_dir=/home/fengqing/log/
target_dir=/data/logs/20180402/


ls $source_dir | while read fileName


do
        if [[ "$fileName" == mylog.log.* ]]; then
        echo $source_dir$fileName $target_dir`hostname`_$fileName
        hadoop fs -put $source_dir$fileName $target_dir`hostname`_$fileName
rm -f $source_dir$fileName
        fi
done
=================================================


用crontab进行调度
http://tool.lu/crontab


crontab -e
* * * * * /home/fengqing/shell/upload.sh




安装crontab
sudo yum install vixie-cron crontabs