CDH集群日志定时清理脚本

脚本功能：

实现对CDH版本中 hadoop-yarn、 hadoop-hdfs、hive、oozie日志的定时清理。

#!/bin/bash


## 01-Handle logic of clean YARN RESOURCEMANAGER & NODEMANAGER logs.
########################################################
su - yarn -c '
        YARN_PATH=/var/log/hadoop-yarn;
        ls $YARN_PATH | grep "hadoop-cmf-yarn-RESOURCEMANAGER-nn1.hadoop.com.log.out.[0-9]\{1,\}" | while read f;  do `cat /dev/null > $YARN_PATH/$f`; done
'
su - yarn -c '
        YARN_PATH=/var/log/hadoop-yarn;
        ls $YARN_PATH | grep "hadoop-cmf-yarn-NODEMANAGER-nn1.hadoop.com.log.out.[0-9]\{1,\}" | while read f;      do `cat /dev/null > $YARN_PATH/$f`; done
'

## 02-Handle logic of clean HDFS NAMENODE & DATANODE & audit logs.
########################################################
su - hdfs -c '
        HDFS_PATH=/var/log/hadoop-hdfs;
        ls $HDFS_PATH | grep "hadoop-cmf-hdfs-NAMENODE-nn1.hadoop.com.log.out.[0-9]\{1,\}$" | while read f;     do `cat /dev/null > $HDFS_PATH/$f`; done   ;
        ls $HDFS_PATH | grep "hadoop-cmf-hdfs-DATANODE-nn1.hadoop.com.log.out.[0-9]\{1,\}$" | while read f;     do `cat /dev/null > $HDFS_PATH/$f`; done   ;
        ls $HDFS_PATH | grep "hdfs-audit.log.[0-9]\{1,\}$" | while read f;                                      do `cat /dev/null > $HDFS_PATH/$f`; done
'

## 03-Handle logic of clean hive HIVEMETASTORE logs.
########################################################
su - hive -c '
        HIVE_PATH=/var/log/hive;
        ls $HIVE_PATH | grep "hadoop-cmf-hive-HIVEMETASTORE-nn1.hadoop.com.log.out.[0-9]\{1,\}$" | while read f;        do `cat /dev/null > $HIVE_PATH/$f`; done
        ls $HIVE_PATH | grep "hadoop-cmf-hive-HIVESERVER2-nn1.hadoop.com.log.out.[0-9]\{1,\}$" | while read f;        do `cat /dev/null > $HIVE_PATH/$f`; done
'

## 04-Handle logic of clean hive Oozie logs.
########################################################
su - root -c '
        HIVE_PATH=/var/log/oozie;
        ls $HIVE_PATH | grep "oozie-instrumentation.log.[0-9]\{1,\}$" | while read f;   do `cat /dev/null > $HIVE_PATH/$f`; done
find $HIVE_PATH -maxdepth 1 -mtime 1 -type f -name "oozie-cmf-oozie-OOZIE_SERVER-nn1.hadoop.com.log.out*"| xargs rm -rf
'

配置crontab调度：

*/10 * * * *  sh /root/bigdata_devops/001-clearup_hadoop_tmp_log_script.sh >> /root/bigdata_devops/001-clearup_hadoop_tmp_log_script.sh.out