定时脚本: 删除HDFS中的过期文件

1. 基本原理：

　　通过hadoop fs -ls *命令获取相关文件或目录的修改时间，然后与设定的过期时间进行比较，之后执行删除操作即可

2. 相关代码：

#!/bin/bash
source ~/.bashrc

# HADOOP所在的bin目录
HADOOP_BIN_PATH=/home/hadoop/hadoop-2.3.0-cdh5.0.0/bin

#待检测的HDFS目录
data1_file=/user/gas/data1
data2_file=/user/gas/data2

#将待检测的目录(可以为多个)加载至数组中
array_check=($data1_file $data2_file)

# 当前时间戳
today_timestamp=$(date -d "$(date +"%Y-%m-%d %H:%M")" +%s)

#Func: 删除指定时间之前的过期，这里设置的是45天前
removeOutDate(){
        $HADOOP_BIN_PATH/hadoop fs -ls $1 > temp.txt
        cat temp.txt | while read quanxian temp user group size day hour filepath
        do
            current_file_time="$day $hour"
            current_file_timestamp=$(date -d "$current_file_time" +%s)
            if [ $(($today_timestamp-$current_file_timestamp)) -ge $((45*24*60*60)) ];then
                echo "$(date +'%Y-%m-%d %H:%M:%S') $filepath"
                $HADOOP_BIN_PATH/hadoop fs -rm -r $filepath > /dev/null 2>&1 
            fi
        done
}

#Func: 执行删除
execute(){
        echo -e "\n\n"
        echo "$(date +'%Y-%m-%d %H:%M:%S') start to remove outdate files in hdfs"
        echo "$(date +'%Y-%m-%d %H:%M:%S') today is: $(date +"%Y-%m-%d %H:%M:%S")"

        for i in ${array_check[@]}
        do
            echo "$(date +'%Y-%m-%d %H:%M:%S') processing filepath: $i"
            removeOutDate $i
            echo -e "\n"
        done

        echo "$(date +'%Y-%m-%d %H:%M:%S') remove outdate files in hdfs finished"
        echo -e "\n\n"
}

# 开始执行
execute

 参考：Hadoop运维-删除目录下过期的数据

转载于:https://www.cnblogs.com/mengrennwpu/p/6114505.html