整理并通过md5删除重复文件

目录下有多种格式的文件，需要分类并去除掉重复文件（文件名可能不同，但内容相同）

文件分类

#!/bin/bash
if [ "$#" -ne 1 ]; then
    echo "use: $0 <download_url>"
    exit 1
fi
TARGET_DIR="$1" 
mkdir -p "$TARGET_DIR/jpg"
mkdir -p "$TARGET_DIR/mp4"
for img in "$TARGET_DIR"/*.jpg; do
    if [ -e "$img" ]; then
        mv "$img" "$TARGET_DIR/jpg/"
    fi
done
for video in "$TARGET_DIR"/*.mp4; do
    if [ -e "$video" ]; then
        mv "$video" "$TARGET_DIR/mp4/"
    fi
done

删除重复文件：由于文件名仅仅是文件的标签，相同的文件不一定是重复文件，根据md5哈希值判断文件是否重复比较稳妥
declare -A file_hashes 定义数组名为file_hashes

cd "$TARGET_DIR/jpg" || exit
for file in *; do
    if [ -e "$file" ]; then
        md5=$(md5sum "$file" | cut -d ' ' -f 1)
        if [ $(ls | grep -e "^$md5" | wc -l) -gt 1 ]; then
            echo "删除重复文件: $file"
            rm "$file"
        fi
    fi
done

将文件分类和去除重复功能封装成函数

#!/bin/bash
if [ "$#" -ne 1 ]; then
    echo "use: $0 <download_url>"
    exit 1
fi
TARGET_DIR="$1" 
mkdir -p "$TARGET_DIR/jpg"
mkdir -p "$TARGET_DIR/mp4"
move_files() {
    local ext="$1"           
    local dest_dir="$2"    
    for file in "$TARGET_DIR"/*."$ext"; do
        if [ -e "$file" ]; then
            mv "$file" "$dest_dir/" && echo "移动文件: $file 到 $dest_dir/"
        fi
    done
}
remove_duplicates() {
    local dir="$1"
    cd "$dir" || exit
    declare -A file_hashes
    for file in *; do
        if [ -e "$file" ]; then
            md5=$(md5sum "$file" | cut -d ' ' -f 1)
            if [[ -n "${file_hashes[$md5]}" ]]; then
                echo "删除重复文件: $file"
                rm "$file"
            else
                file_hashes[$md5]="$file"
            fi
        fi
    done
}
move_files "jpg" "$TARGET_DIR/jpg"
move_files "mp4" "$TARGET_DIR/mp4"
remove_duplicates "$TARGET_DIR/jpg"
remove_duplicates "$TARGET_DIR/mp4"

cd命令存在问题，考虑在移动文件前检查重复文件，再移动，简化代码

#!/bin/bash
if [ "$#" -ne 1 ]; then
    echo "$0:参数个数不对"
    exit 1
fi
TARGET_DIR="$1"
mkdir -p "$TARGET_DIR/jpg" "$TARGET_DIR/mp4"
process_files() {
    local ext="$1"
    local dest_dir="$TARGET_DIR/$ext"
    declare -A file_hashes
    # 遍历目录下指定后缀名文件
    for file in "$TARGET_DIR"/*."$ext"; do
        if [ -e "$file" ]; then  # 检查目录是否存在
            md5=$(md5sum "$file" | cut -d ' ' -f 1)
            if [[ -n "${file_hashes[$md5]}" ]]; then
                echo "删除重复文件: $file"
                rm "$file"
            else
                file_hashes[$md5]="$file"
                mv "$file" "$dest_dir/" && echo "移动文件: $file 到 $dest_dir/"
            fi
        fi
    done
}
process_files "jpg" 
process_files "mp4"
echo "操作完成！"