为你呈上热腾腾的shell多线程并发处理任务脚本

使用shell多线程并发完成任务

batchTask.sh

#!/usr/bin/bash

####################
## Autor: Troll
## Date:  2020-06-15
## Desc:  本地文件批量put性能测试(多线程)

####################

# 并发进程数
thread_num=30

startTime=$(date +%H:%M:%S)
echo "${startTime} task start..."

# mkfifo
tempfifo="upload_hdfs"
mkfifo ${tempfifo}
# 使文件描述符为非阻塞式
exec 6<>${tempfifo}
rm -f ${tempfifo}
# 为文件描述符创建占位信息
for ((i=1;i<=${thread_num};i++))
do
{
    echo 
}
done >&6 


# 数据目录
localDirPath='../data'
# hdfs目录
hdfsDirPath='/tmp/perfomance/data/'

# 删除hdfs目录
$(hdfs dfs -rmr ${hdfsDirPath})

# 创建hdfs目录
$(hdfs dfs -mkdir -p ${hdfsDirPath})
# echo "hdfs dfs -mkdir -p ${hdfsDirPath}"

# 目录赋权
$(hdfs dfs -chmod -R 777 ${hdfsDirPath})
# echo "hdfs dfs -chmod -R 777 ${hdfsDirPath}"

# 文件列表
dataFiles=$(ls "${localDirPath}/pending/")
# echo "dataFiles:${dataFiles}"

# echo -e "$dataFiles"
for line in ${dataFiles}
do
	read -u6
    {

	  
	  # 上传hdfs集群
	  $(hdfs dfs -put ${localDirPath}/pending/${line} ${hdfsDirPath})
	  # echo "hdfs dfs -put ${line} ${hdfsDirPath}"

	  # 移动目录
	  $(mv ${localDirPath}/pending/${line}  ${localDirPath}/finish/ )
	  # echo "mv ${line}  ${localDirPath}/finish/"
      
      # 打印
	  echo "${line} 处理成功！" >&6

	  # 等待1s
	  sleep 1

    } & 



done

# 等待子程序运行完成
wait

# 数据复位
$(mv ${localDirPath}/finish/* ${localDirPath}/pending/ )

# 关闭fd6管道
exec 6>&-


endTime=$(date +%H:%M:%S)
echo "${endTime} task run finish!"