#!bin/sh
#获得目录下面的子目录名称,并保存在数组中
LOCAL_PATH=$1 #/home/hadoop/test/
cd $LOCAL_PATH
function test(){
#创建目录的list
channelArr=("")
i=0
for line in `ls $LOCAL_PATH`
#此处也适合hdfs目录 hdfs dfs -ls $HDFS_PATH
do
#echo $line
channelArr[i]=${line}
echo "$channelArr[i]"
((i=i+1))
done
echo "数组个数:" ${channelArr[@]}
echo "start ..."
#遍历数组
for element in ${channelArr[*]}
do
echo ${element}
done
echo "end ..."
}
test
2、循环获得HDFS的文件,并对文件后缀进行更新操作。
背景:
主要原因是建立的hive外表无法读取.tmp或.pending的文件,所以需要去掉后缀。
1)准备一个二级分区(天、小时)目录,文件放在对应的hour目录下
ls /user/hive/warehouse/tmp_predict/dt=2017-12-27/
/user/hive/warehouse/tmp_predict/dt=2017-12-27/hour=00
…
/user/hive/warehouse/tmp_predict/dt=2017-12-27/hour=23
ls /user/hive/warehouse/tmp_predict/dt=2017-12-27/hour=00
/user/hive/warehouse/tmp_predict/dt=2017-12-27/hour=00/0000000
…
/user/hive/warehouse/tmp_predict/dt=2017-12-27/hour=00/0000000.tmp
2)准备脚本
#!/bin/bash
#print the directory and file
hadoop fs -ls /user/hive/warehouse/tmp_predict/dt=2017-12-27/ | awk '{print $8}' > files0.txt
#循环遍历,提取所需数据
cat ./files0.txt | while read line
do
echo "$line"
hadoop fs -ls $line | awk '{print $8}' > files1.txt
cat ./files1.txt | while read fname
do
echo "$fname"
_path=`echo $fname | awk -F'.' '{print $1}'`
hadoop fs -mv $fname $_path
#echo $_path
done