linux怎么查hdfs日志目录,Linux记录-shell获取hdfs used使用

#!/bin/bash

export JAVA_HOME=xxx

export HADOOP_HOME=xxx

export HADOOP_CONF_DIR=xxx

cd /home/hdfs/xxx

mv host.txt host.txt_`date +"%Y-%m-%d"`

mv host.txt_`date +"%Y-%m-%d"` host/

mv hostdfs.txt hostdfs.txt_`date +"%Y-%m-%d"`

mv hostdfs.txt_`date +"%Y-%m-%d"` host/

rm -f ip.txt

rm -f allhostname.txt

rm -f hostname.txt

rm -f iphostname.txt

rm -f dfused.txt

rm -f minhost.txt

rm -f maxhost.txt

rm -f host.txt

rm -f nohup.out

rm -f dfsreport.txt

echo "start checking hdfs used rate"

$HADOOP_HOME/bin/hdfs dfsadmin -report > dfsreport.txt

cat dfsreport.txt | grep -i "50010" | awk -F ":" '{print $2}'| awk '{print $1}' > ip.txt

cat dfsreport.txt | grep -i "hostname" | awk -F ": " '{print $2}' > allhostname.txt

cat dfsreport.txt | grep "DFS Used%" | awk -F ": " '{print $2}' | awk -F "%" '{print $1}' > dfused.txt

livesum=$(cat dfsreport.txt | grep "Live datanodes" | awk '{print $3}' | awk -F "):" '{print $1}' | awk -F "(" '{print $2}')

echo $livesum

sed -i '1d' dfused.txt

let livesum+=1

sed -i $livesum',$d' dfused.txt

sed -i $livesum',$d' ip.txt

linesum=$(cat ip.txt | wc -l)

echo $linesum

harr=($(awk '{print $1}' ip.txt))

darr=($(awk '{print $1}' dfused.txt))

if [ $linesum -gt 0 ]

then

for (( i = 0 ; i < ${#harr[@]} ; i++ ))

do

for (( j = 0 ; j < ${#darr[@]} ; j++ ))

do

if [ $i -eq $j ]

then

echo ${harr[$i]} ":" ${darr[$j]} >> hostdfs.txt

fi

done

done

else

echo "Not Live Datanodes"

fi

hharr=($(cat hostdfs.txt | awk '{print $3}' |sort -ru | tail -n 60))

ddarr=($(cat hostdfs.txt | awk '{print $3}' |sort -ru | head -n 100))

for (( k = 0; k< ${#hharr[@]} ; k++ ))

do

if [[ $(echo $(cat hostdfs.txt) | grep "${hharr[$k]}") != "" ]]

then

cat hostdfs.txt | grep "${hharr[$k]}" | awk 'NR==1{print $1}' >> minhost.txt

fi

done

for (( m = 0; m< ${#ddarr[@]} ; m++ ))

do

if [[ $(echo $(cat hostdfs.txt) | grep "${ddarr[$m]}") != "" ]]

then

cat hostdfs.txt | grep "${ddarr[$m]}" | awk 'NR==1{print $1}' >> maxhost.txt

fi

done

cat maxhost.txt | awk '{print $1}' >> host.txt

cat minhost.txt | awk '{print $1}' >> host.txt

narr=($(cat allhostname.txt | awk '{print $1}'))

for (( k = 0; k< ${#harr[@]}; k++ ))

do

for (( n = 0; n < ${#narr[@]}; n++))

do

if [ $k -eq $n ]

then

echo ${harr[$k]} ":" ${narr[$n]} >>iphostname.txt

fi

done

done

hostarr=($(cat host.txt | awk '{print $1}'))

for (( c = 0; c < ${#hostarr[@]} ; c++ ))

do

if [[ $(echo $(cat iphostname.txt) | grep "${hostarr[$c]}") != "" ]]

then

cat iphostname.txt | grep "${hostarr[$c]}" | awk 'NR==1{print $3}' >> hostname.txt

fi

done

max_rate=$(cat dfused.txt | sort -r | head -n 1 | awk '{print int($0)}')

avg_used_rate=$(cat dfused.txt | awk '{e+=$1}END{print e/NR}' | awk '{print int($0)}' )

let max_avg_diff=max_rate-avg_used_rate

if [ $max_avg_diff -gt 5 ]

then

jps | grep -i "balancer"

if [ $? -eq 0 ]

then

kill -9 $($JAVA_HOME/bin/jps | grep -i "balancer" | awk 'NR==1{print $1}')

fi

$HADOOP_HOME/bin/hdfs dfs -mv /system/balancer.id /system/balancer.id_`date +"%Y-%m-%d-%H-%M"`

nohup $HADOOP_HOME/bin/hdfs balancer -policy datanode -threshold 5 -include -f host.txt > /home/hdfs/balancer/rebalancer.log 2>&1 &

else

echo "Nothing to do"

fi