统计搜索引擎的每小时抓取量及首页抓取量（第一版）

 

下一版，要改进为在一个AWK内统计出所有数据。

现在要六次循环，效率太差，暂时跑起来先。。

#/bin/sh

log_path="xxx.log"
log_self="xxx.log"
key_search=("http://www.baidu.com/search/spider.html" "http://www.google.com/bot.html"  "http://www.haosou.com/help")

time_inter_minute=60
bool_sendmail="true"
last_time=`date "-d -${time_inter_minute} minute" '+%d/%b/%G:%H:%M:%S'`
now_time=`date '+%d/%b/%G:%H:%M:%S'`
now_time_str="[${now_time}"
last_time_str="[${last_time}"

echo > $log_self
echo "start time: " $last_time >> $log_self
echo "end time: " $now_time >> $log_self

function searchSum()
{
  shell_out=`awk -v a=$1 -v b=$2 '{if ($4 > a && $4 < b) print}' $3|grep $4|wc -l`
  echo $shell_out
}

function searchRootSum()
{
  shell_root_out=`awk -v a=$1 -v b=$2 '{if ($4 > a && $4 < b && $7 == "/") print}' $3|grep $4|wc -l`
  echo $shell_root_out
}
for search_robot in ${key_search[@]}
do
    echo $search_robot >> $log_self
    count=`searchSum $last_time_str $now_time_str $log_path $search_robot`
    echo 全站抓取数量:$count >> $log_self
    countRoot=`searchRootSum $last_time_str $now_time_str $log_path $search_robot`
    echo 首页抓取数量:$countRoot >> $log_self
    if [ $count -eq 0 ];then
    mutt -s "xxx搜索引擎抓取统计"  x@x.com < $log_self
    fi
done

if [ $bool_sendmail == "true" ];then
    echo $bool_sendmail
fi 
mutt -s "xxx搜索引擎抓取统计" xx@xx.com < $log_self