shell与python联合进行日志分析——案例1

show_area.sh【对指定日期的日志进行分析，默认是昨天】

#! /bin/bash
echo "Usage: ./show_area.sh [20200210]"
cd /home/logs/bidder
mkdir area_ad_stat
#declare -a adid_pushid
#today=`date +%Y%m%d`

stat()
{
        cd $1;
        crit_file=`ls |grep crit`
        cat $crit_file |grep rtb_show |awk -F '\1' '{print $8}' > /home/logs/bidder/show_pushid.txt
        cat $crit_file |grep rtb_creative | awk -F '\1' '{print $8"\1"$15"\2"$7}' > /home/logs/bidder/pushid_adid_broadband.txt
        #cat /home/logs/bidder/show_pushid.txt | awk -F '\1' 'BEGIN{{FS="\1"}while(getline < "/home/logs/bidder/pushid_adid_broadband.txt"){print $1" "$2}}' ;{print a[$1]}' >/home/logs/bidder/broadband_adid.txt
        cat /home/logs/bidder/show_pushid.txt | awk -F '\1' 'BEGIN{{FS="\1"}while(getline < "/home/logs/bidder/pushid_adid_broadband.txt"){a[$1]=$2;}};{print a[$1]}' >/home/logs/bidder/adid_broadband.txt
        total=`cat /home/logs/bidder/adid_broadband.txt | wc -l`
        cat /home/logs/bidder/adid_broadband.txt |awk '/^0/' | awk -F '\2' '{print substr($1,1,4)" "$2}'> /home/logs/bidder/adid_area.txt
        area=`cat /home/logs/bidder/adid_area.txt |wc -l`
        echo "total : $total" > /home/logs/bidder/area_ad_stat/result_${1}.txt
        echo "$total $area" |awk '{printf("未知区域 : %d\n",$1 - $2)}'>> /home/logs/bidder/area_ad_stat/result_${1}.txt
        cat /home/logs/bidder/adid_area.txt |sort | uniq -c > /home/logs/bidder/area_ad_stat/area_ad_stat_${1}.txt
        rm /home/logs/bidder/adid_area.txt
        rm /home/logs/bidder/show_pushid.txt
        rm /home/logs/bidder/pushid_adid_broadband.txt
        rm /home/logs/bidder/adid_broadband.txt
}
if [ $# -eq 1 ]
then
        stat $1
else
        echo "no data to process!"
fi

adid.py【调用show_area.sh脚本，获取日志分析结果，访问mysql获取adid与planname 建立关系，合并成完成的日志分析文件】

# coding=utf-8
import time
import datetime
import MySQLdb
import sys
import os
reload(sys)

print "Usage: python adid.py [date/20200210],default is yesterday's date"
#today = time.strftime("%Y%m%d",time.localtime(time.time()))
today = datetime.datetime.now()
offset = datetime.timedelta(days = -1)
yesterday = (today + offset).strftime('%Y%m%d')
if len(sys.argv) == 1:
        date = yesterday
else:
        date = sys.argv[1]

sys.setdefaultencoding('utf-8')
ret = os.system('sh ./show_area.sh '+ date)
if ret != 0:
        print("show_area.sh执行失败！")
        exit()

d1 = {}
d2 = {}
d3 = {}
conn = MySQLdb.connect(host = "**********", user = "root", passwd = "*********", db = "adp",port = 13306, charset='utf8' )
cursor = conn.cursor()

sql = "select adid , plan_id  from adp_ad_info;"
cursor.execute(sql)
adid_planid = cursor.fetchall()
for i in adid_planid:
        d1[i[1]] = i[0]
sql = "select plan_name , plan_id  from adp_plan_info;"
cursor.execute(sql)
groupname_planid = cursor.fetchall()
for i in groupname_planid:
        d2[i[1]] = i[0]

for i in d1:
        for j in d2:
                if i == j:
                        d3[str(d1[i])] = d2[j]

file1 = "/home/logs/bidder/area_ad_stat/area_ad_stat_"  + date + ".txt"
file2 = "/home/logs/bidder/area_ad_stat/result_"  + date + ".txt"
f2 = open(file2,'a')
with open(file1,'r') as f1:
        f2.write("日期 活动名 广告ID 宽带地区号 展示次数\n")
        for line in f1:
                s = line.split(' ')
                adid = s[len(s) -1]
                adid = adid[0:(len(adid)-1)]
                s[len(s) - 1] = adid
                #print type(adid)
                #print d3[adid]
                #print type(d3[adid])
                s.append(d3[adid].encode("utf-8"))
                s.append(date)
                for i in range(len(s)-1,len(s)-6,-1):
                        f2.write(str(s[i]) + " ")
                f2.write('\n')
f2.close()
os.system('rm '+ file1)