show_area.sh【对指定日期的日志进行分析,默认是昨天】
#! /bin/bash
echo "Usage: ./show_area.sh [20200210]"
cd /home/logs/bidder
mkdir area_ad_stat
#declare -a adid_pushid
#today=`date +%Y%m%d`
stat()
{
cd $1;
crit_file=`ls |grep crit`
cat $crit_file |grep rtb_show |awk -F '\1' '{print $8}' > /home/logs/bidder/show_pushid.txt
cat $crit_file |grep rtb_creative | awk -F '\1' '{print $8"\1"$15"\2"$7}' > /home/logs/bidder/pushid_adid_broadband.txt
#cat /home/logs/bidder/show_pushid.txt | awk -F '\1' 'BEGIN{{FS="\1"}while(getline < "/home/logs/bidder/pushid_adid_broadband.txt"){print $1" "$2}}' ;{print a[$1]}' >/home/logs/bidder/broadband_adid.txt
cat /home/logs/bidder/show_pushid.txt | awk -F '\1' 'BEGIN{{FS="\1"}while(getline < "/home/logs/bidder/pushid_adid_broadband.txt"){a[$1]=$2;}};{print a[$1]}' >/home/logs/bidder/adid_broadband.txt
total=`cat /home/logs/bidder/adid_broadband.txt | wc -l`
cat /home/logs/bidder/adid_broadband.txt |awk '/^0/' | awk -F '\2' '{print substr($1,1,4)" "$2}'> /home/logs/bidder/adid_area.txt
area=`cat /home/logs/bidder/adid_area.txt |wc -l`
echo "total : $total" > /home/logs/bidder/area_ad_stat/result_${1}.txt
echo "$total $area" |awk '{printf("未知区域 : %d\n",$1 - $2)}'>> /home/logs/bidder/area_ad_stat/result_${1}.txt
cat /home/logs/bidder/adid_area.txt |sort | uniq -c > /home/logs/bidder/area_ad_stat/area_ad_stat_${1}.txt
rm /home/logs/bidder/adid_area.txt
rm /home/logs/bidder/show_pushid.txt
rm /home/logs/bidder/pushid_adid_broadband.txt
rm /home/logs/bidder/adid_broadband.txt
}
if [ $# -eq 1 ]
then
stat $1
else
echo "no data to process!"
fi
adid.py【调用show_area.sh脚本,获取日志分析结果,访问mysql获取adid与planname 建立关系,合并成完成的日志分析文件】
# coding=utf-8
import time
import datetime
import MySQLdb
import sys
import os
reload(sys)
print "Usage: python adid.py [date/20200210],default is yesterday's date"
#today = time.strftime("%Y%m%d",time.localtime(time.time()))
today = datetime.datetime.now()
offset = datetime.timedelta(days = -1)
yesterday = (today + offset).strftime('%Y%m%d')
if len(sys.argv) == 1:
date = yesterday
else:
date = sys.argv[1]
sys.setdefaultencoding('utf-8')
ret = os.system('sh ./show_area.sh '+ date)
if ret != 0:
print("show_area.sh执行失败!")
exit()
d1 = {}
d2 = {}
d3 = {}
conn = MySQLdb.connect(host = "**********", user = "root", passwd = "*********", db = "adp",port = 13306, charset='utf8' )
cursor = conn.cursor()
sql = "select adid , plan_id from adp_ad_info;"
cursor.execute(sql)
adid_planid = cursor.fetchall()
for i in adid_planid:
d1[i[1]] = i[0]
sql = "select plan_name , plan_id from adp_plan_info;"
cursor.execute(sql)
groupname_planid = cursor.fetchall()
for i in groupname_planid:
d2[i[1]] = i[0]
for i in d1:
for j in d2:
if i == j:
d3[str(d1[i])] = d2[j]
file1 = "/home/logs/bidder/area_ad_stat/area_ad_stat_" + date + ".txt"
file2 = "/home/logs/bidder/area_ad_stat/result_" + date + ".txt"
f2 = open(file2,'a')
with open(file1,'r') as f1:
f2.write("日期 活动名 广告ID 宽带地区号 展示次数\n")
for line in f1:
s = line.split(' ')
adid = s[len(s) -1]
adid = adid[0:(len(adid)-1)]
s[len(s) - 1] = adid
#print type(adid)
#print d3[adid]
#print type(d3[adid])
s.append(d3[adid].encode("utf-8"))
s.append(date)
for i in range(len(s)-1,len(s)-6,-1):
f2.write(str(s[i]) + " ")
f2.write('\n')
f2.close()
os.system('rm '+ file1)