#!/usr/bin/env python#coding=utf-8
‘‘‘Created on 2013-8-22
@author: *****‘‘‘
importMySQLdbimporttimeimportpexpectimportsubprocessimportosimportsocketimporturllib2importsys
mintor_url= ‘http://**************************&msg=‘#短息接口
News_Result_=‘News_Result_‘tempdir= ‘datas/‘tmpexedir= ‘exedatas/‘reasultdir=‘redatas/‘
##########oline###########
pubid=int(14219)
dbip=‘********‘
globalipath
ipath=os.getcwd()+‘/‘
deftelme(info):globalmintor_url
tmp_url= mintor_url+info
socket.setdefaulttimeout(10)
response=urllib2.urlopen(tmp_url)
html=response.read()printhtml
response.close()defgetlogname(hour):return News_Result_+str(time.strftime(‘%Y%m%d%H‘,time.localtime(time.time()-hour*60*60)))+".log"
defdbexe(filename):
ida=str(time.strftime(‘%Y-%m-%d‘,time.localtime(time.time()-24*60*60)))
db=DB()
data_list=db.getpubnews(pubid ,ida)
newsower=db.getnewsowerinfo(ida)print ‘open result file:‘,filename
file=open(filename)
k=200dbresult=[];while 1:print ‘k:‘,kif k <=0:breaklines= file.readlines(200)if notlines:break
for line inlines:try:if k <=0:breakline=line.strip()
lists= line.split(‘|‘)if(lists[0].isdigit() and lists[1].isdigit() and newsower.count(int(lists[0]))<=0):
datainfo=[]
datainfo.append(int(lists[0]))
datainfo.append(int(k))
dbresult.append(datainfo)
k-=1
exceptException, ex:printex
dbresult.reverse()if k>100:print ‘data is to small,so return:‘,filenamereturn
ifdb.addpubnews(pubid,dbresult):if data_list[0] is notNone:
db.delpubnews(pubid, int(data_list[0]), int(data_list[1]))
file.close()
db.destory()defcheckexistsyn(name,num):if os.path.exists(name) ==False:
syndata(num)defawkexe():
log1= ipath+tempdir+getlogname(2)
log2= ipath+tempdir+getlogname(3)
log3= ipath+tempdir+getlogname(4)
checkexistsyn(log1,2)
checkexistsyn(log2,3)
checkexistsyn(log3,4)
reultname= ipath+reasultdir+str(time.strftime(‘%Y%m%d%H%M%S‘,time.localtime(time.time())))+‘.log‘cmd=‘cat‘+log1+‘ ‘+log2+‘ ‘+log3+‘| awk \‘$1 !~/^NULL/\‘ | awk -F "|" \‘{a[$1]+=$2}END{for(x in a)print x"|"a[x]}\‘ | sort -n -r -k 2 -t "|" >>‘+reultnameprint ‘cmd:‘,cmd
re= subprocess.call([cmd],shell=True)if re==0:print ‘success‘dbexe(reultname)else:
telme(‘awk-exe-fail-oa‘)defsyndata(hour):
logn=getlogname(hour)
command= ‘rsync -v user@**.**.***:*************t/‘+logn +" "+ipath+tempdirprint ‘command:‘,command
path= ipath+tempdir+logn
foo=pexpect.spawn(command)
index= foo.expect(["(?i)yes/no", "(?i)password", pexpect.EOF, pexpect.TIMEOUT])if (index ==0):print ‘need---------------yes‘foo.sendline(‘yes‘)
foo.expect("(?i)password")
foo.sendline(‘*******‘)
foo.expect(pexpect.EOF)
foo.close()print ‘syndata w success‘
elif(index == 1):print ‘need---------------pass‘foo.sendline(‘*******‘)
foo.expect(pexpect.EOF)
foo.close()print ‘syndata q success‘
else:print ‘has error‘
returnos.path.exists(path)defgethour():
d4=time.localtime(time.time())returnint(d4.tm_hour)defcheckDate():
hour=gethour()#if hour==9 or hour==12 or hour==17 or hour==20 or hour==0:
#return True
returnTruedefstartjob():
k=0while k < 10:
k+=1
if(syndata(2)):ifcheckDate():
awkexe()return
else:return
else:
time.sleep(60*5)
telme(‘-ryn-data-error-oa-‘+str(k))defteststartjob():passk=0while k < 1:
k+=1
if(syndata(2)) and (syndata(3)) and syndata(4):#if checkDate():
awkexe()return
else:
time.sleep(60*5)
telme(‘-ryn-data-error-oa‘)classDB(object):defdestory(self):if self.cur !=None:print ‘close db cur‘self.cur.close()if self.conn !=None:print ‘close db conn‘self.conn.close()print ‘close db‘
def __init__(self):
self.conn= MySQLdb.connect(host=dbip, user=‘***‘,passwd=‘*****‘, db=‘****‘, port=3306, charset=‘utf8‘)
self.cur=self.conn.cursor()defgetnewsowerinfo(self,startTime):
sql= "select newsid from ****** where (ownertype =3 or (ownertype=1 &&(ownerid in(1,2))) or (ownertype=2 && ownerid = 13793)) and createTime >=‘%s‘"
printsql
list=(startTime)
self.cur.execute(sql%list)
data_list=[]
data_list.extend( self.cur.fetchall())
result=[]for data indata_list:try:
result.append(int(data[0]))exceptException, eo:print "a date has error:",eoreturnresultdefaddpubnews(self,ipubid,datas):
idate=str(time.strftime(‘%Y-%m-%d %H:%M:%S‘,time.localtime(time.time())))sql= "insert into ********** (pubid, newsid,weight,newscreatetime,createtime,publishtime,createuser) values(%d,%d,%d,‘%s‘,‘%s‘,‘%s‘,‘%s‘)"
for data indatas:
list=(ipubid,data[0],data[1],idate,idate,idate,‘lixuan‘)#print "add sql:",sql%list
self.cur.execute(sql%list)#self.cur.executemany(sql,relist)
self.conn.commit()returnTruedefdelpubnews(self,ipubid, maxid,minid):print ‘start dele data , maxid:%d,pubid:%d‘%(maxid,ipubid)
sql= ‘delete from ********** where id <= %d and id >=%d and pubid = %d‘list=(maxid,minid,ipubid)
self.cur.execute(sql%list)
self.conn.commit()defgetpubnews(self, ipubid,idate):#and createtime < ‘%s‘
sql = "select max(id),min(id) ,count(1) from *********** where pubid =%d"list=(ipubid)
self.cur.execute(sql%list)
data_list=[]
data_list.extend( self.cur.fetchall())returndata_list[0];if __name__ == ‘__main__‘:#global ipath
if len(sys.argv) >1:
ipath= sys.argv[1]
idate=str(time.strftime(‘%Y-%m-%d %H:%M:%S‘,time.localtime(time.time())))print ‘start service...,‘,idate#teststartjob()
startjob()