python自动登录网站下载文件_python模拟登陆之下载

#!/usr/bin/python#coding:utf-8

importsysimportosimportdjango

reload(sys)

sys.setdefaultencoding('utf8')

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) #把manage.py所在目录添加到系统目录

os.environ['DJANGO_SETTINGS_MODULE'] = 'business.settings' #设置setting文件

django.setup() #初始化Django环境

importrequestsimportreimportloggingimportbase64importxlrdimportdatetimeimporttimeimportMySQLdbimportthreadpoolfrom business importsettingsfrom train.depends.platform importPlatformfrom train.models importTbTomasOrder,TbTomasEpay,TtTicketThomas,TbTomasLinkmanfrom train importutilsfrom train.status importOrderStatusfrom django.core.mail importEmailMultiAlternativesfrom train.busi importinsert_order,insert_ticket,insert_epay,insert_linkman

logger= logging.getLogger('django')classTbTomas(object):

succ_number=0

fail_number=0

fail_order=[]def __init__(self,thread_num = 3):#配置初始化

self.session_obj =requests.session()

self.fail_order=[]

self.succ_number=0

self.fail_number=0

self.thread_num=thread_num

self.start_date= ""self.end_date= ""self.trade_date=utils.now()deflogin_thomas(self,thomas_username,thomas_password):

hello_url= 'https://huoche.alitrip.com/hello.htm'hello_response=self.session_obj.get(hello_url)

h_u_s= re_search('', hello_response.text)

h_u_s=base64.b64encode(h_u_s)

headers={'Accept': 'text/html, application/xhtml+xml, image/jxr, */*','Referer': 'https://huoche.alitrip.com/hello.htm','Accept-Language': 'zh-CN','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586','Content-Type': 'application/x-www-form-urlencoded','Accept-Encoding': 'gzip, deflate','Host': 'huoche.alitrip.com','Content-Length': '73','Connection': 'Keep-Alive','Cache-Control': 'no-cache'}

post_data={'h_u_s': base64.b64encode(h_u_s),'h_u_n': thomas_username,'h_u_p': base64.b64encode(thomas_password)

}

index_url= 'https://huoche.alitrip.com/index.htm'index_response= self.session_obj.post(index_url, headers=headers, data=post_data)

logger.info(u"登陆成功，等待下载文件...")defdownload_file(self,thomas_username,thomas_password,args):for _ in xrange(3):try:

self.login_thomas(thomas_username,thomas_password)break

exceptException as e:

logger.error(e)continue

#处理时间

all_time =self.date_time_handle(args)if notall_time:

logger.error(u"日期格式错误!!")return

for trade_date inall_time:try:

self.trade_date=trade_date

post_data={'orderExportDate': trade_date

}

download_url= 'https://huoche.alitrip.com/orderlistexp.do'sheet_content= ""

for _ in xrange(3):try:#得到exal文件流

download_response = self.session_obj.post(download_url, data=post_data)#打开exal文件

xls_content = xlrd.open_workbook(file_contents=download_response.content)

sheet_content=xls_content.sheets()[0]

logger.info(u"下载文件成功，正在拿取订单号")break

exceptException as e:

logger.error(u"下载文件超时，正在等待重新登录后下载...")

self.login_thomas(thomas_username, thomas_password)continueorder_item=[]if notsheet_content:

logger.error(u'下载文件失败，正在重新登录...')continue

for line_num inrange(sheet_content.nrows):

line_item=sheet_content.row_values(line_num)if line_item[2] and line_item[2] not inorder_item:

order_item.append(line_item[2], ) #订单号 order_no

#获取到所有订单号

order_item = order_item[1:]#根据订单号去拿订单详情

logger.info(u"正在写入数据库")#多线程去执行

pool =threadpool.ThreadPool(self.thread_num)

reqs=threadpool.makeRequests(self.create_order_info, order_item)

[pool.putRequest(req)for req inreqs]

pool.wait()

logger.info(u'写入完成，完成时间为：%s'%self.trade_date)

content=self.add_content(len(order_item), self.succ_number, self.fail_number, self.fail_order)

self.send_mail(content=content)

self.succ_number,self.fail_order=0,0

self.fail_order=[]#self.create_order_info(order_item)

exceptException as e:

logger.error(e)defdate_time_handle(self,args):

all_time=[]ifargs:if len(args) == 1:

self.start_date= datetime.datetime.strptime(args[0], "%Y-%m-%d").date()

self.end_date= datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d").date()elif len(args) == 2:

self.start_date= datetime.datetime.strptime(args[0], "%Y-%m-%d").date()

self.end_date= datetime.datetime.strptime(args[1], "%Y-%m-%d").date()elif len(args) == 3:

self.start_date= datetime.datetime.strptime(args[0], "%Y-%m-%d").date()

self.end_date= datetime.datetime.strptime(args[1], "%Y-%m-%d").date()

self.thread_num= int(args[2])else:

logger.error(u"传入参数错误，请重新执行")returni=0whileTrue:

tomoary= self.start_date + datetime.timedelta(days=i)

trade_date= tomoary.strftime('%Y-%m-%d')

all_time.append(trade_date)

i+= 1

if tomoary ==self.end_date:break

else:

today=datetime.datetime.now()

yesterday= today + datetime.timedelta(days=-1)

trade_date= yesterday.strftime('%Y-%m-%d')

all_time.append(trade_date)returnall_timedefcreate_order_info(self, order):

platform_obj=Platform()

order_info=platform_obj.get_order(order)if notorder_info:

self.fail_order.append(order)

self.fail_number+= 1logger.error('获取订单号:[%s]失败'%order)return

try:#插入order表

if TbTomasOrder.objects.filter(order_no=order).exists():

logger.error('订单号:[%s]已经存在于TbTomasOrder'%order)

self.fail_order.append(order)

self.fail_number+= 1

return

else:

insert_order(order_info,order,self.trade_date)

self.succ_number+= 1

#插入ticket表

insert_ticket(order_info,order,self.trade_date)#插入联系人

if TbTomasLinkman.objects.filter(order_no=order).exists():

logger.error('订单号:[%s]已经存在于TbTomasLinkman'%order)else:

insert_linkman(order_info,order,self.trade_date)#插入epay表

if TbTomasEpay.objects.filter(order_no=order).exists():

logger.error('订单号:[%s]已经存在于TbTomasEpay'%order)else:

insert_epay(order_info,order,self.trade_date)exceptException as e:

logger.error(e)

self.fail_number+=1

defadd_content(self,total,succ_number,fail_number,fail_order):

content= u'''

托马斯导入订单报表

日期	总单数	成功单数	失败单数	失败订单号
%s	%s	%s	%s	%s

'''%(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"),total,succ_number,fail_number,fail_order)returncontentdefsend_mail(self, content):

time_target=self.trade_date

subject= u'托马斯数据抓取邮件 %s' %(time_target)

logger.info(u'准备发送邮件....%s', subject)

mail_address=settings.mail_address_thomas

to_addr=[]ifisinstance(mail_address, list):

to_addr+=mail_addresselifisinstance(mail_address, str):

to_addr.append(mail_address)

logger.debug(to_addr)

from_email=settings.DEFAULT_FROM_EMAIL

msg= EmailMultiAlternatives(subject, 'result', from_email, to_addr)

msg.attach_alternative(content,"text/html")

flag=msg.send()ifflag:

logger.info(u'%s发送成功', subject)else:

logger.error(u'%s发送失败', subject)return

defrun(self, username,passwd,args):#登陆托马斯后台

for _ in xrange(3):try:

self.download_file(username,passwd,args)break

exceptException as e:

logger.error(e)continue

defre_search(regex, subject):

subject=str(subject)

obj=re.compile(regex)

match=obj.search(subject)ifmatch:

result= match.group(1)else:

result= ''

returnresultdefmain():

username=base64.b64decode(settings.THOMAS_USERNAME)

passwd=base64.b64decode(settings.THOMAS_PASSWORD)

args= sys.argv[1:] if sys.argv[1:] else ""TbTomas().run(username,passwd,args)if __name__ == "__main__":

main()