python模拟登录页面下载_python模拟登陆之下载

#!/usr/bin/python#coding:utf-8importsysimportosimportdjangoreload(sys)sys.setdefaultencoding('utf8')sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) #把manage.py所在目录添加到系统目录os.environ['DJANGO_SETTINGS_MODULE'] = 'business.settings' #设置setting文件django.setup() #初始化Django环境importrequestsimportreimportloggingimportbase64importxlrdimportdatetimeimporttimeimportMySQLdbimportthreadpoolfrom business importsettingsfrom train.depends.platform importPlatformfrom train.models importTbTomasOrder,TbTomasEpay,TtTicketThomas,TbTomasLinkmanfrom train importutilsfrom train.status importOrderStatusfrom django.core.mail importEmailMultiAlternativesfrom train.busi importinsert_order,insert_ticket,insert_epay,insert_linkmanlogger= logging.getLogger('django')classTbTomas(object): succ_number=0 fail_number=0 fail_order=[]def __init__(self,thread_num = 3):#配置初始化 self.session_obj =requests.session() self.fail_order=[] self.succ_number=0 self.fail_number=0 self.thread_num=thread_num self.start_date= ''self.end_date= ''self.trade_date=utils.now()deflogin_thomas(self,thomas_username,thomas_password): hello_url= 'https://huoche.alitrip.com/hello.htm'hello_response=self.session_obj.get(hello_url) h_u_s= re_search('', hello_response.text) h_u_s=base64.b64encode(h_u_s) headers={'Accept': 'text/html, application/xhtml+xml, image/jxr, */*','Referer': 'https://huoche.alitrip.com/hello.htm','Accept-Language': 'zh-CN','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586','Content-Type': 'application/x-www-form-urlencoded','Accept-Encoding': 'gzip, deflate','Host': 'huoche.alitrip.com','Content-Length': '73','Connection': 'Keep-Alive','Cache-Control': 'no-cache'} post_data={'h_u_s': base64.b64encode(h_u_s),'h_u_n': thomas_username,'h_u_p': base64.b64encode(thomas_password) } index_url= 'https://huoche.alitrip.com/index.htm'index_response= self.session_obj.post(index_url, headers=headers, data=post_data) logger.info(u'登陆成功，等待下载文件...')defdownload_file(self,thomas_username,thomas_password,args):for _ in xrange(3):try: self.login_thomas(thomas_username,thomas_password)break exceptException as e: logger.error(e)continue #处理时间 all_time =self.date_time_handle(args)if notall_time: logger.error(u'日期格式错误!!')return for trade_date inall_time:try: self.trade_date=trade_date post_data={'orderExportDate': trade_date } download_url= 'https://huoche.alitrip.com/orderlistexp.do'sheet_content= '' for _ in xrange(3):try:#得到exal文件流 download_response = self.session_obj.post(download_url, data=post_data)#打开exal文件 xls_content = xlrd.open_workbook(file_contents=download_response.content) sheet_content=xls_content.sheets()[0] logger.info(u'下载文件成功，正在拿取订单号')break exceptException as e: logger.error(u'下载文件超时，正在等待重新登录后下载...') self.login_thomas(thomas_username, thomas_password)continueorder_item=[]if notsheet_content: logger.error(u'下载文件失败，正在重新登录...')continue for line_num inrange(sheet_content.nrows): line_item=sheet_content.row_values(line_num)if line_item[2] and line_item[2] not inorder_item: order_item.append(line_item[2], ) #订单号 order_no #获取到所有订单号 order_item = order_item[1:]#根据订单号去拿订单详情 logger.info(u'正在写入数据库')#多线程去执行 pool =threadpool.ThreadPool(self.thread_num) reqs=threadpool.makeRequests(self.create_order_info, order_item) [pool.putRequest(req)for req inreqs] pool.wait() logger.info(u'写入完成，完成时间为：%s'%self.trade_date) content=self.add_content(len(order_item), self.succ_number, self.fail_number, self.fail_order) self.send_mail(content=content) self.succ_number,self.fail_order=0,0 self.fail_order=[]#self.create_order_info(order_item) exceptException as e: logger.error(e)defdate_time_handle(self,args): all_time=[]ifargs:if len(args) == 1: self.start_date= datetime.datetime.strptime(args[0], '%Y-%m-%d').date() self.end_date= datetime.datetime.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d').date()elif len(args) == 2: self.start_date= datetime.datetime.strptime(args[0], '%Y-%m-%d').date() self.end_date= datetime.datetime.strptime(args[1], '%Y-%m-%d').date()elif len(args) == 3: self.start_date= datetime.datetime.strptime(args[0], '%Y-%m-%d').date() self.end_date= datetime.datetime.strptime(args[1], '%Y-%m-%d').date() self.thread_num= int(args[2])else: logger.error(u'传入参数错误，请重新执行')returni=0whileTrue: tomoary= self.start_date + datetime.timedelta(days=i) trade_date= tomoary.strftime('%Y-%m-%d') all_time.append(trade_date) i+= 1 if tomoary ==self.end_date:break else: today=datetime.datetime.now() yesterday= today + datetime.timedelta(days=-1) trade_date= yesterday.strftime('%Y-%m-%d') all_time.append(trade_date)returnall_timedefcreate_order_info(self, order): platform_obj=Platform() order_info=platform_obj.get_order(order)if notorder_info: self.fail_order.append(order) self.fail_number+= 1logger.error('获取订单号:[%s]失败'%order)return try:#插入order表 if TbTomasOrder.objects.filter(order_no=order).exists(): logger.error('订单号:[%s]已经存在于TbTomasOrder'%order) self.fail_order.append(order) self.fail_number+= 1 return else: insert_order(order_info,order,self.trade_date) self.succ_number+= 1 #插入ticket表insert_ticket(order_info,order,self.trade_date)#插入联系人 if TbTomasLinkman.objects.filter(order_no=order).exists(): logger.error('订单号:[%s]已经存在于TbTomasLinkman'%order)else: insert_linkman(order_info,order,self.trade_date)#插入epay表 if TbTomasEpay.objects.filter(order_no=order).exists(): logger.error('订单号:[%s]已经存在于TbTomasEpay'%order)else: insert_epay(order_info,order,self.trade_date)exceptException as e: logger.error(e) self.fail_number+=1 defadd_content(self,total,succ_number,fail_number,fail_order): content= u'''

托马斯导入订单报表

日期	总单数	成功单数	失败单数	失败订单号
%s	%s	%s	%s	%s

'''%(datetime.datetime.now().strftime('%Y-%m-%d %H:%M'),total,succ_number,fail_number,fail_order)returncontentdefsend_mail(self, content): time_target=self.trade_date subject= u'托马斯数据抓取邮件 %s' %(time_target) logger.info(u'准备发送邮件....%s', subject) mail_address=settings.mail_address_thomas to_addr=[]ifisinstance(mail_address, list): to_addr+=mail_addresselifisinstance(mail_address, str): to_addr.append(mail_address) logger.debug(to_addr) from_email=settings.DEFAULT_FROM_EMAIL msg= EmailMultiAlternatives(subject, 'result', from_email, to_addr) msg.attach_alternative(content,'text/html') flag=msg.send()ifflag: logger.info(u'%s发送成功', subject)else: logger.error(u'%s发送失败', subject)return defrun(self, username,passwd,args):#登陆托马斯后台 for _ in xrange(3):try: self.download_file(username,passwd,args)break exceptException as e: logger.error(e)continuedefre_search(regex, subject): subject=str(subject) obj=re.compile(regex) match=obj.search(subject)ifmatch: result= match.group(1)else: result= '' returnresultdefmain(): username=base64.b64decode(settings.THOMAS_USERNAME) passwd=base64.b64decode(settings.THOMAS_PASSWORD) args= sys.argv[1:] if sys.argv[1:] else ''TbTomas().run(username,passwd,args)if __name__ == '__main__': main()