import requests
import json
import re
from urllib import parse
requests=requests.session()
url0="https://900458.private.mabangerp.com/index.htm"
url ="https://900458.private.mabangerp.com/index.php?mod=main.doLogin"
headers0 ={
'Host':'900458.private.mabangerp.com','Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8','Accept-Language':'zh-CN,zh;q=0.9','User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36','Upgrade-Insecure-Requests':'1','Cache-Control':'max-age=0','Connection':'keep-alive'}
payload ="isMallRpcFinds=&username=111111111&password=jieyi123"
headers ={
'Origin':'https://900458.private.mabangerp.com','Accept-Encoding':'gzip, deflate, br','Accept-Language':'zh-CN,zh;q=0.9','User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36','Content-Type':'application/x-www-form-urlencoded; charset=UTF-8','Accept':'application/json, text/javascript, */*; q=0.01','X-Requested-With':'XMLHttpRequest','Connection':'keep-alive'}
requests.request("GET", url, headers=headers0)
response = requests.request("POST", url, headers=headers, data = payload)
t_start='2020-07-01'
t_end='2020-10-01'
url ="https://900458.private.mabangerp.com/index.php?mod=order.orderSearch"
payload="PrintCenterOrderIdssql=&fbaFlag=&Order.orderStatus=&OrderCurrency.beforeStatus=&printCount=&labelMultipleChoiceWhere=cross&TextVal=weight&TextZx=&TextZd=&TextFee=OrderFee&minOrderFee=&maxOrderFee=&queryTime=paidTime&startTime1="+t_start+"+00%3A00%3A00&endTime1="+t_end+"+00%3A00%3A00&itemCount=&queryTime2=&OrderSearch.fuzzySearchKey=&OrderSearch.fuzzySearchKey1=&grid=&providerName=&OrderItem.developerId=&smtSearchVal=&orderhighfastsearch=&parentCategoryId=&categoryId=&OrderItem.stockStatus=&orderSearchHistory=&orderPageKey=00542f581084128b7dced69b48849733&goPaypalRefundStatus=1&page=1&rowsPerPage=100&Order_isCloud=2&m=order&OrderPlus.isNewOrder=2&a=orderalllist&isNewOrderPage=1&post_tableBase=1&showError=undefined&pageListC=undefined&1=1"# payload = "PrintCenterOrderIdssql=&fbaFlag=&Order.orderStatus=&OrderCurrency.beforeStatus=&printCount=&labelMultipleChoiceWhere=cross&TextVal=weight&TextZx=&TextZd=&TextFee=OrderFee&minOrderFee=&maxOrderFee=&queryTime=createDate&startTime1="+t_start+"+00%3A00%3A00&endTime1="+t_end+"+00%3A00%3A00&itemCount=&queryTime2=&OrderSearch.fuzzySearchKey=&OrderSearch.fuzzySearchKey1=&grid=&providerName=&OrderItem.developerId=&smtSearchVal=&orderhighfastsearch=&parentCategoryId=&categoryId=&OrderItem.stockStatus=&orderSearchHistory=&orderPageKey=11d7ca827bdc33f37e8f2f5f50fb0973&goPaypalRefundStatus=1&page=1&rowsPerPage=100&Order_isCloud=2&m=order&OrderPlus.isNewOrder=2&a=orderalllist&isNewOrderPage=1&post_tableBase=1&showError=undefined&pageListC=undefined&1=1"
response = requests.request("POST", url, headers=headers, data = payload)print(divmod(response.json()['pageCount'],5000))
rj=response.json()
content = rj["pageHtml"]# print(content)# r='{"data":"(.*?)"}</span>' #待配货
r='"data":"(.*?)","page"'
r=re.compile(r)# print(parse.quote(r.findall(content)[0]))
pd=parse.quote(r.findall(content)[0])
dm=divmod(response.json()['pageCount'],5000)
toltalpage= dm[0]+1if dm[1]>0else dm[0]print(toltalpage)
allrecords=[]
cookies=requests.cookies.get_dict()print(cookies)import asyncio
import aiohttp
import time
import csv
defstep1(cookies,pd,toltalpage):# start=time.time()asyncdefpost(url,headers,data,cookies):# conn = aiohttp.TCPConnector(limit=2)
conn = aiohttp.TCPConnector(limit_per_host=1)
session=aiohttp.ClientSession(headers=headers,cookies=cookies,connector=conn)
response=await session.post(url,data=data)# dlist.append(await response.json()['gourl'])# print(json.loads(str(await response.text()))['gourl'])# await asyncio.sleep(1)
allrecords.append(json.loads(str(await response.text()))['hasPlatformOrderId'])# await response.text()await session.close()# return responseasyncdefrequest(pd,i):
data ="PrintCenterOrderIdssql="+pd+"&fbaFlag=&Order.orderStatus=2&OrderCurrency.beforeStatus=&printCount=&TextVal=weight&TextZx=&TextZd=&TextFee=OrderFee&minOrderFee=&maxOrderFee=&queryTime=&itemCount=&queryTime2=&OrderSearch.fuzzySearchKey=&OrderSearch.fuzzySearchKey1=&grid=&providerName=&OrderItem.developerId=&smtSearchVal=&orderhighfastsearch=&parentCategoryId=&categoryId=&OrderItem.stockStatus=&orderSearchHistory=&page="+str(i)+"&ordersearchtype=exportsearch&exportsearchnum=5000&p_tableBase=&employeeId=308469"
headers ={
'Accept':'application/json, text/javascript, */*; q=0.01','Referer':'https://private-amz.mabangerp.com/index.php?mod=order.exportOrderByTemplate&isCloud=2&tableBase=&os=99&orderItemOrderBy=platformSku+asc%2CisCombo+asc%2CstockId+asc%2Cid+asc&cMKey=0bac0855cad15bb5eb5fbf3117311c5e&platformOrderIdsMkey=EXPORT_ORDER_BY_TEMPLATE_INTERFACE_308469&platformOrderIdssqlMkey=EXPORT_ORDER_BY_TEMPLATE_SQL_INTERFACE_308469&lang=cn','Origin':'https://private-amz.mabangerp.com','User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36','Content-Type':'application/x-www-form-urlencoded; charset=UTF-8'}
url ="https://print-private.mabangerp.com/index.php?mod=order.orderSearch"try:await post(url,headers,data,cookies)except:await post(url,headers,data,cookies)
tasks =[asyncio.ensure_future(request(pd,i))for i inrange(1,toltalpage+1)]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))# with open('111.csv', 'w', newline='') as csvfile:# writer = csv.writer(csvfile,delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL)# for rows in allrecords:# for row in rows:# writer.writerow(row)# end = time.time()# print(end - start)# for i in range(1,toltalpage+1):# print(i)# url = "https://print-private.mabangerp.com/index.php?mod=order.orderSearch"# payload = "PrintCenterOrderIdssql="+parse.quote(r.findall(content)[0])+"&fbaFlag=&Order.orderStatus=2&OrderCurrency.beforeStatus=&printCount=&TextVal=weight&TextZx=&TextZd=&TextFee=OrderFee&minOrderFee=&maxOrderFee=&queryTime=&itemCount=&queryTime2=&OrderSearch.fuzzySearchKey=&OrderSearch.fuzzySearchKey1=&grid=&providerName=&OrderItem.developerId=&smtSearchVal=&orderhighfastsearch=&parentCategoryId=&categoryId=&OrderItem.stockStatus=&orderSearchHistory=&page="+str(i)+"&ordersearchtype=exportsearch&exportsearchnum=5000&p_tableBase=&employeeId=308469"# headers = {
# 'Accept': 'application/json, text/javascript, */*; q=0.01',# 'Referer': 'https://private-amz.mabangerp.com/index.php?mod=order.exportOrderByTemplate&isCloud=2&tableBase=&os=99&orderItemOrderBy=platformSku+asc%2CisCombo+asc%2CstockId+asc%2Cid+asc&cMKey=0bac0855cad15bb5eb5fbf3117311c5e&platformOrderIdsMkey=EXPORT_ORDER_BY_TEMPLATE_INTERFACE_308469&platformOrderIdssqlMkey=EXPORT_ORDER_BY_TEMPLATE_SQL_INTERFACE_308469&lang=cn',# 'Origin': 'https://private-amz.mabangerp.com',# 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36',# 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'# }# response = requests.request("POST", url, headers=headers, data = payload)# allrecords.append(response.json()['hasPlatformOrderId'])
headers ={
'Accept':'application/json, text/javascript, */*; q=0.01','Referer':'https://private-amz.mabangerp.com/index.php?mod=order.exportOrderByTemplate&isCloud=2&tableBase=&os=99&orderItemOrderBy=platformSku+asc%2CisCombo+asc%2CstockId+asc%2Cid+asc&cMKey=0bac0855cad15bb5eb5fbf3117311c5e&platformOrderIdsMkey=EXPORT_ORDER_BY_TEMPLATE_INTERFACE_308469&platformOrderIdssqlMkey=EXPORT_ORDER_BY_TEMPLATE_SQL_INTERFACE_308469&lang=cn','Origin':'https://private-amz.mabangerp.com','User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36','Content-Type':'application/x-www-form-urlencoded; charset=UTF-8'}
url='https://private-amz.mabangerp.com/index.php?mod=order.exportOrderByTemplate&isCloud=2&tableBase=&os=&orderItemOrderBy=stockId+asc%2Cid+asc&cMKey=0bac0855cad15bb5eb5fbf3117311c5e&platformOrderIdsMkey=EXPORT_ORDER_BY_TEMPLATE_INTERFACE_308469&platformOrderIdssqlMkey=EXPORT_ORDER_BY_TEMPLATE_SQL_INTERFACE_308469&lang=cn'
requests.request("GET", url, headers=headers)print(requests.cookies.get_dict())# print(len(allrecords[0]))
step1(cookies,pd,toltalpage)print(len(allrecords))
dlist=[]
cookies=requests.cookies.get_dict()defstep2(cookies):
start=time.time()asyncdefpost(url,headers,data,cookies):
conn = aiohttp.TCPConnector(limit_per_host=1)
session=aiohttp.ClientSession(headers=headers,cookies=cookies,connector=conn)try:
response=await session.post(url,data=data)# await asyncio.sleep(1)# dlist.append(await response.json()['gourl'])print(json.loads(str(await response.text()))['gourl'])
dlist.append(json.loads(str(await response.text()))['gourl'])# await response.text()except Exception as e:
response=await session.post(url,data=data)# await asyncio.sleep(1)# dlist.append(await response.json()['gourl'])print(json.loads(str(await response.text()))['gourl'])
dlist.append(json.loads(str(await response.text()))['gourl'])finally:await session.close()# return responseasyncdefrequest(li):
data ='backUrl=&orderIds='+'%0D%0A'.join(li)+'&fieldlabel=uq101&fieldlabel=uq102&fieldlabel=uq113&fieldlabel=uq115&fieldlabel=uq128&fieldlabel=uq130&fieldlabel=uq135&fieldlabel=uq136&fieldlabel=uq141&fieldlabel=uq195&fieldlabel=uq149&fieldlabel=uq160&fieldlabel=uq216&fieldlabel=uq226&fieldlabel=uq104&fieldlabel=uq105&fieldlabel=uq108&fieldlabel=uq109&fieldlabel=uq110&fieldlabel=uq111&fieldlabel=uq145&fieldlabel=uq152&fieldlabel=uq201&fieldlabel=uq146&fieldlabel=uq250&fieldlabel=uq156&fieldlabel=uq255&fieldlabel=uq119&fieldlabel=uq120&fieldlabel=uq196&fieldlabel=uq260&map-name%5B%5D=%E4%BB%98%E6%AC%BE%E6%97%B6%E9%97%B4&map-uq%5B%5D=uq115&map-text%5B%5D=&map-name%5B%5D=%E8%AE%A2%E5%8D%95%E7%BC%96%E5%8F%B7&map-uq%5B%5D=uq101&map-text%5B%5D=&map-name%5B%5D=%E5%BA%97%E9%93%BA%E5%90%8D&map-uq%5B%5D=uq135&map-text%5B%5D=&map-name%5B%5D=%E4%BA%A4%E6%98%93%E7%BC%96%E5%8F%B7&map-uq%5B%5D=uq102&map-text%5B%5D=&map-name%5B%5D=%E7%8A%B6%E6%80%81&map-uq%5B%5D=uq136&map-text%5B%5D=&map-name%5B%5D=%E8%AE%A2%E5%8D%95%E5%8E%9F%E5%A7%8B%E6%80%BB%E9%87%91%E9%A2%9D%EF%BC%88%E9%A6%96%E8%AE%B0%E5%BD%95%EF%BC%89&map-uq%5B%5D=uq250&map-text%5B%5D=&map-name%5B%5D=%E5%8E%9F%E5%A7%8B%E5%95%86%E5%93%81%E6%80%BB%E9%87%91%E9%A2%9D&map-uq%5B%5D=uq146&map-text%5B%5D=&map-name%5B%5D=%E5%B8%81%E7%A7%8D&map-uq%5B%5D=uq156&map-text%5B%5D=&map-name%5B%5D=%E5%AE%A2%E6%88%B7%E5%A7%93%E5%90%8D&map-uq%5B%5D=uq104&map-text%5B%5D=&map-name%5B%5D=%E7%94%B5%E8%AF%9D1&map-uq%5B%5D=uq105&map-text%5B%5D=&map-name%5B%5D=%E8%81%94%E7%B3%BB%E9%82%AE%E7%AE%B1&map-uq%5B%5D=uq145&map-text%5B%5D=&map-name%5B%5D=%E5%9B%BD%E5%AE%B6(%E4%B8%AD)&map-uq%5B%5D=uq152&map-text%5B%5D=&map-name%5B%5D=%E6%89%80%E5%B1%9E%E5%9C%B0%E5%8C%BA&map-uq%5B%5D=uq108&map-text%5B%5D=&map-name%5B%5D=%E6%89%80%E5%B1%9E%E5%9F%8E%E5%B8%82&map-uq%5B%5D=uq109&map-text%5B%5D=&map-name%5B%5D=%E9%82%AE%E6%94%BF%E7%BC%96%E7%A0%81&map-uq%5B%5D=uq110&map-text%5B%5D=&map-name%5B%5D=%E9%97%A8%E7%89%8C%E5%8F%B7&map-uq%5B%5D=uq201&map-text%5B%5D=&map-name%5B%5D=%E9%82%AE%E5%AF%84%E5%9C%B0%E5%9D%801(%E5%AE%8C%E6%95%B4%E5%AF%BC%E5%87%BA)&map-uq%5B%5D=uq111&map-text%5B%5D=&map-name%5B%5D=Paypal+ID&map-uq%5B%5D=uq195&map-text%5B%5D=&map-name%5B%5D=%E7%89%A9%E6%B5%81%E6%B8%A0%E9%81%93&map-uq%5B%5D=uq128&map-text%5B%5D=&map-name%5B%5D=%E8%B4%A7%E8%BF%90%E5%8D%95%E5%8F%B7&map-uq%5B%5D=uq130&map-text%5B%5D=&map-name%5B%5D=%E5%86%85%E9%83%A8%E5%8D%95%E5%8F%B7&map-uq%5B%5D=uq160&map-text%5B%5D=&map-name%5B%5D=%E5%95%86%E5%93%81%E6%80%BB%E9%87%8D%E9%87%8F&map-uq%5B%5D=uq255&map-text%5B%5D=&map-name%5B%5D=%E5%8F%91%E8%B4%A7%E6%97%B6%E9%97%B4&map-uq%5B%5D=uq149&map-text%5B%5D=&map-name%5B%5D=%E8%AE%A2%E5%8D%95%E5%95%86%E5%93%81%E5%90%8D%E7%A7%B0&map-uq%5B%5D=uq120&map-text%5B%5D=&map-name%5B%5D=%E5%B9%B3%E5%8F%B0SKU&map-uq%5B%5D=uq196&map-text%5B%5D=&map-name%5B%5D=%E5%A4%87%E6%B3%A8&map-uq%5B%5D=uq113&map-text%5B%5D=&map-name%5B%5D=%E5%95%86%E5%93%81%E6%98%AF%E5%90%A6%E7%BC%BA%E8%B4%A7&map-uq%5B%5D=uq260&map-text%5B%5D=&map-name%5B%5D=%E8%87%AA%E5%AE%9A%E4%B9%89%E5%88%86%E7%B1%BB&map-uq%5B%5D=uq226&map-text%5B%5D=&map-name%5B%5D=%E4%BA%A4%E8%BF%90%E5%BC%82%E5%B8%B8%E5%8E%9F%E5%9B%A0&map-uq%5B%5D=uq216&map-text%5B%5D=&map-name%5B%5D=PayPal%E9%82%AE%E7%AE%B1&map-uq%5B%5D=uq141&map-text%5B%5D=&map-name%5B%5D=SKU&map-uq%5B%5D=uq119&map-text%5B%5D=&templateName=&templateId=42834&standardVersion=1&orderItemOrderBy=stockId+asc%2Cid+asc&pageSave=1&tableBase=&hbddgyxx=1'
headers ={
'Accept':'application/json, text/javascript, */*; q=0.01','Referer':'https://private-amz.mabangerp.com/index.php?mod=order.exportOrderByTemplate&isCloud=2&tableBase=&os=&orderItemOrderBy=stockId+asc%2Cid+asc&cMKey=0bac0855cad15bb5eb5fbf3117311c5e&platformOrderIdsMkey=EXPORT_ORDER_BY_TEMPLATE_INTERFACE_308469&platformOrderIdssqlMkey=EXPORT_ORDER_BY_TEMPLATE_SQL_INTERFACE_308469&lang=cn','Origin':'https://private-amz.mabangerp.com','User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36','Content-Type':'application/x-www-form-urlencoded; charset=UTF-8'}
url ="https://private-amz.mabangerp.com/index.php?mod=order.doExportByTemplateData"await post(url,headers,data,cookies)
tasks =[asyncio.ensure_future(request(li))for li in allrecords]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
end = time.time()print(end - start)# dlist.append(test())
step2(cookies)print(dlist)# start=time.time()# for li in allrecords:# url = "https://private-amz.mabangerp.com/index.php?mod=order.doExportByTemplateData"# payload = 'backUrl=&orderIds='+'%0D%0A'.join(li)+'&fieldlabel=uq101&fieldlabel=uq102&fieldlabel=uq113&fieldlabel=uq115&fieldlabel=uq128&fieldlabel=uq130&fieldlabel=uq135&fieldlabel=uq136&fieldlabel=uq141&fieldlabel=uq195&fieldlabel=uq149&fieldlabel=uq160&fieldlabel=uq216&fieldlabel=uq226&fieldlabel=uq104&fieldlabel=uq105&fieldlabel=uq107&fieldlabel=uq108&fieldlabel=uq109&fieldlabel=uq110&fieldlabel=uq111&fieldlabel=uq145&fieldlabel=uq201&fieldlabel=uq146&fieldlabel=uq250&fieldlabel=uq156&fieldlabel=uq255&fieldlabel=uq120&fieldlabel=uq196&fieldlabel=uq260&map-name%5B%5D=%E4%BB%98%E6%AC%BE%E6%97%B6%E9%97%B4&map-uq%5B%5D=uq115&map-text%5B%5D=&map-name%5B%5D=%E8%AE%A2%E5%8D%95%E7%BC%96%E5%8F%B7&map-uq%5B%5D=uq101&map-text%5B%5D=&map-name%5B%5D=%E5%BA%97%E9%93%BA%E5%90%8D&map-uq%5B%5D=uq135&map-text%5B%5D=&map-name%5B%5D=%E4%BA%A4%E6%98%93%E7%BC%96%E5%8F%B7&map-uq%5B%5D=uq102&map-text%5B%5D=&map-name%5B%5D=%E7%8A%B6%E6%80%81&map-uq%5B%5D=uq136&map-text%5B%5D=&map-name%5B%5D=%E8%AE%A2%E5%8D%95%E5%8E%9F%E5%A7%8B%E6%80%BB%E9%87%91%E9%A2%9D%EF%BC%88%E9%A6%96%E8%AE%B0%E5%BD%95%EF%BC%89&map-uq%5B%5D=uq250&map-text%5B%5D=&map-name%5B%5D=%E5%8E%9F%E5%A7%8B%E5%95%86%E5%93%81%E6%80%BB%E9%87%91%E9%A2%9D&map-uq%5B%5D=uq146&map-text%5B%5D=&map-name%5B%5D=%E5%B8%81%E7%A7%8D&map-uq%5B%5D=uq156&map-text%5B%5D=&map-name%5B%5D=%E5%AE%A2%E6%88%B7%E5%A7%93%E5%90%8D&map-uq%5B%5D=uq104&map-text%5B%5D=&map-name%5B%5D=%E7%94%B5%E8%AF%9D1&map-uq%5B%5D=uq105&map-text%5B%5D=&map-name%5B%5D=%E8%81%94%E7%B3%BB%E9%82%AE%E7%AE%B1&map-uq%5B%5D=uq145&map-text%5B%5D=&map-name%5B%5D=%E5%9B%BD%E5%AE%B6&map-uq%5B%5D=uq107&map-text%5B%5D=&map-name%5B%5D=%E6%89%80%E5%B1%9E%E5%9C%B0%E5%8C%BA&map-uq%5B%5D=uq108&map-text%5B%5D=&map-name%5B%5D=%E6%89%80%E5%B1%9E%E5%9F%8E%E5%B8%82&map-uq%5B%5D=uq109&map-text%5B%5D=&map-name%5B%5D=%E9%82%AE%E6%94%BF%E7%BC%96%E7%A0%81&map-uq%5B%5D=uq110&map-text%5B%5D=&map-name%5B%5D=%E9%97%A8%E7%89%8C%E5%8F%B7&map-uq%5B%5D=uq201&map-text%5B%5D=&map-name%5B%5D=%E9%82%AE%E5%AF%84%E5%9C%B0%E5%9D%801(%E5%AE%8C%E6%95%B4%E5%AF%BC%E5%87%BA)&map-uq%5B%5D=uq111&map-text%5B%5D=&map-name%5B%5D=Paypal+ID&map-uq%5B%5D=uq195&map-text%5B%5D=&map-name%5B%5D=%E7%89%A9%E6%B5%81%E6%B8%A0%E9%81%93&map-uq%5B%5D=uq128&map-text%5B%5D=&map-name%5B%5D=%E8%B4%A7%E8%BF%90%E5%8D%95%E5%8F%B7&map-uq%5B%5D=uq130&map-text%5B%5D=&map-name%5B%5D=%E5%86%85%E9%83%A8%E5%8D%95%E5%8F%B7&map-uq%5B%5D=uq160&map-text%5B%5D=&map-name%5B%5D=%E5%95%86%E5%93%81%E6%80%BB%E9%87%8D%E9%87%8F&map-uq%5B%5D=uq255&map-text%5B%5D=&map-name%5B%5D=%E5%8F%91%E8%B4%A7%E6%97%B6%E9%97%B4&map-uq%5B%5D=uq149&map-text%5B%5D=&map-name%5B%5D=%E8%AE%A2%E5%8D%95%E5%95%86%E5%93%81%E5%90%8D%E7%A7%B0&map-uq%5B%5D=uq120&map-text%5B%5D=&map-name%5B%5D=%E5%B9%B3%E5%8F%B0SKU&map-uq%5B%5D=uq196&map-text%5B%5D=&map-name%5B%5D=%E5%A4%87%E6%B3%A8&map-uq%5B%5D=uq113&map-text%5B%5D=&map-name%5B%5D=%E5%95%86%E5%93%81%E6%98%AF%E5%90%A6%E7%BC%BA%E8%B4%A7&map-uq%5B%5D=uq260&map-text%5B%5D=&map-name%5B%5D=%E8%87%AA%E5%AE%9A%E4%B9%89%E5%88%86%E7%B1%BB&map-uq%5B%5D=uq226&map-text%5B%5D=&map-name%5B%5D=%E4%BA%A4%E8%BF%90%E5%BC%82%E5%B8%B8%E5%8E%9F%E5%9B%A0&map-uq%5B%5D=uq216&map-text%5B%5D=&map-name%5B%5D=PayPal%E9%82%AE%E7%AE%B1&map-uq%5B%5D=uq141&map-text%5B%5D=&templateName=&templateId=42834&standardVersion=1&orderItemOrderBy=stockId+asc%2Cid+asc&pageSave=1&tableBase=&hbddgyxx=1'# headers = {
# 'Accept': 'application/json, text/javascript, */*; q=0.01',# 'Referer': 'https://private-amz.mabangerp.com/index.php?mod=order.exportOrderByTemplate&isCloud=2&tableBase=&os=&orderItemOrderBy=stockId+asc%2Cid+asc&cMKey=0bac0855cad15bb5eb5fbf3117311c5e&platformOrderIdsMkey=EXPORT_ORDER_BY_TEMPLATE_INTERFACE_308469&platformOrderIdssqlMkey=EXPORT_ORDER_BY_TEMPLATE_SQL_INTERFACE_308469&lang=cn',# 'Origin': 'https://private-amz.mabangerp.com',# 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36',# 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'# }# response = requests.request("POST", url, headers=headers, data = payload)# print(response.json()['gourl'])# dlist.append(response.json()['gourl'])# end = time.time()# print(end - start)
rl={
}defstep3():asyncdefget(url,headers):
conn = aiohttp.TCPConnector(limit_per_host=1)
session=aiohttp.ClientSession(headers=headers,connector=conn)asyncwith session.get(url)as resp:
con=await resp.read()
rl[url]= con
await session.close()asyncdefrequest(url):
headers ={
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'}await get(url,headers)
tasks =[asyncio.ensure_future(request(url))for url in dlist]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
step3()for k,v in rl.items():withopen('C:/Users/lx/Desktop/w/'+k[55:],"wb")as f:try:
f.write(v)except FileExistsError:pass# print(response.text)# url = "https://upload-private.mabangerp.com/stock/orderexport/159506163484242700297.xls"# headers = {
# 'Connection': 'keep-alive',# 'Upgrade-Insecure-Requests': '1',# 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36',# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',# 'Referer': 'https://private-amz.mabangerp.com/index.php?mod=order.exportOrderByTemplate&isCloud=2&tableBase=&os=&orderItemOrderBy=stockId+asc%2Cid+asc&cMKey=0bac0855cad15bb5eb5fbf3117311c5e&platformOrderIdsMkey=EXPORT_ORDER_BY_TEMPLATE_INTERFACE_308469&platformOrderIdssqlMkey=EXPORT_ORDER_BY_TEMPLATE_SQL_INTERFACE_308469&lang=cn',# 'Accept-Encoding': 'gzip, deflate, br',# 'Accept-Language': 'zh-CN,zh;q=0.9',# 'Cookie': 'CRAWL_KANDENG_KEY=yluGNNKDf8xXsv3YNEkT5FIOgl%2FrB42PstVnYIBIHUxJLFguCl98%2FE6Yop54DHesiyRMTt%2BTSOS3G9xxMyNhdg%3D%3D; MULTI_LANGUAGE_TYPE=%2BYjZ6oacL7xJ%2FKOcmBg9Z7cTOqi7UgOUgujRs4KQ4Ms%3D; CRAWL_KANDENG_KEY=yluGNNKDf8xXsv3YNEkT5FIOgl%2FrB42PstVnYIBIHUxJLFguCl98%2FE6Yop54DHesiyRMTt%2BTSOS3G9xxMyNhdg%3D%3D; MULTI_LANGUAGE_TYPE=%2BYjZ6oacL7xJ%2FKOcmBg9Z7cTOqi7UgOUgujRs4KQ4Ms%3D'# }# response = requests.request("GET", url, headers=headers, data = {})# response.encoding='UTF-8'# with open ('t.xls',"wb") as f:# try:# f.write(response.content)# except FileExistsError:# pass# print(response.text)
t2
import pandas as pd
import datetime
import re
import glob
import os
deftoint(a):try:
b=int(a)except ValueError as e:
b=Noneexcept TypeError as e:
b=Nonereturn b
deftodatetime(a):# a1=str(a)defti(c):try:
b=datetime.datetime.strptime(c,a)except TypeError as e:
b=c
except ValueError as e:
b=Nonereturn b
return ti
deftr2df(folder_path,usecols,r='',sheetname=None,distinct=True,subset=None,keep='last',converters=None):
p = re.compile(r)# all_d=[]
alpath =[i for i in glob.glob(os.path.join(folder_path,'*.*'))iflen(p.findall(os.path.basename(i)))>0]# alpath=sorted(alpath, key=lambda x:int(x[x.find('.')-8:x.find('.')]))# for fname in alpath:# all_d.append(fname)# all_f=all_d # if len(all_d)==0:# raise ValueError('所在文件夹找不到关键字标识的文件')# all_df=dict(zip(all_d,all_f))
col_name=usecols
adf=[]for k in alpath:if k[k.index('.')+1:].lower()=='csv'or k[k.index('.')+1:].lower()=='xlsx'or k[k.index('.')+1:].lower()=='xls':# print(1)try:if k[k.index('.')+1:].lower()=='csv':try:
df0=pd.read_csv(k,encoding='utf-8',usecols=usecols,low_memory=False)except:
df0=pd