爬虫HDK并写入到excel文档中源码

最新推荐文章于 2023-03-30 14:37:44 发布

py编程

最新推荐文章于 2023-03-30 14:37:44 发布

阅读量327

点赞数

分类专栏： python

本文链接：https://blog.csdn.net/zfs1570/article/details/113259805

版权

python 专栏收录该内容

20 篇文章 0 订阅

订阅专栏

小程序成语猜题已开流量主欢迎体验

数据库表结构：

class hdkconf(models.Model):
hdk_linkuserid=models.IntegerField(null=True,blank=True)
hdk_userstatus=models.IntegerField(default=0) #0到期
hdk_endtime=models.DateTimeField(null=True,blank=True)

def __unicode__(self):
return self.hdk_linkuserid

#hdk任务表
class hdktask(models.Model):
hdktask_linkuserid=models.IntegerField(null=True,blank=True)
hdktask_status=models.IntegerField(default=0) #0暂停
def __unicode__(self):
return self.hdktask_linkuserid

视图层

def hdkhtml(request):
uname=request.session.get("username","")
user_exist=bkuser.objects.filter(bkuser_name=uname)
admin_exist=bkadmin.objects.filter(bkadmin_name=uname)
if user_exist:
userinfo=bkuser.objects.get(bkuser_name=uname)
hdkconfinfo=hdkconf.objects.filter(hdk_linkuserid=userinfo.id)
if hdkconfinfo:
#配置信息
hdkconfinfo=hdkconf.objects.get(hdk_linkuserid=userinfo.id)
hdktaskinfo=hdktask.objects.filter(hdktask_linkuserid=userinfo.id)
if hdktaskinfo:
#任务信息
hdktaskinfo=hdktask.objects.get(hdktask_linkuserid=userinfo.id)
return render(request,"hdkhtml.html",{"uname":uname,
"hdkconfinfo":hdkconfinfo,
"hdktaskinfo":hdktaskinfo
})
else:
return render(request,"hdkhtml.html",{"uname":uname,
"hdkconfinfo":hdkconfinfo
})

else:
return HttpResponse("您没有菜单权限")


elif admin_exist:
return redirect("/boss/")
else:
return redirect("/login/")

#下载hdk
@csrf_exempt
def downhdk(request):
uname=request.session.get("username","")
user_exist=bkuser.objects.filter(bkuser_name=uname)
admin_exist=bkadmin.objects.filter(bkadmin_name=uname)
if user_exist:
filename="media/hdk/"+str(uname)+".xlsx"#要下载的文件路径
with open(filename, 'rb') as model_excel:
result = model_excel.read()
response = HttpResponse(result)
response['Content-Disposition'] = 'attachment;filename="hdkdata.xlsx"'
return response
elif admin_exist:
return redirect("/boss/")
else:
return redirect("/login/")

#执行任务
@csrf_exempt
def runhdkdata(request):
uname=request.session.get("username","")
user_exist=bkuser.objects.filter(bkuser_name=uname)
admin_exist=bkadmin.objects.filter(bkadmin_name=uname)
if user_exist:
userinfo=bkuser.objects.get(bkuser_name=uname)
hdkconfinfo=hdkconf.objects.get(hdk_linkuserid=userinfo.id)
if request.method=="POST":
sortid=request.POST.get("sortid")
shopid=request.POST.get("shopid")
if hdkconfinfo.hdk_userstatus==0:
return HttpResponse('{"hdkcode":"timeend"}',content_type="application/json")
else:
#存入数据到数据库
taskinfo=hdktask.objects.get(hdktask_linkuserid=userinfo.id)
taskinfo.hdktask_status=1
taskinfo.save()
#线程执行
rhdk=threading.Thread(target=runhdktask, args=(uname,userinfo.id,sortid,shopid))
rhdk.start()
return HttpResponse('{"hdkcode":"success"}',content_type="application/json")
elif admin_exist:
return redirect("/boss/")
else:
return redirect("/login/")

对应的接口逻辑

def hdktest(url,userid):
try:
pcres=urllib.request.urlopen(url)
pcres=pcres.read().decode()
return pcres
#异常处理
except urllib.error.HTTPError as reason:
taskinfo=hdktask.objects.get(hdktask_linkuserid=userid)
taskinfo.hdktask_status=0
taskinfo.save(update_fields=["hdktask_status"])
print(reason)

#执行hdk任务
def runhdktask(uname,userid,sortid,shopid):
serurl="https://xxx.com/allitem/new_get_allitem_list?keyword=&shopsearch=&user_name=&brand_name=&cid="+str(sortid)+"&min_price=&max_price=&filtrate_type=&min_sale=&min_rate=&min_tkMoney=&min_avg=&min_coupon=&p=1&shoptype="+str(shopid)+"&discount=&favorites_count=&min_itemprice=&max_itemprice=&sort=1"
res=hdktest(serurl,userid)
pagenum=re.findall(r'"num_page":(\d+),',str(res))
hdkexcel=xlwt.Workbook()
worksheet=hdkexcel.add_sheet(uname,cell_overwrite_ok=True)
worksheet.write(0,0,"商品名称")
worksheet.write(0,1,"券后价格")
worksheet.write(0,2,"今日销量")
worksheet.write(0,3,"店铺")
# 遍历页数
for page in range(1,int(pagenum[0])+1):
pageurl="https://xxx.com/allitem/new_get_allitem_list?keyword=&shopsearch=&user_name=&brand_name=&cid="+str(sortid)+"&min_price=&max_price=&filtrate_type=&min_sale=&min_rate=&min_tkMoney=&min_avg=&min_coupon=&p="+str(page)+"&shoptype="+str(shopid)+"&discount=&favorites_count=&min_itemprice=&max_itemprice=&sort=1"
serres=hdktest(pageurl,userid)

serlist=re.findall(r'"itemendprice".+?},',str(serres))
#判断页面获取数据不为空
if len(serlist)!=0:
#遍历每个商品
for num in range(0,len(serlist)):
#商品标题
itemtitle=re.findall(r'"itemtitle":"(.+?)"',str(serlist[num]))
#获取券后价列表
itemendprice=re.findall(r'"itemendprice":(\d+[.]{0,1}[\d+]{0,2})',str(serlist[num]))
#今日销量
todaysale=re.findall(r'"todaysale":(\d+)',str(serlist[num]))
#获取店铺名
shopname=re.findall(r'"shopname":"(.+?)"',str(serlist[num]))
time.sleep(1)
worksheet.write(num+1+100*(page-1),0,str(itemtitle[0]))
worksheet.write(num+1+100*(page-1),1,str(itemendprice[0]))
worksheet.write(num+1+100*(page-1),2,str(todaysale[0]))
worksheet.write(num+1+100*(page-1),3,str(shopname[0]))
hdkpath=settings.MEDIA_ROOT+"/hdk/"
hdkexcel.save(hdkpath+"/"+uname+".xlsx")
taskinfo=hdktask.objects.get(hdktask_linkuserid=userid)
taskinfo.hdktask_status=0
taskinfo.save(update_fields=["hdktask_status"])

以及前端源码：

{% load customtag %}
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">

<meta name="description" content="">
<link rel="icon" type="image/png" href="../media/img/default/icon.png">
<title>HDK系统</title>

<link href="/static/css/bootstrap.min.css" rel="stylesheet">
<link href="/static/css/custom.css" rel="stylesheet">
</head>
<body>
<div id="topcss">
<div style="height:30px;"><div style="padding-top:10px;padding-left:10px;float:left;width:50%;">{{uname}},欢迎回来!</div>
<div style="padding-top:10px;padding-right:10px;float:right;width:50%;"><a href="/logout" style="float:right;">注销</a></div>
</div>
<div style="height:30px;padding-top:10px;padding-left:10px;">到期时间:<k style="color:red;">{{hdkconfinfo.hdk_endtime|date:"Y-m-d H:i:s"}}</k></div>

<div style="height:30px;padding-top:10px;padding-left:10px;">状态: <k style="font-size:13px;">
{% if hdktaskinfo.hdktask_status == 1 %}
<d style="color:#33FF00;">(执行中)</d>
{% else %}
<d style="color:red;">(未执行)</d>
{% endif %}
</k></div>

<div style="height:30px;padding-top:10px;padding-left:10px;">分类:  <input id="0" type="radio" name="hdksort" checked />全部  <input id="1" type="radio" name="hdksort"/>女装  <input id="2" type="radio" name="hdksort"/>男装  <input id="3" type="radio" name="hdksort"/>内衣  <input id="4" type="radio" name="hdksort"/>美妆</div>
<div style="height:30px;padding-top:10px;padding-left:10px;">          <input id="5" type="radio" name="hdksort"/>饰品  <input id="6" type="radio" name="hdksort" />鞋品  <input id="7" type="radio" name="hdksort"/>箱包  <input id="8" type="radio" name="hdksort"/>儿童  <input id="9" type="radio" name="hdksort"/>母婴</div>
<div style="height:30px;padding-top:10px;padding-left:10px;">          <input id="10" type="radio" name="hdksort"/>居家  <input id="11" type="radio" name="hdksort" />美食  <input id="12" type="radio" name="hdksort"/>数码  <input id="13" type="radio" name="hdksort"/>家电  <input id="14" type="radio" name="hdksort"/>其他</div>
<div style="height:30px;padding-top:10px;padding-left:10px;">          <input id="15" type="radio" name="hdksort"/>车品  <input id="16" type="radio" name="hdksort" />文体  <input id="17" type="radio" name="hdksort"/>宠物  </div>

<div style="height:10px;margin-left:10px;width:100%;font-size:13px;"></div>

<div style="height:30px;padding-top:10px;padding-left:10px;">店铺:  <input id="1" type="radio" name="hdkshop" checked />天猫  <input id="2" type="radio" name="hdkshop"/>官网旗舰店  <input id="3" type="radio" name="hdkshop"/>阿里健康大药房  <input id="4" type="radio" name="hdkshop"/>天猫超市</div>
<div style="height:30px;padding-top:10px;padding-left:10px;">          <input id="7" type="radio" name="hdkshop" />极有家  <input id="8" type="radio" name="hdkshop"/>ifashion  <input id="9" type="radio" name="hdkshop"/>天猫国际  <input id="10" type="radio" name="hdkshop"/>全球购</div>
<div style="height:30px;padding-top:10px;padding-left:10px;">          <input id="11" type="radio" name="hdkshop" />品牌直销  <input id="12" type="radio" name="hdkshop"/>金牌卖家  <input id="13" type="radio" name="hdkshop"/>企业店铺</div>
<div style="height:10px;margin-left:10px;width:100%;font-size:13px;"></div>
<div style="height:15px;width:100%;"></div>
{% if hdktaskinfo.hdktask_status == 1 %}
<span style="margin-left:10px;height:30px;width:40px;font-size:13px;"><a>执行中...请稍后下载查看</a></span>
{% else %}
<span style="margin-left:10px;height:30px;width:40px;border:1px solid red;border-radius:10%;cursor:pointer;" οnclick="runhdk()"><a>执行</a></span>
{% endif %}
<div style="height:25px;width:100%;"></div>
<div style="margin-left:10px;height:30px;width:auto">最新成功的excel请点击<a href="{% url 'downhdk' %}">这里</a>下载</div>
</div>
<script src="/static/js/jquery.min.js"></script>
<script src="/static/js/bootstrap.min.js"></script>
<script src="/static/js/custom.js"></script>
<script src="/static/js/layer/layer.js"></script>
</body>
</html>