python3爬淘宝一个店铺内的所有商品'收藏','名字','评论数','交易成功数','价格'

置顶 于 2019-10-25 23:02:15 发布
阅读量1.8k 收藏 12
点赞数 1
分类专栏： 爬虫 文章标签： python
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/pachongpapa/article/details/102751189
版权

最近找工作遇到一个公司给的面试的题，写一个脚本，爬取一个店铺所有商品的淘宝价，累计评论，交易成功数，宝贝收藏人气
在这里插入图片描述
因为考虑只是一个简单demo对下载速度并没有要求所有也没有使用ip池，
脚本执行时打印的爬取到数据处理成一个列表
在这里插入图片描述
把爬到的数据存入了csv
在这里插入图片描述
请求函数代码发出来吧。
因为只用半天以前也没爬过，所以些许有些潦草

# 打开店铺主页获取每个商品的id
def get_goodsId(page):
    ''''''
    url = "https://lp930428.taobao.com/i/asynSearch.htm"
#https://lp930428.taobao.com/search.htm?spm=a1z10.1-c-s.w5002-21751133133.1.6757669drR9RHI&search=y
    querystring = {"_ksTS": "1571966776769_186", "mid": "w-21751133153-0", "wid": "21751133153", "path": "/search.htm",
                   "search": "y", "spm": "a1z10.1-c-s.w5002-21751133133.1.6757669drR9RHI", "orderType": "newOn_desc",
                   "pageNo": page}

    headers = {
        'Host': "lp930428.taobao.com",
        'Cookie': "t=e3ced4867bbbb33ad8e43a6664f1b440; thw=cn; cookie2=1ae89e1b7eb4f38050c2a8484ca7f9e3; _tb_token_=fe3fdd7e7787e; enc=FCtEbwsd4d4qFaGEEU1%2Bh6zQqkT31zH7ZjTng%2ByjgFV1IKn98igFAnRQ80yP1MGcCWXDBuTJBB%2Fv2BN0HrnsFw%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; cna=cfrcFZLIz0UCAXxBtKpIjGyJ; v=0; unb=3342625761; uc3=vt3=F8dByucj%2BPu98CEsPjg%3D&lg2=VFC%2FuZ9ayeYq2g%3D%3D&id2=UNN5E%2BCNxCTmeQ%3D%3D&nk2=symzrp14SFvM8w%3D%3D; csg=e158c6a6; lgc=%5Cu5C0F%5Cu9F99cxcgao; cookie17=UNN5E%2BCNxCTmeQ%3D%3D; dnk=%5Cu5C0F%5Cu9F99cxcgao; skt=5acee91bdd655b99; existShop=MTU3MTk4NzgwNA%3D%3D; uc4=id4=0%40UgQxlJz5L5Tdls4BK%2BsdqbH4MLk8&nk4=0%40sVYTNakktI2J71HcZK%2B0xxDC%2BDng; tracknick=%5Cu5C0F%5Cu9F99cxcgao; _cc_=W5iHLLyFfA%3D%3D; tg=0; _l_g_=Ug%3D%3D; sg=o1f; _nk_=%5Cu5C0F%5Cu9F99cxcgao; cookie1=B0OsmmuZ3XRBlfT4RmnPR0sbhg2pIYLBZ6M6p3hMTnE%3D; uc1=cookie16=U%2BGCWk%2F74Mx5tgzv3dWpnhjPaQ%3D%3D&cookie21=UtASsssmeW6lpyd%2BB%2B3t&cookie15=Vq8l%2BKCLz3%2F65A%3D%3D&existShop=false&pas=0&cookie14=UoTbnKo8UDBHXA%3D%3D&tag=8&lng=zh_CN; mt=ci=0_1; l=dBgSXBauqGXlvNsDBOCwVuI8YdQOAIRAguPRwhApi_5B9_T6fvbOkZkD2E96cjWftzTB4DlUQKv9-etXiDt6Qtk8sxAJvxDc.; isg=BNLSgblL29-ZPicjbaEizx2QI550S9VMW8fzYZwrzgVwr3KphHIYjcTNG0s2304V",
        'accept': "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01",
        'x-requested-with': "XMLHttpRequest",
        'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.2 Safari/605.1.15",
        'sec-fetch-mode': "cors",
        'sec-fetch-site': "same-origin",
        'referer': "https://lp930428.taobao.com/search.htm?spm=a1z10.3-c-s.w4002-21751133153.93.1d6d1c25zjRRJk&_ksTS=1571991134154_186&callback=jsonp187&mid=w-21751133153-0&wid=21751133153&path=%2Fsearch.htm&search=y&pageNo=2",
        'accept-language': "zh-CN,zh;q=0.9",
        'cache-control': "no-cache",
        # 'Postman-Token': "798702b1-342e-4e2a-8176-c2cc057b0351"
    }

    requests.packages.urllib3.disable_warnings()
    response = requests.request("GET", url, headers=headers, params=querystring, verify=False)

    res = response.text
    # print(res)
    json_data = re.findall('itemIds=(.*?)&source', res,re.S) #解析获取到商品ID
    # print(json_data)
    return json_data

# 根据索取到的商品ID进入到商品详情页
def get_goodsDetails(i):
    url = "https://item.taobao.com/item.htm?spm=a1z10.3-c-s.w4002-21751133153.37.66ab1c25jaTgVP&id={}".format(i)

    headers = {
        'Host': "item.taobao.com",
        'Cookie': "t=e3ced4867bbbb33ad8e43a6664f1b440; thw=cn; cookie2=1ae89e1b7eb4f38050c2a8484ca7f9e3; _tb_token_=fe3fdd7e7787e; enc=FCtEbwsd4d4qFaGEEU1%2Bh6zQqkT31zH7ZjTng%2ByjgFV1IKn98igFAnRQ80yP1MGcCWXDBuTJBB%2Fv2BN0HrnsFw%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; cna=cfrcFZLIz0UCAXxBtKpIjGyJ; v=0; unb=3342625761; uc3=vt3=F8dByucj%2BPu98CEsPjg%3D&lg2=VFC%2FuZ9ayeYq2g%3D%3D&id2=UNN5E%2BCNxCTmeQ%3D%3D&nk2=symzrp14SFvM8w%3D%3D; csg=e158c6a6; lgc=%5Cu5C0F%5Cu9F99cxcgao; cookie17=UNN5E%2BCNxCTmeQ%3D%3D; dnk=%5Cu5C0F%5Cu9F99cxcgao; skt=5acee91bdd655b99; existShop=MTU3MTk4NzgwNA%3D%3D; uc4=id4=0%40UgQxlJz5L5Tdls4BK%2BsdqbH4MLk8&nk4=0%40sVYTNakktI2J71HcZK%2B0xxDC%2BDng; tracknick=%5Cu5C0F%5Cu9F99cxcgao; _cc_=W5iHLLyFfA%3D%3D; tg=0; _l_g_=Ug%3D%3D; sg=o1f; _nk_=%5Cu5C0F%5Cu9F99cxcgao; cookie1=B0OsmmuZ3XRBlfT4RmnPR0sbhg2pIYLBZ6M6p3hMTnE%3D; uc1=cookie16=U%2BGCWk%2F74Mx5tgzv3dWpnhjPaQ%3D%3D&cookie21=UtASsssmeW6lpyd%2BB%2B3t&cookie15=Vq8l%2BKCLz3%2F65A%3D%3D&existShop=false&pas=0&cookie14=UoTbnKo8UDBHXA%3D%3D&tag=8&lng=zh_CN; mt=ci=0_1; l=dBgSXBauqGXlvNsDBOCwVuI8YdQOAIRAguPRwhApi_5B9_T6fvbOkZkD2E96cjWftzTB4DlUQKv9-etXiDt6Qtk8sxAJvxDc.; isg=BNLSgblL29-ZPicjbaEizx2QI550S9VMW8fzYZwrzgVwr3KphHIYjcTNG0s2304V",
        'upgrade-insecure-requests': "1",
        'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36",
        'sec-fetch-mode': "navigate",
        'sec-fetch-user': "?1",
        'accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
        'sec-fetch-site': "same-site",
        'referer': "https://lp930428.taobao.com/search.htm?spm=a1z10.1-c-s.0.0.21c2669dXxI29C&search=y",
        'accept-language': "zh-CN,zh;q=0.9",
        'cache-control': "no-cache",
        'Postman-Token': "7c7dc56c-6a76-425d-8f60-98609da11052"
    }

    requests.packages.urllib3.disable_warnings()
    response = requests.request("GET", url, headers=headers, verify=False)
    # print(response)

    res = response.text
    # print(res)

    resp = re.findall('class="tb-main-title".*?data-title="(.*?)"', res)
    count = re.findall("sign=(.*?)&", res)
    '''获取的是商品名称，和要抓取宝贝人气的链接'''
    return resp, count


#  获取累计评论数

def get_comments(i):
    url = "https://rate.taobao.com/detailCount.do?_ksTS=1571993971073_117&callback=jsonp118&itemId={}".format(i)

    headers = {
        'Host': "rate.taobao.com",
        'Cookie': "t=e3ced4867bbbb33ad8e43a6664f1b440; thw=cn; cookie2=1ae89e1b7eb4f38050c2a8484ca7f9e3; _tb_token_=fe3fdd7e7787e; enc=FCtEbwsd4d4qFaGEEU1%2Bh6zQqkT31zH7ZjTng%2ByjgFV1IKn98igFAnRQ80yP1MGcCWXDBuTJBB%2Fv2BN0HrnsFw%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; cna=cfrcFZLIz0UCAXxBtKpIjGyJ; v=0; unb=3342625761; uc3=vt3=F8dByucj%2BPu98CEsPjg%3D&lg2=VFC%2FuZ9ayeYq2g%3D%3D&id2=UNN5E%2BCNxCTmeQ%3D%3D&nk2=symzrp14SFvM8w%3D%3D; csg=e158c6a6; lgc=%5Cu5C0F%5Cu9F99cxcgao; cookie17=UNN5E%2BCNxCTmeQ%3D%3D; dnk=%5Cu5C0F%5Cu9F99cxcgao; skt=5acee91bdd655b99; existShop=MTU3MTk4NzgwNA%3D%3D; uc4=id4=0%40UgQxlJz5L5Tdls4BK%2BsdqbH4MLk8&nk4=0%40sVYTNakktI2J71HcZK%2B0xxDC%2BDng; tracknick=%5Cu5C0F%5Cu9F99cxcgao; _cc_=W5iHLLyFfA%3D%3D; tg=0; _l_g_=Ug%3D%3D; sg=o1f; _nk_=%5Cu5C0F%5Cu9F99cxcgao; cookie1=B0OsmmuZ3XRBlfT4RmnPR0sbhg2pIYLBZ6M6p3hMTnE%3D; uc1=cookie16=U%2BGCWk%2F74Mx5tgzv3dWpnhjPaQ%3D%3D&cookie21=UtASsssmeW6lpyd%2BB%2B3t&cookie15=Vq8l%2BKCLz3%2F65A%3D%3D&existShop=false&pas=0&cookie14=UoTbnKo8UDBHXA%3D%3D&tag=8&lng=zh_CN; mt=ci=0_1; l=dBgSXBauqGXlvNsDBOCwVuI8YdQOAIRAguPRwhApi_5B9_T6fvbOkZkD2E96cjWftzTB4DlUQKv9-etXiDt6Qtk8sxAJvxDc.; isg=BNLSgblL29-ZPicjbaEizx2QI550S9VMW8fzYZwrzgVwr3KphHIYjcTNG0s2304V",
        'sec-fetch-mode': "no-cors",
        'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36",
        'accept': "*/*",
        'sec-fetch-site': "same-site",
        'referer': "https://item.taobao.com/item.htm?spm=a1z10.3-c-s.w4002-21751133153.48.10971c25BMetQJ&id={}".format(i),
        'accept-language': "zh-CN,zh;q=0.9",
        'cache-control': "no-cache"
        # 'Postman-Token': "0e7548b8-43dd-46e0-9467-a1310ebbc0d9"
    }

    requests.packages.urllib3.disable_warnings()
    response = requests.request("GET", url, headers=headers, verify=False)
    res = response.text
    count = re.findall('count.*?(\d+)', res)
    return count

# 获取淘宝价以及交易成功
def get_pice(i):
    url = "https://detailskip.taobao.com/service/getData/1/p1/item/detail/sib.htm"

    querystring = {"itemId": i, "sellerId": "1091756444",
                   "modules": "dynStock,qrcode,viewer,price,duty,xmpPromotion,delivery,activity,fqg,zjys,couponActivity,soldQuantity,page,originalPrice,tradeContract"}

    headers = {
        'Sec-Fetch-Mode': "no-cors",
        'Cookie': "t=e3ced4867bbbb33ad8e43a6664f1b440; thw=cn; cookie2=1ae89e1b7eb4f38050c2a8484ca7f9e3; _tb_token_=fe3fdd7e7787e; enc=FCtEbwsd4d4qFaGEEU1%2Bh6zQqkT31zH7ZjTng%2ByjgFV1IKn98igFAnRQ80yP1MGcCWXDBuTJBB%2Fv2BN0HrnsFw%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; cna=cfrcFZLIz0UCAXxBtKpIjGyJ; v=0; unb=3342625761; uc3=vt3=F8dByucj%2BPu98CEsPjg%3D&lg2=VFC%2FuZ9ayeYq2g%3D%3D&id2=UNN5E%2BCNxCTmeQ%3D%3D&nk2=symzrp14SFvM8w%3D%3D; csg=e158c6a6; lgc=%5Cu5C0F%5Cu9F99cxcgao; cookie17=UNN5E%2BCNxCTmeQ%3D%3D; dnk=%5Cu5C0F%5Cu9F99cxcgao; skt=5acee91bdd655b99; existShop=MTU3MTk4NzgwNA%3D%3D; uc4=id4=0%40UgQxlJz5L5Tdls4BK%2BsdqbH4MLk8&nk4=0%40sVYTNakktI2J71HcZK%2B0xxDC%2BDng; tracknick=%5Cu5C0F%5Cu9F99cxcgao; _cc_=W5iHLLyFfA%3D%3D; tg=0; _l_g_=Ug%3D%3D; sg=o1f; _nk_=%5Cu5C0F%5Cu9F99cxcgao; cookie1=B0OsmmuZ3XRBlfT4RmnPR0sbhg2pIYLBZ6M6p3hMTnE%3D; uc1=cookie16=U%2BGCWk%2F74Mx5tgzv3dWpnhjPaQ%3D%3D&cookie21=UtASsssmeW6lpyd%2BB%2B3t&cookie15=Vq8l%2BKCLz3%2F65A%3D%3D&existShop=false&pas=0&cookie14=UoTbnKo8UDBHXA%3D%3D&tag=8&lng=zh_CN; mt=ci=0_1; l=dBgSXBauqGXlvNsDBOCwVuI8YdQOAIRAguPRwhApi_5B9_T6fvbOkZkD2E96cjWftzTB4DlUQKv9-etXiDt6Qtk8sxAJvxDc.; isg=BNLSgblL29-ZPicjbaEizx2QI550S9VMW8fzYZwrzgVwr3KphHIYjcTNG0s2304V",
        'accept': "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01",

        'Referer': "https://item.taobao.com/item.htm?spm=a1z10.3-c-s.w4002-21751133153.33.7ffe1c25zHNgcL&id={}".format(i),
        'User-Agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36",
        'cache-control': "no-cache"
    }

    requests.packages.urllib3.disable_warnings()
    response = requests.request("GET", url, headers=headers, params=querystring, verify=False)

    res = response.text
    # print(res)
    datadict = json.loads(res)
    # print(str)
    '''现获取交易成功数'''
    trading = datadict['data']['soldQuantity']['confirmGoodsCount'] # 交易成功的数
    pice = datadict['data']['promotion']['promoData']['def'][0]['price']

    return trading, pice

# 宝贝收藏
def get_collection(i,sing):
    url = "https://count.taobao.com/counter3?_ksTS=1571993971066_103&callback=jsonp104&inc=ICVT_7_{}&sign={}&keys=DFX_200_1_{},ICVT_7_{},ICCP_1_{},SCCP_2_101463431".format(i, sing, i, i, i)

    headers = {
        'Host': "count.taobao.com",
        'Cookie': "t=e3ced4867bbbb33ad8e43a6664f1b440; thw=cn; cookie2=1ae89e1b7eb4f38050c2a8484ca7f9e3; _tb_token_=fe3fdd7e7787e; enc=FCtEbwsd4d4qFaGEEU1%2Bh6zQqkT31zH7ZjTng%2ByjgFV1IKn98igFAnRQ80yP1MGcCWXDBuTJBB%2Fv2BN0HrnsFw%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; cna=cfrcFZLIz0UCAXxBtKpIjGyJ; v=0; unb=3342625761; uc3=vt3=F8dByucj%2BPu98CEsPjg%3D&lg2=VFC%2FuZ9ayeYq2g%3D%3D&id2=UNN5E%2BCNxCTmeQ%3D%3D&nk2=symzrp14SFvM8w%3D%3D; csg=e158c6a6; lgc=%5Cu5C0F%5Cu9F99cxcgao; cookie17=UNN5E%2BCNxCTmeQ%3D%3D; dnk=%5Cu5C0F%5Cu9F99cxcgao; skt=5acee91bdd655b99; existShop=MTU3MTk4NzgwNA%3D%3D; uc4=id4=0%40UgQxlJz5L5Tdls4BK%2BsdqbH4MLk8&nk4=0%40sVYTNakktI2J71HcZK%2B0xxDC%2BDng; tracknick=%5Cu5C0F%5Cu9F99cxcgao; _cc_=W5iHLLyFfA%3D%3D; tg=0; _l_g_=Ug%3D%3D; sg=o1f; _nk_=%5Cu5C0F%5Cu9F99cxcgao; cookie1=B0OsmmuZ3XRBlfT4RmnPR0sbhg2pIYLBZ6M6p3hMTnE%3D; uc1=cookie16=U%2BGCWk%2F74Mx5tgzv3dWpnhjPaQ%3D%3D&cookie21=UtASsssmeW6lpyd%2BB%2B3t&cookie15=Vq8l%2BKCLz3%2F65A%3D%3D&existShop=false&pas=0&cookie14=UoTbnKo8UDBHXA%3D%3D&tag=8&lng=zh_CN; mt=ci=0_1; _m_h5_tk=f1425a910c31b9877a2b467419fa4bfc_1572002857779; _m_h5_tk_enc=25325b15bf3ab20673732ed53229f569; l=dBx1kOlIq1-Yna4UBOCidweULX7OSIRAguWK8C09i_5CW6Ls_U7OkZlM8Fp6VjWfGeTB4DlUQKv9-etXmxeT62k8sxAJwxDc.; isg=BJiYNvaHccm3pl2NmLBHZTCVacCDfP38DcXpT9KJ5FOGbThXepHMm65LoeV4_bTj",
        'sec-fetch-mode': "no-cors",
        'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36",
        'accept': "*/*",
        'sec-fetch-site': "same-site",
        'referer': "https://item.taobao.com/item.htm?spm=a1z10.3-c-s.w4002-21751133153.39.56a81c25HPoZfb&id={}".format(i),
        'accept-language': "zh-CN,zh;q=0.9",
        'cache-control': "no-cache",
        'Postman-Token': "1420a7da-b2b9-4249-829b-edddeb105432"
    }

    requests.packages.urllib3.disable_warnings()
    response = requests.request("GET", url, headers=headers, verify=False)

    res = response.text
    count = re.findall('\((.*?)\)', res)[0]
    resp = eval(count)["ICCP_1_{}".format(i)]
    return resp

潦草是潦草了些，希望对大家有所帮助

不会python.
关注
  • 1
    点赞
  • 踩
  • 12
    收藏
    觉得还不错? 一键收藏
  • 4
    评论
专栏目录
评论 4
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值