python爬取网页数据(例如淘宝)

爬取网页数据(例如淘宝)

现在淘宝商品页面不能直接爬取，需要登录，所以我们得实现模拟登录，如下即可实现模拟登录：

import requests
cookie_str = r'cna=QsJDGKPtOQUCAXlFXn56tO/s; xlly_s=1; l=eBQUzrqIOlT7oVE9BOfZnurza77TIIRAguPzaNbMiOCPOO1p5qNdWZ7huN89CnGVhsNWR3u14VQUBeYBqImRv7aW0XW42kkmn;'
cookies = {}
for line in cookie_str.split(';'):
    key, value = line.split('=', 1)
    cookies[key] = value
#resp = requests.get(url, headers = headers, cookies = cookies) #cookies可以在发送请求时这样传过去

cookie来源：
打开淘宝登录页面登录然后打开调试器，点击network中的请求数据链接，查看cookies

爬取数据全部代码：re 模块使 Python 语言拥有全部的正则表达式功能

import re
import requests

url = 'https://s.taobao.com/search?q=%E8%AE%A1%E7%AE%97%E6%9C%BA%E7%BD%91%E7%BB%9C&imgfile=&js=1&stats_click=search_radio_all%3A1&initiative_id=staobaoz_20201126&ie=utf8'
payload = {'q': 'g_page_config','s': '1','ie':'utf8'}  #字典传递url参数
file = open('C:/Users/YHAA-1ED2A3/Desktop/taobao_product.txt','w',encoding='utf-8')
cookie_str = r'cna=QsJDGKPtOQUCAXlFXn56tO/s; xlly_s=1; l=eBQUzrqIOlT7oVE9BOfZnurza77TIIRAguPzaNbMiOCPOO1p5qNdWZ7huN89CnGVhsNWR3u14VQUBeYBqImRv7aW0XW42kkmn;'
cookies = {}
headers = {'User-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}
for line in cookie_str.split(';'):
    key, value = line.split('=', 1)
    cookies[key] = value
for k in range(0,1):        #1次，就是1个页的商品数据
    payload ['s'] = 44*k+1   #数量
    resp = requests.get(url, headers = headers, cookies = cookies)
    # print(resp.text)          #打印文本
    # print(resp.url)          #打印访问的网址
    resp.encoding = 'utf-8'  #设置编码
    # file.write(resp.text)

    title = re.findall(r'"raw_title":"([^"]+)"',resp.text,re.I)  #正则保存所有raw_title的内容，这个是名称
    price = re.findall(r'"view_price":"([^"]+)"',resp.text,re.I) #价格
    loc = re.findall(r'"item_loc":"([^"]+)"',resp.text,re.I) #地址
    x = len(title)           #每一页商品的数量
    
    for i in range(0,x) :    #把列表的数据保存到文件中
        file.write(str(k*44+i+1)+'名称：'+title[i]+'\n'+'价格：'+price[i]+'\n'+'地址：'+loc[i]+'\n\n')
        print(title[i])
file.close()