7.1.4 Selenium 爬取京东商品信息实战

学编程的菜恐龙

已于 2024-04-28 14:50:17 修改

阅读量1.4k

点赞数 25

分类专栏： Python3 网络爬虫开发实战学习文章标签： selenium 测试工具

于 2024-04-05 19:31:28 首次发布

本文链接：https://blog.csdn.net/weixin_51100340/article/details/137331334

版权

Python3 网络爬虫开发实战学习专栏收录该内容

22 篇文章 0 订阅

订阅专栏

1、实战内容

爬取京东笔记本电脑商品的信息(如：价格、商品名、评论数量)，存入 MySQL 中

2、思路

京东需要登录才能搜索进入，所以首先从登录页面出发，自己登录，然后等待，得到 Cookies，之后带着 Cookies 模拟访问页面，得到页面的 response，再对 response 进行分析，将分析内容存入 MySQL 中。

3、分析 url

来到京东搜索笔记本电脑，之后观察首页之后的几页的链接，如下：

不难发现，除了首页之外，其他链接的区别是从第四行的 page 开始，大胆猜测链接就到此即可，且规律也显而易见，就可以直接构造链接。

4、开始操作

1、得到 Cookies

from selenium import webdriver
import json, time

def get_cookie(url):
    browser = webdriver.Chrome()
    browser.get(url)    # 进入登录页面
    time.sleep(10)      # 停一段时间，自己手动登录
    cookies_dict_list = browser.get_cookies()  # 获取list的cookies
    cookies_json = json.dumps(cookies_dict_list)  # 列表转换成字符串保存
    with open('cookies.txt', 'w') as f:
        f.write(cookies_json)
    print('cookies保存成功！')
    browser.close()


if __name__ == '__main__':
    url_login = 'https://passport.jd.com/uc/login?'     # 登录页面url
    get_cookie(url_login)

之后就可看到 txt 文件中的 cookies(多个)，之后模拟登录构造 cookie 时，对照其中的即可.

[{"domain": ".jd.com", "expiry": 1714467009, "httpOnly": false, "name": "PCSYCityID", "path": "/", "sameSite": "Lax", "secure": false, "value": "CN_420000_420100_0"}, {"domain": ".jd.com", "expiry": 1714726209, "httpOnly": false, "name": "ipLoc-djd", "path": "/", "sameSite": "Lax", "secure": false, "value": "17-1381-0-0"}, {"domain": ".jd.com", "expiry": 1744966205, "httpOnly": false, "name": "3AB9D23F7A4B3CSS", "path": "/", "sameSite": "Lax", "secure": false, "value": "jdd037TJJKUMONRNQCO62PWGC7ODM7JROGOIDPQU2BCCFMUCIQO2I5VAQVW5O4SGRCXDYVRRXXKOX4UHIGKDZ3PKGOR4N34AAAAMPBIS6BOYAAAAACW2RUF62NZ4QJ4X"}, {"domain": ".jd.com", "expiry": 1714726209, "httpOnly": false, "name": "areaId", "path": "/", "sameSite": "Lax", "secure": false, "value": "17"}, {"domain": ".jd.com", "expiry": 1713864005, "httpOnly": false, "name": "__jdb", "path": "/", "sameSite": "Lax", "secure": false, "value": "76161171.2.1713862194482764265847|1.1713862194"}, {"domain": ".jd.com", "httpOnly": false, "name": "__jdc", "path": "/", "sameSite": "Lax", "secure": false, "value": "76161171"}, {"domain": ".jd.com", "httpOnly": true, "name": "pinId", "path": "/", "sameSite": "Lax", "secure": false, "value": "r20XJXxEiQzXTfkF3qkirrV9-x-f3wj7"}, {"domain": ".jd.com", "expiry": 1716454204, "httpOnly": true, "name": "_tp", "path": "/", "sameSite": "Lax", "secure": false, "value": "wIxNS9FTQdjLJ59X04W8g6fl%2FjckyFa3Feur52OoG9Y%3D"}, {"domain": ".jd.com", "expiry": 1713862325, "httpOnly": false, "name": "_gia_d", "path": "/", "sameSite": "Lax", "secure": false, "value": "1"}, {"domain": ".jd.com", "expiry": 1716454204, "httpOnly": false, "name": "npin", "path": "/", "sameSite": "Lax", "secure": false, "value": "jd_5d8dd3c5fae0c"}, {"domain": ".jd.com", "httpOnly": true, "name": "flash", "path": "/", "sameSite": "None", "secure": true, "value": "2_1tx2gsoqN74RnTuh5J9_hiyfyDDRKj1lOWvBR9jOelV6TaWYrKHzreFsBwGmAmxKUfKqsKFul1zUaa7ejv-yFs5gGqTSIBZ1nyTld2SLlYj*"}, {"domain": ".jd.com", "httpOnly": true, "name": "thor", "path": "/", "sameSite": "None", "secure": true, "value": "822A26DCAF32B0E7A92FFFBC434794B69743B9393BD9418B46A25947C5DDE6C8E11F6B24FC8180A2D134A279B137ACE831B4042213D12DA43DA5286659940540769A4DACFA4AEE4920A41DA34717F4AB6EBCB6C39F80DD4C370630DD00CC3CDD599FFCD2C528A732610F316B83AFBE0E4564A746BC571F7572B8B53A8200B6B807FE0E168428C065E1A5AE60A040942FC62FAEBA43345764A371D23DA468D1A8"}, {"domain": ".jd.com", "expiry": 1716454204, "httpOnly": false, "name": "pin", "path": "/", "sameSite": "Lax", "secure": false, "value": "jd_5d8dd3c5fae0c"}, {"domain": ".jd.com", "expiry": 1715158194, "httpOnly": false, "name": "__jdv", "path": "/", "sameSite": "Lax", "secure": false, "value": "95931165|direct|-|none|-|1713862194483"}, {"domain": ".jd.com", "expiry": 1716454204, "httpOnly": false, "name": "logintype", "path": "/", "sameSite": "Lax", "secure": false, "value": "qq"}, {"domain": "www.jd.com", "expiry": 1745398205, "httpOnly": false, "name": "o2State", "path": "/", "sameSite": "Lax", "secure": false, "value": "{%22webp%22:true%2C%22avif%22:true}"}, {"domain": ".jd.com", "expiry": 1729414205, "httpOnly": false, "name": "__jda", "path": "/", "sameSite": "Lax", "secure": false, "value": "76161171.1713862194482764265847.1713862194.1713862194.1713862194.1"}, {"domain": ".jd.com", "expiry": 1716454204, "httpOnly": true, "name": "_pst", "path": "/", "sameSite": "Lax", "secure": false, "value": "jd_5d8dd3c5fae0c"}, {"domain": ".jd.com", "expiry": 1744966205, "httpOnly": false, "name": "3AB9D23F7A4B3C9B", "path": "/", "sameSite": "Lax", "secure": false, "value": "7TJJKUMONRNQCO62PWGC7ODM7JROGOIDPQU2BCCFMUCIQO2I5VAQVW5O4SGRCXDYVRRXXKOX4UHIGKDZ3PKGOR4N34"}, {"domain": ".jd.com", "expiry": 1729414213, "httpOnly": false, "name": "__jdu", "path": "/", "sameSite": "Lax", "secure": false, "value": "1713862194482764265847"}, {"domain": ".jd.com", "expiry": 1716454204, "httpOnly": false, "name": "unick", "path": "/", "sameSite": "Lax", "secure": false, "value": "jxnu-2019"}, {"domain": ".jd.com", "httpOnly": false, "name": "wlfstk_smdl", "path": "/", "sameSite": "Lax", "secure": false, "value": "vfk8t4rwomo4ptggg0jbjy5e7h4zf18b"}]

2、访问页面，得到 response

当 selenium 访问时，若页面是笔记本电脑页，直接进行爬取；若不是，即在登录页，构造 cookie，添加，登录；

js 语句是使页面滑倒最低端，等待数据包刷新出来，才能得到完整的页面 response(这里的 resposne 相当于使用 request 时候的 response.text );

def get_html(url):    # 此 url 为分析的笔记本电脑页面的 url
    browser = webdriver.Chrome()
    browser.get(url)
    if browser.current_url == url:
        js = "var q=document.documentElement.scrollTop=100000"
        browser.execute_script(js)
        time.sleep(2)
        responses = browser.page_source
        browser.close()
        return responses
    else:
        with open('cookies.txt', 'r', encoding='utf8') as f:
            listCookies = json.loads(f.read())
        for cookie in listCookies:
            cookie_dict = {
                'domain': '.jd.com',
                'name': cookie.get('name'),
                'value': cookie.get('value'),
                "expires": '1746673777',    # 表示 cookies到期时间
                'path': '/',            # expiry必须为整数，所以使用 expires
                'httpOnly': False,
                "sameSite": "Lax",
                'Secure': False
            }
            browser.add_cookie(cookie_dict) # 将获得的所有 cookies 都加入
        time.sleep(1)
        browser.get(url)
        js = "var q=document.documentElement.scrollTop=100000"
        browser.execute_script(js)
        time.sleep(2)
        response = browser.page_source
        browser.close()
        return response

cookies 到期时间是用时间戳来记录的，可用工具查看时间戳(Unix timestamp)转换工具 - 在线工具

3、解析页面

这里我使用了 Xpath 的解析方法，并将结果构造成字典列表，输出。

def parse(response):
    html = etree.HTML(response)
    products_lis = html.xpath('//li[contains(@class, "gl-item")]')
    products_list = []
    for product_li in products_lis:
        product = {}
        price = product_li.xpath('./div/div[@class="p-price"]/strong/i/text()')
        name = product_li.xpath('./div/div[contains(@class, "p-name")]/a/em/text()')
        comment = product_li.xpath('./div/div[@class="p-commit"]/strong/a/text()')

        product['price'] = price[0]
        product['name'] = name[0].strip()
        product['comment'] = comment[0]
        products_list.append(product)
    print(products_list)
    print(len(products_list))

4、存入 MySQL

上述已经得到每一个商品数据的字典，所以直接对字典或字典列表进行存储即可。

参考链接：4.4 MySQL存储-CSDN博客 5.2 Ajax 数据爬取实战-CSDN博客

5、1-3步总代码

from selenium import webdriver
import json, time
from lxml import etree


def get_cookie(url):
    browser = webdriver.Chrome()
    browser.get(url)    # 进入登录页面
    time.sleep(10)      # 停一段时间，自己手动登录
    cookies_dict_list = browser.get_cookies()  # 获取list的cookies
    cookies_json = json.dumps(cookies_dict_list)  # 转换成字符串保存
    with open('cookies.txt', 'w') as f:
        f.write(cookies_json)
    print('cookies保存成功！')
    browser.close()


def get_html(url):
    browser = webdriver.Chrome()
    browser.get(url)
    if browser.current_url == url:
        js = "var q=document.documentElement.scrollTop=100000"
        browser.execute_script(js)
        time.sleep(2)
        responses = browser.page_source
        browser.close()
        return responses
    else:
        with open('cookies.txt', 'r', encoding='utf8') as f:
            listCookies = json.loads(f.read())
        for cookie in listCookies:
            cookie_dict = {
                'domain': '.jd.com',
                'name': cookie.get('name'),
                'value': cookie.get('value'),
                "expires": '1746673777',
                'path': '/',
                'httpOnly': False,
                "sameSite": "Lax",
                'Secure': False
            }
            browser.add_cookie(cookie_dict) # 将获得的所有 cookies 都加入
        time.sleep(1)
        browser.get(url)
        js = "var q=document.documentElement.scrollTop=100000"
        browser.execute_script(js)
        time.sleep(2)
        response = browser.page_source
        browser.close()
        return response


def parse(response):
    html = etree.HTML(response)
    products_lis = html.xpath('//li[contains(@class, "gl-item")]')
    products_list = []
    for product_li in products_lis:
        product = {}
        price = product_li.xpath('./div/div[@class="p-price"]/strong/i/text()')
        name = product_li.xpath('./div/div[contains(@class, "p-name")]/a/em/text()')
        comment = product_li.xpath('./div/div[@class="p-commit"]/strong/a/text()')

        product['price'] = price[0]
        product['name'] = name[0].strip()
        product['comment'] = comment[0]
        products_list.append(product)
    print(products_list)
    print(len(products_list))


if __name__ == '__main__':
    keyword = '笔记本电脑'
    url_login = 'https://passport.jd.com/uc/login?'     # 登录页面url
    get_cookie(url_login)
    for page in range(1, 20, 2):
        base_url = f'https://search.jd.com/Search?keyword={keyword}&wq=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&pvid=222071d82c0344059f4aac713b1679b4&isList=0&page={page}'
        response = get_html(base_url)
        parse(response)

文章参考：python爬虫之使用selenium爬取京东商品信息并把数据保存至mongodb数据库_seleniu获取京东cookie-CSDN博客

文章到此结束，本人新手，若有错误，欢迎指正；若有疑问，欢迎讨论。若文章对你有用，点个小赞鼓励一下，谢谢大家，一起加油吧！