使用playwright--python 爬虫简单使用（vscode、tb、保存到CSV文件）

最新推荐文章于 2024-06-21 15:44:03 发布

dbgine

最新推荐文章于 2024-06-21 15:44:03 发布

阅读量1.9k

点赞数

分类专栏：爬虫文章标签：爬虫 vscode python csv

本文链接：https://blog.csdn.net/weixin_49546561/article/details/122263487

版权

爬虫专栏收录该内容

1 篇文章 0 订阅

订阅专栏

分析结构

价格、付款人数
文本
厂家地址
活动（不使用）

安装

pip install --upgrade pip
pip install playwright
playwright install

爬取数据基本api

page.query_selector_all(selector)--元素选择器，返回标签列表<List[ElementHandle]>
element_handle.query_selector_all（selector）--子元素也可使用query
page.wait_for_timeout(1000)--等待时间
page.wait_for_selector(selector, **kwargs)--等待元素
element_handle.get_attribute（name）--获取元素标签属性值
page.inner_text(selector, **kwargs)--标签文本
page.set_default_timeout(timeout)--等待时间

代码

import asyncio
from playwright.async_api import async_playwright
import csv


async def main():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=False)
        page = await browser.new_page()
        fp = open("./tbData.csv", 'a', newline='', encoding='utf-8-sig')
        writer = csv.writer(fp)
        # 写入内容
        writer.writerow(('价格', '购买人数', '文本', '厂址信息'))
        await page.goto("https://s.taobao.com/search?initiative_id=tbindexz_20170306&ie=utf8&spm=a21bo.jianhua.201856-taobao-item.2&sourceId=tb.index&search_type=item&ssid=s5-e&commend=all&imgfile=&q=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&suggest=history_1&_input_charset=utf-8&wq=&suggest_query=&source=suggest&bcoffset=-14&ntoffset=-14&p4ppushleft=2%2C48&s=264")
        # url = await page.get_attribute("#a1 > div.dplayer-video-wrap > video", " src")
        await page.wait_for_timeout(60000)
        for i in range(100):
            await page.wait_for_timeout(10000)
            await page.goto("https://s.taobao.com/search?q=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&bcoffset="+str(4-i*3)+"&ntoffset="+str(4-i*3)+"&p4ppushleft=2%2C48&s="+str(i*44))
            await page.wait_for_timeout(10000)
            list = await page.query_selector_all("#mainsrp-itemlist > div > div > div:nth-child(1) > div > div.ctx-box.J_MouseEneterLeave.J_IconMoreNew")
            # print(await j.inner_text())
            for j in list:
                price = await j.query_selector(".price.g_price.g_price-highlight")
                count = await j.query_selector(".deal-cnt")
                txt = await j.query_selector(".row.row-2.title")
                home = await j.query_selector(".row.row-3.g-clearfix")

                price_csv = await price.inner_text()
                count_csv = await count.inner_text()
                txt_csv = await txt.inner_text()
                home_csv = await home.inner_text()
                writer.writerow((str(price_csv), str(count_csv),
                                str(txt_csv), str(home_csv)))

                print('价格', price_csv)
                print('购买人数', count_csv)
                print('文本', txt_csv)
                print('地址店家', home_csv)
        await browser.close()
        fp.close()
asyncio.run(main())