目录
分析结构
安装
爬取数据基本api
代码
分析结构
![](https://i-blog.csdnimg.cn/blog_migrate/436fe30a05709e818d5b52710ee18e51.png)
安装
pip install --upgrade pip
pip install playwright
playwright install
爬取数据基本api
- page.query_selector_all(selector)--元素选择器,返回标签列表<List[ElementHandle]>
- element_handle.query_selector_all(selector)--子元素也可使用query
- page.wait_for_timeout(1000)--等待时间
- page.wait_for_selector(selector, **kwargs)--等待元素
- element_handle.get_attribute(name)--获取元素标签属性值
- page.inner_text(selector, **kwargs)--标签文本
- page.set_default_timeout(timeout)--等待时间
代码
import asyncio
from playwright.async_api import async_playwright
import csv
async def main():
async with async_playwright() as p:
browser = await p.chromium.launch(headless=False)
page = await browser.new_page()
fp = open("./tbData.csv", 'a', newline='', encoding='utf-8-sig')
writer = csv.writer(fp)
# 写入内容
writer.writerow(('价格', '购买人数', '文本', '厂址信息'))
await page.goto("https://s.taobao.com/search?initiative_id=tbindexz_20170306&ie=utf8&spm=a21bo.jianhua.201856-taobao-item.2&sourceId=tb.index&search_type=item&ssid=s5-e&commend=all&imgfile=&q=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&suggest=history_1&_input_charset=utf-8&wq=&suggest_query=&source=suggest&bcoffset=-14&ntoffset=-14&p4ppushleft=2%2C48&s=264")
# url = await page.get_attribute("#a1 > div.dplayer-video-wrap > video", " src")
await page.wait_for_timeout(60000)
for i in range(100):
await page.wait_for_timeout(10000)
await page.goto("https://s.taobao.com/search?q=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&bcoffset="+str(4-i*3)+"&ntoffset="+str(4-i*3)+"&p4ppushleft=2%2C48&s="+str(i*44))
await page.wait_for_timeout(10000)
list = await page.query_selector_all("#mainsrp-itemlist > div > div > div:nth-child(1) > div > div.ctx-box.J_MouseEneterLeave.J_IconMoreNew")
# print(await j.inner_text())
for j in list:
price = await j.query_selector(".price.g_price.g_price-highlight")
count = await j.query_selector(".deal-cnt")
txt = await j.query_selector(".row.row-2.title")
home = await j.query_selector(".row.row-3.g-clearfix")
price_csv = await price.inner_text()
count_csv = await count.inner_text()
txt_csv = await txt.inner_text()
home_csv = await home.inner_text()
writer.writerow((str(price_csv), str(count_csv),
str(txt_csv), str(home_csv)))
print('价格', price_csv)
print('购买人数', count_csv)
print('文本', txt_csv)
print('地址店家', home_csv)
await browser.close()
fp.close()
asyncio.run(main())