爬取腾讯新闻
1.了解ajax加载
2.通过chrome的开发者工具,监控网络请求,并分析
3.用selenium完成爬虫:
- 用selenium爬取https://news.qq.com/ 的热点精选
import time
import random
from selenium import webdriver
from lxml import etree
driver = webdriver.Chrome()
driver.get('https://news.qq.com/')
# 因为是随机滑动,所以无法控制爬取的条数
for i in range(20):
height = random.randint(800,1000)
driver.execute_script(f'window.scrollBy(0,{height})')
time.sleep(random.random()+0.2) #[0.2,1.2)
html = driver.page_source
tree = etree.HTML(html)
infos = tree.xpath('//ul[@class="list"]/li/div[@class="detail"]/h3/a')
infolist = []
count = 0
for i,info in enumerate(infos):
title = info.xpath('text()')[0]
href = info.xpath('@href')[0]
# print(i+1,title,href)
infolist.append([i+1,title,href])
# 保存为csv
import pandas as pd
name = ['序号','新闻标题','新闻链接']
# 多于50条只保存前50条
try:
df = pd.DataFrame(columns=name, data=infolist[:50])
except:
df = pd.DataFrame(columns=name, data=infolist)
# 注意此处的编码,不设置会出现乱码
df.to_csv('腾讯新闻热点.csv',index=False,encoding='utf_8_sig')
df