Datawhale爬虫组队学习Task 04

爬取腾讯新闻
1.了解ajax加载
2.通过chrome的开发者工具，监控网络请求，并分析
3.用selenium完成爬虫:

• 用selenium爬取https://news.qq.com/ 的热点精选

import time
import random
from selenium import webdriver
from lxml import etree

driver = webdriver.Chrome()
driver.get('https://news.qq.com/')

# 因为是随机滑动，所以无法控制爬取的条数
for i in range(20):
    height = random.randint(800,1000)
    driver.execute_script(f'window.scrollBy(0,{height})')
    time.sleep(random.random()+0.2) #[0.2,1.2)

html = driver.page_source
tree = etree.HTML(html)
infos = tree.xpath('//ul[@class="list"]/li/div[@class="detail"]/h3/a')

infolist = []
count = 0

for i,info in enumerate(infos):
    title = info.xpath('text()')[0]
    href = info.xpath('@href')[0]
    # print(i+1,title,href)
    infolist.append([i+1,title,href])

# 保存为csv
import pandas as pd
name = ['序号','新闻标题','新闻链接']
# 多于50条只保存前50条
try:
    df = pd.DataFrame(columns=name, data=infolist[:50])
except:
    df = pd.DataFrame(columns=name, data=infolist)

# 注意此处的编码，不设置会出现乱码
df.to_csv('腾讯新闻热点.csv',index=False,encoding='utf_8_sig')
df