利用request_html爬取百度热榜并存入csv文件中

工具：python，request_html库，csv库

废话不多说直接上代码

from requests_html import HTMLSession
import  csv

session = HTMLSession()
headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.106 Safari/537.36"
}
# 分析页面找到内容
def parse_html(url):
    r = session.get(url,headers = headers)
    top = r.html.xpath('//td[@class="first"]/span/text()')
    content = r.html.xpath('//td[@class="keyword"]/a[1]/text()')
    hook = r.html.xpath('//td[@class="last"]/span/text()')
    return top,content,hook

def download_info_to_csv(top,content,hook):
    with open('百度热搜榜.csv','w',newline='',encoding='utf-8-sig') as f:
        writer = csv.writer(f)
        # 写入表头
        print('正在写入....')
        writer.writerow(['排名','标题','热度'])
        for i,number in enumerate(top):
            writer.writerow([number,content[i],hook[i]])

def main():
    url = 'http://top.baidu.com/buzz?b=1&fr=topindex'
    top,content,hook= parse_html(url)
    download_info_to_csv(top,content,hook)
    print('写入成功！')

if __name__ == '__main__':
    main()

代码比较简单我就不讲解了，有不懂的可以评论上问，不同的是我这里是用的request_html库的xpath来获取信息。

效果图