Python网络爬虫之动态网页爬取及使用selenium模块爬取

使用requests模块爬取动态网页数据

"""
使用requests模块爬取动态网页数据
今日头条：某条新闻的评论信息
"""
import requests

#路径
url = "https://www.toutiao.com/api/comment/list/?group_id=6749065854995939854&item_id=6749065854995939854&offset=0&count=15"

# 响应头
headers = {
   
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3314.0 Safari/537.36 SE 2.X MetaSr 1.0",
}

# 解决日志中的大量warning信息
requests.packages.urllib3.disable_warnings()

# 请求，获取响应
response = requests.get(url, headers=headers, verify=False)
if response.status_code == 200:
    # print(response.text)
    # print(response.json())
    with open("今日评论。txt", "w", encoding="UTF8"