import json
from lxml import etree
import requests
def load_page(url):
header = {
# 浏览器身份信息,伪装程序是浏览器
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
}
request = requests.get(url,headers=header) # 发送请求,获取响应
return request.text
def parse_html(html):
text = etree.HTML(html)
node_list = text.xpath('//th[contains(@class,"new") and contains(@class,"forumtit")]')
items = []
num = 0
for node in node_list:
try:
title = node.xpath('./a[1]/text()')[0]
url = node.xpath('./a[1]/@href')[0]
author = node.xpath(
'./div[@class="foruminfo"]//a/span/text()'
)[0]
release_time = node.xpath('./div[2]/i/span[1]/text()')[0].strip().replace('@','')
reply_num = node.xpath('./div[2]/i[1]/span[2]/a/text()')[0].strip().replace('\r\n','')
items = {
"文章标题":title,
"文章链接":url,
"文章作者":author,
'发布时间':release_time,
'回复数':reply_num
}
items.append(items)
num +=1
except Exception as e:
pass
return items
def save_file(items):
try:
with open('heima.jso',mode='w+',encoding='utf-8') as f:
f.write(json.dumps(items, ensure_ascii-False,indent=2))
except Exception as e:
print(e)
def heima_forum(begin_page,end_page):
li_data = []
for page in range(begin_page,end_page+1):
url = f'http://bbs.itheima.com/forum-424-{page}.html'
file_name = "正在请求第"+str(page)+"页"
print(file_name)
html = load_page(html)
data = parse_html(html)
li_data += data
save_file(li_data)
def __name__== "__main__":
begin_page = int(input("请输入起始页码:"))
end_page = int(input("请输入结束页码:"))
heima_forum(begin_page, end_page)
报错暂存明天修改
最新推荐文章于 2024-07-16 19:26:54 发布