import requests
from pyquery import PyQuery
import re
import os
# 构造请求头
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, "
"like Gecko) Chrome/81.0.4044.138 Safari/537.36",
"Connection": "close"
}
# TODO 1.根据小说链接得到小说目录和对应的url
def get_catalogue(url):
# 发送请求
response = requests.get(url=url, headers=headers)
response.encoding = "gbk" # 指定编码
chapter_info_list = [] # 用来存储获取到的所有url和章节标题数据
doc = PyQuery(response.text)
# print(doc)
pages_url = doc(".list-charts li a").items()
# print(pages_url)
for page_url in pages_url:
title = page_url.text()
chapter_url = "http://www.kuhu168.com" + page_url.attr("href")
# print(title, chapter_url)
# 使用字典存储url和title
chapter = {"title": title, "url": chapter
酷虎小说网爬虫
最新推荐文章于 2023-08-31 11:04:55 发布
import requestsfrom pyquery import PyQueryimport reimport os# 构造请求头headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, " "like Gecko) Chrome/81.0.4044.138 Safari/537.36", "Co
摘要由CSDN通过智能技术生成