import os
import requests
from lxml import etree
import time
class Novel:
def __init__(self):
self.url = 'https://www.wxsy.net/novel/71254/'
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54'
}
def get_data(self, url):
resp = requests.get(url, headers=self.headers)
if resp.status_code == 200:
return resp.text
else:
return None
def parse_data(self, response):
html = etree.HTML(response)
dds = html.xpath('//div[@class="pt-chapter-cont-detail full"]//a/@href')
return dds
def parse_url(self, parse):
html = etree.HTML(parse)
title = html.xpath('//h1[@class="lh100 size26 mb20"]/a/@title')[0].replace('?', '')
content = html.xpath('//p[@class="content_detail"]/text()')
return title, content
def download(self, content):
with open('./这些大人物其实都是我/'+content[0]+'.text', 'w', encoding='utf-8') as f:
f.writelines(content[1])
print(content[0]+'下载成功')
def main(self):
response = self.get_data(self.url)
href = self.parse_data(response)
os.mkdir('这些大人物其实都是我')
for data in href:
url = 'https://www.wxsy.net' + data
parse = self.get_data(url)
content = self.parse_url(parse)
self.download(content)
time.sleep(0.5)
if __name__ == '__main__':
novel = Novel()
novel.main()
爬取 小说
最新推荐文章于 2024-08-19 13:32:52 发布