Python 爬取小说

Python爬取小说

import os

import requests
from lxml import etree



#获取想要爬取的链接
#网址 + 页面比编号
urls = ['https://xxxxx/{}'.format (i) for i in range(15339098,15342660)]
#分了三页
#测试是否获取成功
# for url in urls:
#   print(url+".html")
#   print(url+"_2.html")
#   print(url+"_3.html")


#防拦截
header = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"
}


#设置保持的路径



#获取小说内容并保存
# 方法


def get_text(url):
    r = requests.get(url,headers = header)
    r.encoding = 'gbk'
    selector = etree.HTML(r.text)
#f12 中找到页面元素 copy 当前xpath 值
    #获取文章标题
    title = selector.xpath('//*[@id="read"]/div[1]/span[1]/text()')
    #获取小说正文内容//*[@id="chaptercontent"]
    text = selector.xpath('//*[@id="chaptercontent"]/text()')
    print(title)
    print()
   #保存至当前路劲 的xxx文本文件中
    with open('xxx.txt', 'a', encoding='utf-8') as f:
        for items in text:
            f.write(items)

for url in urls:
    get_text(url+".html")
    get_text(url+"_2.html")
    get_text(url+"_3.html")
# 'xiaoshuo/%s.txt'%title[0]