python 爬虫

一、支持任意章节链接或目录链接

# -*- coding: utf-8 -*-
"""
Created on Thu Jan  6 12:38:42 2022

@author: shiyu
"""

import time
import requests
from lxml import etree
import re

def spider(url):
    html=requests.get(url,headers=headers)
    html_=etree.HTML(html.text)
    words=html_.xpath('//div[@id="content"]/text()')
    
    k=r'(<title>)(.*?)(</title>)'
    title=re.search(k,html.text)
    
    with open('{}.txt'.format(title2[0]),'a',encoding='utf-8') as f:
        f.write(title.group(2)+'\n')
        for line in words:
            line=line.replace('\\xa0',' ')
            f.write(line+'\n')
        f.write('\n')

headers={'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,\
         like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62'
    }
#替换这里的url
url='https://www.81zw.com/book/69283/'
url_='https://www.81zw.com{}'
if url.find('html')!=-1:
    request=requests.get(url,headers=headers)
    k=r'(&larr; <a href=")(.*?)(">章节目录</a>)'
    link_content=re.search(k,request.text).group(2)
    content_url=url_.format(link_content)
    html_=requests.get(content_url,headers=headers)
else:
    html_=requests.get(url,headers=headers)
html=etree.HTML(html_.text)
links=html.xpath('//dd/a/@href')
title2=html.xpath('//meta[@property="og:title"]/@content')

with open('{}.txt'.format(title2[0]),'w',encoding='utf-8') as f:
    pass

for pos,link in enumerate(links):
    url_con=url_.format(link)
    spider(url_con)
    time.sleep(0.1)
    print('第'+str(pos+1)+'章')