81zw 下载

# -*- coding: utf-8 -*-
"""
Created on Thu Jan  6 12:38:42 2022

@author: shiyu
"""

import time
import requests
from lxml import etree
import re
import threading
import csv
import pandas as pd

def spider(con,dic):
    url=con[0]
    print(con[1])
    html=requests.get(url,headers=headers)
    html_=etree.HTML(html.text)
    words=html_.xpath('//div[@id="content"]/text()')
    
    k=r'(<title>)(.*?)(</title>)'
    title=re.search(k,html.text)
    
    s=''
    s+=title.group(2)+'\n'
    for line in words:
        line=line.replace('\\xa0',' ')
        s+=line+'\n'
    s+='\n'
    dic[con[1]]=s




headers={'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,\
         like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62'
    }
dic={}
#替换这里的url
url='https://www.81zw.com/book/21776/'
url_='https://www.81zw.com{}'
if url.find('html')!=-1:
    request=requests.get(url,headers=headers)
    k=r'(&larr; <a href=")(.*?)(">章节目录</a>)'
    link_content=re.search(k,request.text).group(2)
    content_url=url_.format(link_content)
    html_=requests.get(content_url,headers=headers)
else:
    html_=requests.get(url,headers=headers)
html=etree.HTML(html_.text)
links=html.xpath('//dd/a/@href')
title2=html.xpath('//meta[@property="og:title"]/@content')

url_cons=[]
for pos,link in enumerate(links):
    url_con=url_.format(link)
    con=[url_con,pos+1]
    url_cons.append(con)
    if (pos+1)%50==0 or len(links)==pos+1:
        thread=[]
        for i in range(50):
            try:
                thread.append(threading.Thread(target=spider,args=(url_cons[i],dic,)))
            except:
                break
        for it in thread:
            it.start()
        for it in thread:
            it.join()
        url_cons.clear()
        

keys=sorted(dic.keys())
with open('{}.txt'.format(title2[0]),'w',encoding='utf-8') as f:
    for key in keys:
        f.write(dic[key])