python线程详解爬小说_python--多线程爬取顶点小说（）

import requests

from lxml import etree

from threading import Thread

from queue import Queue

class MyThread(Thread):

def __init__(self, q):

Thread.__init__(self)

self.q = q

def run(self):

global index

while not self.q.empty():

data = self.q.get()

url = root + ''.join(data[1])

response = requests.get(url, headers=headers)

page = etree.HTML(response.content)

chapter = page.xpath("//h1/text()")

chapter = ''.join(chapter)

print("爬取 -> %s" % chapter,index)

content = page.xpath("//div[@id='content']/text()")

content = '\n'.join(content)

content = content.replace("\xa0\xa0\xa0\xa0", "\t")

# 如果当前标记比保存的小说章节序号大于1，阻塞

while data[0] > index + 1:

pass

# 刚好大于1时，通过，保存章节

if data[0] == index + 1:

print("保存 -> %s" % chapter,index)

f.write('\n' + chapter + '\n')

f.write(content)

index += 1

if __name__ == '__main__':

root = "http://www.booktxt.net/8_8455/"

headers = {

'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'

}

index = -1 # 章节标记，表示保存的章数

response = requests.get(root, headers=headers)

page = etree.HTML(response.content)

title = ''.join(page.xpath("//h1/text()")) # 小说名

print(title)

with open("%s.txt" % title, 'w', encoding='utf8') as f:

f.write(title) # 先写入小说名

hrefs = page.xpath("//div[@id='list']/dl/dt[2]/following-sibling::dd/a/@href")

q = Queue()

for i,href in enumerate(hrefs):

q.put((i,href))

ts = []

for i in range(5):

t = MyThread(q)

t.start()

ts.append(t)

for t in ts:

t.join()