多线程爬取糗事百科

1.所用到的模块
request ，queue，time，etree
2.对于多线程知识有一定了解，类的继承，异常处理等
3.构思自己大致的思路。

import threading
import queue
import requests
import time
from lxml import etree
class thread1(threading.Thread):
	def __init__(self, threadname,pagequeue,dataqueue):
		super(thread1, self).__init__()
		self.threadname = threadname
		self.pagequeue = pagequeue
		self.dataqueue = dataqueue
		self.header = {"User-Agent" : "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}

	def run(self):
		print('启动进程',self.threadname)
		while not flag1:
			try:
				page = self.pagequeue.get()
				url = 'https://www.qiushibaike.com/8hr/page/'+str(page)+'/'
				rep = requests.get(url,headers=self.header).text
				time.sleep(0.5)
				self.dataqueue.put(rep)
			except Exception as e:
				pass

		print('结束进程',self.threadname)

class thread2(threading.Thread):

	def __init__(self,threadname,dataqueue,filename):
		super(thread2, self).__init__()
		self.dataqueue = dataqueue
		self.threadname = threadname
		self.filename = filename

	def run(self):
		print('启动进程',self.threadname)
		while not flag2:
			try:
				rep2 = self.dataqueue.get()
				html = etree.HTML(rep2)
				non_list =html.xpath('//div/a[@class="recmd-content"]')
				for x in non_list:
					self.filename.write(x+'\n')

			except Exception as e:
				pass

		print('结束进程',self.threadname)

flag1 = False

flag2 = False

def main():

	pagequeue = queue.Queue(10)

	for i in range(1,11):
		pagequeue.put(i)

	dataqueue = queue.Queue()

	filename = open(r"C:\offline_FtnInfo.txt","a")

	t1 = thread1('采集线程',pagequeue,dataqueue)
	t1.start()
	t2 = thread2('解析线程',dataqueue,filename)
	t2.start()

	while not pagequeue.empty():
		pass
	global flag1
	flag1 = True

	while not dataqueue.empty():
		pass
	global flag2
	flag2 = True

	t1.join()
	t2.join()

	filename.close()
	print('结束')



if __name__ == '__main__':
	main()