1.所用到的模块
request ,queue,time,etree
2.对于多线程知识有一定了解,类的继承,异常处理等
3.构思自己大致的思路。
import threading
import queue
import requests
import time
from lxml import etree
class thread1(threading.Thread):
def __init__(self, threadname,pagequeue,dataqueue):
super(thread1, self).__init__()
self.threadname = threadname
self.pagequeue = pagequeue
self.dataqueue = dataqueue
self.header = {"User-Agent" : "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}
def run(self):
print('启动进程',self.threadname)
while not flag1:
try:
page = self.pagequeue.get()
url = 'https://www.qiushibaike.com/8hr/page/'+str(page)+'/'
rep = requests.get(url,headers=self.header).text
time.sleep(0.5)
self.dataqueue.put(rep)
except Exception as e:
pass
print('结束进程',self.threadname)
class thread2(threading.Thread):
def __init__(self,threadname,dataqueue,filename):
super(thread2, self).__init__()
self.dataqueue = dataqueue
self.threadname = threadname
self.filename = filename
def run(self):
print('启动进程',self.threadname)
while not flag2:
try:
rep2 = self.dataqueue.get()
html = etree.HTML(rep2)
non_list =html.xpath('//div/a[@class="recmd-content"]')
for x in non_list:
self.filename.write(x+'\n')
except Exception as e:
pass
print('结束进程',self.threadname)
flag1 = False
flag2 = False
def main():
pagequeue = queue.Queue(10)
for i in range(1,11):
pagequeue.put(i)
dataqueue = queue.Queue()
filename = open(r"C:\offline_FtnInfo.txt","a")
t1 = thread1('采集线程',pagequeue,dataqueue)
t1.start()
t2 = thread2('解析线程',dataqueue,filename)
t2.start()
while not pagequeue.empty():
pass
global flag1
flag1 = True
while not dataqueue.empty():
pass
global flag2
flag2 = True
t1.join()
t2.join()
filename.close()
print('结束')
if __name__ == '__main__':
main()