多线程的使用(上)
from threading import Thread
from queue import Queue
from fack useragent import UserAgent
import requests
爬虫类
class crawlInfo(Thread):
def init(self):
Thread.init(self,url_queue)
self.url_queue = url_queue
def run(self);
headers = {
“User-Agent”:UserAgent().random
}
response = requests.get(self.url_queue,headers=headers)
print(response .text)
if mame = ‘main’:
存储url的容器
url_queue = Queue()
base_url = “http://www.qiushibaike.com/text/page/{}/”
for i in rangr(1,14):
new_url = base_url.format()
url_queue.put(new_url)