代码
import urllib.request
import time
import queue
from threading import Thread
import threading
url = "https://www.pythontab.com/html/pythonjichu/"(可以根据自己的实际情况选取url)
queue = queue.Queue()
queue.put(url)
for i in range(2,10):
new_url = url+str(i)+'.html'
queue.put(new_url)
def fetchurl(urlQueue):
while True:
try:
urlQueue.get_nowait()
number = urlQueue.qsize()
print(number)
except Exception as e:
break
print('当前的url是:','url')
print('当前线性{}获取的url是:{}'.format(threading.currentThread().name,url))
try:
response = urllib.request.urlopen(url)
status_code = response.getcode()
if status_code == 200:
time.sleep(0.5)
except Exception as e:
continue
stat_time = time.time()
threads = []
thread_num = 10(线程的个数)
for i in range(thread_num):
thread = Thread(target=fetchurl,args=(queue,))
threads.append(thread)
for t in threads:
t.start()
for t in threads:
t.join()
end_time = time.time()
print('消耗的时间是:',end_time-stat_time)
爬出结果展示