python并发1000个http请求_用Python发送100,000个HTTP请求的最快方法是什么？

小编典典

无捻解决方案：

from urlparse import urlparse

from threading import Thread

import httplib, sys

from Queue import Queue

concurrent = 200

def doWork():

while True:

url = q.get()

status, url = getStatus(url)

doSomethingWithResult(status, url)

q.task_done()

def getStatus(ourl):

try:

url = urlparse(ourl)

conn = httplib.HTTPConnection(url.netloc)

conn.request("HEAD", url.path)

res = conn.getresponse()

return res.status, ourl

except:

return "error", ourl

def doSomethingWithResult(status, url):

print status, url

q = Queue(concurrent * 2)

for i in range(concurrent):

t = Thread(target=doWork)

t.daemon = True

t.start()

try:

for url in open('urllist.txt'):

q.put(url.strip())

q.join()

except KeyboardInterrupt:

sys.exit(1)

这比扭曲的解决方案要快一点，并且使用的CPU更少。

2020-02-16