python爬虫怎么写多线程_python爬虫如何利用多线程？

from __future__ import print_function

import time

from multiprocessing.dummy import Pool as ThreadPool

from pprint import pprint

import logging

logging.basicConfig(

level=logging.INFO,

format='[%(levelname)1.1s%(asctime)s]%(message)s',

)

def request(url):

import requests

resp = requests.get(url, timeout=5)

resp.raise_for_status()

return resp.status_code

urls = [

'http://www.python.org',

'http://www.python.org/about/',

'http://www.onlamp.com/pub/a/python/2003/04/17/metaclasses.html',

'http://www.python.org/doc/',

'http://www.python.org/download/',

'http://www.python.org/getit/',

'http://www.python.org/community/',

'https://wiki.python.org/moin/',

'http://planet.python.org/',

'https://wiki.python.org/moin/LocalUserGroups',

'http://www.python.org/psf/',

'http://docs.python.org/devguide/',

'http://www.python.org/community/awards/',

'http://www.xxxyyy.xxcom',

]

# urls = ["http://konghy.cn"] * 80

time1 = time.time()

# Make the Pool of workers

pool = ThreadPool(4)

# Open the urls in their own threads

# and return the results

try:

results = pool.map(request, urls)

except Exception as e:

print(">"*50, e)

raise

#close the pool and wait for the work to finish

pool.close()

pool.join()

pprint(results)

time2 = time.time()

print("-" * 100)

for url in urls:

print(request(url))

time3 = time.time()

print(time2 - time1, time3 - time2)