python多线程获取网络数据，与单线程进行比较

-- coding:UTF-8 --

import requests
import re
import time
from multiprocessing.dummy import Pool as ThreadPool
import sys

reload(sys)
sys.setdefaultencoding(‘utf8’)

def loadingContent(url):
”’
网络获取信息
:param url:网址
:return:
”’
return requests.get(url).text

def getAllContent(url,start,end):
”’
获取所有数据
:param url:网址的前半段
:param start: 开始页
:param end: 结束页
:return:
”’
for i in range(start,end):
u = url + str(i)
loadingContent(u)

def threadLoading(fun,urls):
”’
并行获取数据
:param fun:
:param urls:
:return:
”’
pool = ThreadPool(2)
results = pool.map(fun,urls)
pool.close()
pool.join()

main

if name == ‘main‘:
url = ‘http://tieba.baidu.com/f?kw=%E8%8B%B1%E9%9B%84%E8%81%94%E7%9B%9F&ie=utf-8&pn=’
time1 = time.time()
getAllContent(url,1,20)
time2 = time.time()
print str(time2 - time1)

time3 = time.time()
urls = []
for i in range(1,20):
    urls.append(url+str(i))
threadLoading(loadingContent,urls)
time4 = time.time()
print str(time4 - time3)