python文字处理dummy_python学习之使用multiprocessing.dummy多线程爬虫

#-*- coding:utf8 -*-

import requests

import re# 导入正则表达式的模块

from multiprocessing.dummy import Pool as ThreadPool

import time

#导入下面这几行用中文就不会出错

import sys

reload(sys)

sys.setdefaultencoding("utf-8")

def changepage(url,total_page):

"""获得所有url的列表"""

nowPage = int(re.search('pageNum=(\d+)',url).group(1))#re.search只找到第一个符合条件的 .group(1)是只要一个括号里面的

all_link=[]

for i in range(nowPage,total_page+1):

# 替换

link = re.sub('pageNum=\d+','pageNum=%s'%i,url,re.S)

all_link.append(link)

return all_link

def geteveryclass(url):

html = requests.get(url)

classinfo=[]

everyclass = re.findall('(deg="0" >.*?)',html.text,re.S)

for each in everyclass:

info = getinfo(each)

classinfo.append(info)

print classinfo

def getinfo(eachclass):

"""getinfo用来从每个课程块中提取出我们需要的信息"""

info = {}

info['title'] = re.search('class="lessonimg" title="(.*?)>',eachclass,re.S).group(1)

info['content'] = re.search('display: none;">(.*?)

',eachclass,re.S).group(1)

return info

######################################################################################################################

if __name__ == "__main__":

url = 'http://www.jikexueyuan.com/course/?pageNum=1'

url_list = changepage(url,5)

pool = ThreadPool(4)#4核

time1 = time.time()

results = pool.map(geteveryclass, url_list)#需要执行的方法和地址列表

pool.close()

pool.join()#等待线程都结束后再执行主模块

time2 = time.time()

print u'多线程耗时：' + str(time2-time1)