python学习之使用multiprocessing.dummy多线程爬虫

#-*- coding:utf8 -*-
import requests
import re# 导入正则表达式的模块
from multiprocessing.dummy import Pool as ThreadPool
import time

#导入下面这几行用中文就不会出错
import sys
reload(sys)
sys.setdefaultencoding("utf-8")


def changepage(url,total_page):
    """获得所有url的列表"""
    nowPage = int(re.search('pageNum=(\d+)',url).group(1))#re.search只找到第一个符合条件的 .group(1)是只要一个括号里面的
    all_link=[]
    for i in range(nowPage,total_page+1):
         # 替换
         link = re.sub('pageNum=\d+','pageNum=%s'%i,url,re.S)
         all_link.append(link)
    return all_link

def geteveryclass(url):
    html = requests.get(url)
    classinfo=[]
    everyclass = re.findall('(deg="0" >.*?</li>)',html.text,re.S)
    for each in everyclass:
        info = getinfo(each)
        classinfo.append(info)
    print classinfo


def getinfo(eachclass):
    """getinfo用来从每个课程块中提取出我们需要的信息"""
    info = {}
    info['title'] = re.search('class="lessonimg" title="(.*?)>',eachclass,re.S).group(1)
    info['content'] = re.search('display: none;">(.*?)</p>',eachclass,re.S).group(1)

    return info
######################################################################################################################
if __name__ == "__main__":

    url = 'http://www.jikexueyuan.com/course/?pageNum=1'
    url_list = changepage(url,5)
    pool = ThreadPool(4)#4核
    time1 = time.time()
    results = pool.map(geteveryclass, url_list)#需要执行的方法和地址列表
    pool.close()
    pool.join()#等待线程都结束后再执行主模块
    time2 = time.time()
    print u'多线程耗时：' + str(time2-time1)