python学习之简单正则与爬虫

python正则表达式常用方法：

http://www.runoob.com/python/python-reg-expressions.html

技巧一：匹配xx123xx的123时，可以用xx(.*？)xx进行匹配 非贪心匹配

#-*- coding:utf8 -*-
import requests
import re# 导入正则表达式的模块

#导入下面这几行用中文就不会出错
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

class Spider():
    def getGtmlSource(self,url):
        """获得html"""
        html = requests.get(url).text
        return html

    def changepage(self,url,total_page):
        """获得所有url的列表"""
        nowPage = int(re.search('pageNum=(\d+)',url).group(1))#re.search只找到第一个符合条件的 .group(1)是只要一个括号里面的
        all_link=[]
        for i in range(nowPage,total_page+1):
             # 替换
             link = re.sub('pageNum=\d+','pageNum=%s'%i,url,re.S)
             all_link.append(link)
        return all_link

    def geteveryclass(self,source):
        everyclass = re.findall('(deg="0" >.*?</li>)',source,re.S)
        return everyclass

    def getinfo(self,eachclass):
        """getinfo用来从每个课程块中提取出我们需要的信息"""

        info = {}
        info['title'] = re.search('class="lessonimg" title="(.*?)>',eachclass,re.S).group(1)
        info['content'] = re.search('display: none;">(.*?)</p>',eachclass,re.S).group(1)

        return info

if __name__ == "__main__":
    s = Spider()
    classinfo = []
    url = 'http://www.jikexueyuan.com/course/?pageNum=1'
    url_list = s.changepage(url,2)
    for link in url_list:
        print u'正在处理页面：' + link
        html = s.getGtmlSource(link)
        # print html
        everyclass = s.geteveryclass(html)
        for each in everyclass:
            info = s.getinfo(each)
            classinfo.append(info)
    print classinfo