极客学院课程爬虫Requests

2019独角兽企业重金招聘Python工程师标准>>>

1、Requests抓取网页

2、re.sub换页

3、正则表达式匹配内容

# -*- coding: utf-8 -*-
import requests
import re
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
# 在windows下面命令提示符默认编码是GBK 要爬内容为UTF-8

class spider(object):
    def changepage(self, url, total_page):
        now_page = int(re.search('pageNum=(\d+)',url,re.S).group(1))
        page_group = []
        for i in range(now_page,total_page+1):
            link = re.sub('pageNum=\d+','pageNum=%s'%i,url,re.S)
            page_group.append(link)
        return page_group
    def getsource(self,url):
        html = requests.get(url)
        return html.text
    def geteveryclass(self,source):
        everyclass = re.findall('<div class="lesson-infor(.*?)</li>',source,re.S)
        return everyclass
    def getinfo(self,eachclass):
        info = {}
        info['title'] = re.search('<h2 class="lesson-info-h2"><a href="(.*?)">(.*?)</a></h2>',eachclass,re.S).group(2)
        info['des'] = re.search('<p(.*?)">(.*?)</p>',eachclass,re.S).group(2)
        timeandlevel = re.findall('<em>(.*?)</em>',eachclass,re.S)
        info['classtime'] = timeandlevel[0]
        info['classlevel'] = timeandlevel[1]
        return info
    def saveinfo(self,classinfo):
        f = open('info.txt','a')
        for each in classinfo:
            f.writelines('title: ' + each['title'] + '\n')
            f.writelines('des: ' + each['des'] + '\n')
            f.writelines('time: ' + each['classtime'] + '\n')
            f.writelines('level: ' + each['classlevel'] + '\n')
            f.writelines('\n\n')
        f.close()

if __name__ == '__main__':
    classinfo=[]
    url = 'http://www.jikexueyuan.com/course/?pageNum=1'
    jikespider = spider() # 类的实例 jikespider
    all_links = jikespider.changepage(url,3) # 得到所有的链接
    # print all_links # 测试
    for link in all_links:
        print '正在处理页面：' + link
        html = jikespider.getsource(link)
        everyclass = jikespider.geteveryclass(html)
        # print everyclass # 得到网页内容测试
        for each in everyclass:
            info = jikespider.getinfo(each)
            classinfo.append(info)
    jikespider.saveinfo(classinfo)

转载于:https://my.oschina.net/houzhong/blog/625400