python爬虫之糗百段子

ec06cumt

于 2016-02-24 10:24:58 发布

阅读量494

点赞数

分类专栏： python

本文链接：https://blog.csdn.net/ec06cumt/article/details/50728276

版权

python 专栏收录该内容

4 篇文章 0 订阅

订阅专栏

python的基础学习，参考网上的文章，修改了正则表达式，支持最新的糗百的HTML(2016/02/22)

import urllib
import urllib2
import re
import sys
import time

class jokedownTool:
    def __init__ (self,str):
        self.pageindex = 1
        self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
        self.headers = { 'User-Agent' : self.user_agent }
        self.stories = []
        self.enable = False
        self.strcomple = str
        
    def getpage(self,pageindex):
        try:
            url = 'http://www.qiushibaike.com/hot/page/' + str(pageindex)
            request = urllib2.Request(url,headers = self.headers)
            response = urllib2.urlopen(request)
            pageCode = response.read().decode('utf-8')
            return pageCode
        except urllib2.URLError, e:
            if hasattr(e,"reason"):
                print str("connection failed"),e.reason
                return None
            
    def getpageitems(self,pageindex):
        pagecode = self.getpage(pageindex)
        if not pagecode:
            print "load page failed...."
            return None
        pattern = re.compile(self.strcomple,re.S)
        items = re.findall(pattern,pagecode)
        pageStories = []
        for item in items:
            replaceBR = re.compile('<br/>')
            text = re.sub(replaceBR,"\n",item[1])
            pageStories.append([item[0].strip(),text.strip(),item[3].strip(),item[2].strip()])
        return pageStories
        
    def loadPage(self):
        if self.enable == True:
            if len(self.stories) < 2:
                pageStories = self.getpageitems(self.pageindex)
                if pageStories:
                    self.stories.append(pageStories)
                    self.pageindex += 1 
    def getOneStory(self,pageStories,page):
        for story in pageStories:
            input = raw_input()
            self.loadPage()
            if input == "Q":
                self.enable = False
                return
            print "page:%d\tAuthor:%s\tfavar:%s,date:%s\n%s" %(page,story[0],story[2],time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(story[3]))),story[1])  
            
    def start(self):
        print "reading joke press'enter'to get latest,'Q'"
        self.enable = True
        self.loadPage()
        nowPage = 0
        while self.enable:
            if len(self.stories)>0:
                pageStories = self.stories[0]
                nowPage += 1
                del self.stories[0]
                self.getOneStory(pageStories,nowPage)
                
    
    
spider = jokedownTool('<h2>(.*?)</h2>.*?<div class="content">(.*?)<!--(.*?)-->.*?<span class="stats-vote"><i class="number">(.*?)</i>')
spider.start()

按回车，输入一个段子，输入Q退出，支持分页。。。