糗事百科爬虫改进

无事，抓糗事！

看到一个哥们的代码，无事拿来改改，抓糗事百科文字内容

#!/usr/bin/env python
'''
for qiushibaike.com
'''

import  urllib2
# import  urllib
import  re
import  thread
import  time


class Spider_Model():
    def __init__(self):
        self.page = 1
        self.pages = []
        self.enable = False

    def GetPage(self,page):
        myurl = r'http://www.qiushibaike.com/textnew/page/'+page
        user_agent = 'Mozilla/5.0 (X11; Linux x86_64)'
        headers = {'User-Agent':user_agent}

        req = urllib2.Request(myurl,headers=headers)
        myres = urllib2.urlopen(req)
        mypage = myres.read()


        unicodepage = mypage.decode('utf-8')
        myItems = re.findall('<div.*?class="content">(.*?)<!--.*?-->.*?</div>',unicodepage,re.S)
        Items = []
        # print myItems

        for item in myItems:
            # print  item
            item = item.replace('\n','')
            Items.append(item.replace(r'<br/>','\n'))
            # Items.append(item[0])

        return  Items

    def LoadPage(self):
        while self.enable:
            if len(self.pages) < 2:
                try:
                    mypage = self.GetPage(str(self.page))
                    self.page += 1
                    self.pages.append(mypage)
                except:
                    print 'can not connected to the url.'
            else:
                time.sleep(1)

    def ShowPage(self,nowPage,page):
        print '\n\n############################ Page %d #################################\n\n' % page

        for item in nowPage:
            print item

            myinput = raw_input()
            if myinput == 'quit':
                self.enable = False
                break

    def start(self):
        page = self.page
        self.enable = True

        print(u'waiting..............')

        thread.start_new_thread(self.LoadPage,())

        while self.enable:
            if self.pages:
                nowpage = self.pages[0]
                del self.pages[0]
                self.ShowPage(nowpage,page)
                page +=1


if __name__ == '__main__':
    #---------the begin of program-----------------
    print u'''
    -------------------------------------------------
    xxxx
    x
    xxx
    xxx
    -------------------------------------------------
    '''

    print 'Press any key,to continue......'
    raw_input()
    mymodel = Spider_Model()
    mymodel.start()

一切从简。不解释不说明，随便拍！

详细内容请参考：http://blog.csdn.net/pleasecallmewhy/article/details/8932310