爬取糗事百科

自己写的爬取代码，还需进一步优化，仅供参考

import requests
import re
import pandas as pd

class spider():
    def numbers(link):
        number=('正在爬取：'+link+'\n\n')
        return(number)

    def page(url,totalpage):
        changepage=[]
        for i in range(1,totalpage+1):
            page_number=re.sub('/page/\d+','/page/%s'%i,url,re.S)
            changepage.append(page_number)
        return(changepage)

    def htmls(url):         #网页

        html=requests.get(url)
        txt=html.text
        return(txt)

    def offer(home):
        whole=re.findall('<div class="article block untagged mb15"(.*?)<div class="single-clear">',txt,re.S)
        return(whole)

    def author(source):
        name=re.findall('<h2>(.*?)</h2>',source,re.S)
        return('发布人：'+name[0]+'\n\n')


    def contents(cont):
        content=re.findall('<div class="content">(.*?)</div>',cont,re.S)
        return('发布内容：'+content[0])


if __name__=='__main__':

    url='http://www.qiushibaike.com/8hr/page/1/?s=4897603'
    links=spider.page(url,3)

    file=open('jiusihi.txt','w')

    info=[]
    for link in links:
        prin=spider.numbers(link)
        info.append(prin)

        txt=spider.htmls(link)
        datas=spider.offer(txt)
        for data in datas:
            print1=spider.author(data)
            info.append(print1)
            print2=spider.contents(data)
            info.append(print2)
    for each in info:
        file.write(each)
    file.close()