正则表达式 爬取

#!/usr/bin/env python
# _*_ coding utf-8  _*_
# time:09-01-2017 10:55
#version:1.0
#__author__ = 'lijin'

import requests
import re

class Spider(object):
    def loadpage(self, page):
        #生成url
        url = 'http://www.neihan8.com/article/list_5_{}.html'.format(page)
        response = requests.get(url)
        html = response.content.decode("gbk")
        pattern = re.compile(r'<div class="f18 mb20">.*?</div>',re.S)
        items = pattern.findall(html)
        return items

    def printpage(self, items, page):
        print("******正在打印第{}页*******".format(page))
        for item in items:
            print("=========================")

            # item = item.replace("<p>", " ").replace("</p>", " ")\
            #     .replace('<div class="f18 mb20">', " ").replace("</div>", " ")\
            #     .replace('<br />','')

            item = re.sub(r"<.*?>","",item)
            item = re.sub(r"&.*?;","",item)
            self.savedata(item)
            print(item)
        print("******第{}页打印完毕*******".format(page))

    def savedata(self, item):
        with open("段子.txt","a") as f:
            f.write("====================")
            f.write(item)

    def main(self, page):
        items = self.loadpage(page)
        self.printpage(items, page)


spider = Spider()
for i in range(1,2):
    spider.main(i)

转载于:https://www.cnblogs.com/xidou/p/7488388.html