Python3 学习札记（五）_path = r'labels-CSDN博客

本文链接：https://blog.csdn.net/getf2016/article/details/77503397

参考‘逆風的薔薇’的教程，《Python3爬虫》-单线程爬取我的CSDN全部博文

代码


import urllib.request, re, time, random, gzip

# 定义保存文件函数
def saveFile(data, i):
    # path = r'E:\Python\VSCode_Python\CSDN\project\05_papers\papers_"+str(i+1)+".txt'
    path = "E:\\Python\\VSCode_Python\\CSDN\\project\\05_papers\\paper_"+str(i+1)+".txt"
    file = open(path, 'wb')
    page = '当前页:'+str(i+1)+'\n'
    # 换行符采用\n，则生成的txt文件用Windows自带的记事本打开后没有换行，需要用其他文本编辑器，如果换行符采用\r\n即回车换行，则自带记事本也可以
    file.write(page.encode('gbk'))
    # 将博文信息写入文件（以utf-8保存的文件声明为gbk）
    for d in data:
        d =  str(d)+'\n'
        file.write(d.encode('gbk'))
    file.close()

# 解压缩数据
def ungzip(data):
    try:
        data = gzip.decompress(data)
    except:
        print('未经压缩，无需解压...')
    return data

# CSDN爬虫类
class CSDNSpider:

    def __init__(self, pageIdx = 1, url = 'http://blog.csdn.net/fly_yr/article/list/1'):
        self.pageIdx = pageIdx
        self.url = url[0:url.rfind('/') + 1] + str(pageIdx) # rfind() 返回字符串最后一次出现的位置(从右向左查询)，如果没有匹配项则返回-1
        self.headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3141.7 Safari/537.36',
            # 'Referer': 'http://blog.csdn.net/fly_yr/article/details/72187057',
            'Upgrade-Insecure-Requests': '1',
            'Host': 'blog.csdn.net',
            'Proxy-Connection': 'keep-alive'
        }

    def getPages(self):
        req = urllib.request.Request(url=self.url, headers=self.headers)
        res = urllib.request.urlopen(req)
        # 解压缩
        data = res.read()
        data = ungzip(data)
        data = data.decode('utf-8')

        pages = r'<div.*?pagelist">.*?<span>.*?共(.*?)页</span>'
        # 计算博文总页数
        pattern = re.compile(pages, re.DOTALL)
        pagesNum = re.findall(pattern, data)[0]
        return pagesNum

    # 设置要抓取的博文页面
    def setPage(self, idx):
        self.url = self.url[0:self.url.rfind('/')+1] + str(idx)

    # 读取博文信息
    def readData(self):
        ret = []
        # chrome浏览器 开发者模式查看源代码，从而确定正则表达式
        str = r'<span class="link_title"><a href="(.*?)">\s*(.*?)\s*</a>.*?<span class="link_postdate">(.*?)</span>.*?'+\
              r'<span class="link_view" title="阅读次数">.*?阅读</a>\((.*?)\)</span>.*?'+\
              r'<span class="link_comments" title="评论次数">.*?评论</a>\((.*?)\)</span>'
        req = urllib.request.Request(url=self.url, headers=self.headers)
        res = urllib.request.urlopen(req)

        # 内容解压缩
        data = res.read()
        data = ungzip(data)
        data = data.decode('utf-8')
        pattern = re.compile(str, re.DOTALL)    # (DOTALL模式下，.也能匹配换行符)
        items = re.findall(pattern, data)
        for item in items:
            # ret.append(item[0]+'年'+item[1]+'月'+item[2]+'日'+'\t'+item[3]+'\n标题'+item[5]
            # +'\n连接：http://blog.csdn.net'+item[4]
            # +'\n'+'阅读：'+item[6]+'\t评论：'+item[7]+'\n')
            ret.append('时间：'+item[2]+'\n'+'标题：'+item[1]
            +'\n'+'链接：http://blog.csdn.net'+item[0]
            +'\n'+'阅读：'+item[3]+'\t'+'评论：'+item[4]+'\n')
        return ret

cs = CSDNSpider()   # 实例化一个类时，只执行__init__
pagesNum = int(cs.getPages())
print('博文总页数：', pagesNum)

for idx in range(pagesNum):
    cs.setPage(idx)
    print('当前页：', idx+1)
    papers = cs.readData()
    saveFile(papers, idx)