python百度贴吧 发帖时间_python爬取贴吧帖子

一、介绍

我们常遇到一些很长的贴吧连载帖子想存到本地再看

此文就是运用python爬取指定百度贴吧的帖子并存到本地满足需求

环境:python2.7

目标网页:【长篇连载】剑网3的正史和野史——从头开始讲剧情故事

源码存放:源码github

本文参考:静觅博客python实战系列

二、页面的抓取

目标网页网址为https://tieba.baidu.com/p/2196794546

满足可以选择是否只看楼主的抓取我们点一下 只看楼主 并点一下后页的链接来获取只看楼主和翻页的参数

这时候可以看到只看楼主多出的参数 see_lz 和当前页的参数 pn

接下来定义一个DEMO类开始获取整个网页

# -*- coding:utf-8 -*-

import urllib

import urllib2

import re

class DEMO:

def __init__(self,baseUrl):

self.baseURL = baseUrl

def getPage(self):

url = baseURL

request = urllib2.Request(url)

response = urllib2.urlopen(request)

print response.read()

baseURL = 'https://tieba.baidu.com/p/2196794546?see_lz=1&pn=3'

demo = DEMO(baseURL)

demo.getPage()

运行结果:

爬取网页代码成功后我们完善一下代码,将只看楼主和页码提为参数

def __init__(self,baseUrl,seeLZ):

self.baseURL = baseUrl

self.seeLZ = '?see_lz=' +str(seeLZ)

self.tool = Tool()

def getPage(self,pageNum):

try:

url = self.baseURL+self.seeLZ + '&pn=' + str(pageNum)

request = urllib2.Request(url)

response = urllib2.urlopen(request)

return response.read().decode('utf-8')

except urllib2.URLError, e:

if hasattr(e,"reson"):

print u"链接失败,原因:",e.reason

return None

三、提取出想要的信息

打开目标网页审查元素(F12)

首先找到帖子标题的代码段:

【长篇连载】剑网3的正史和野史——从头开始讲剧情故事

我们将提取此

里的文本则正则表达式为:

(.*?)

',re.S)

result = re.search(pattern,page)

if result:

return result.group(1).strip()

else:

return None

同理找到帖子页数的代码段,可得出提取页数的正则:

.*?(.*?)

提取页数的方法:

def getPageNum(self,page):

page = self.getPage(1)

pattern = re.compile('

.*?(.*?)',re.S)

result = re.search(pattern,page)

if result:

return result.group(1).strip()

else:

return None

接下来是我们主要目的,提取正文,正则为:

removeImg = re.compile('| {7}|')

removeAddr = re.compile('|')

replaceLine = re.compile('

|

|

|')

replaceTD = re.compile('

')

replacePara = re.compile('

')

replaceBR = re.compile('

')

replaceBR = re.compile('
|
')

removeExtraTag = re.compile('<.>

def replace(self,x):

x = re.sub(self.removeImg,"",x)

x = re.sub(self.removeAddr,"",x)

x = re.sub(self.replaceLine,"\n",x)

x = re.sub(self.replaceTD,"\t",x)

x = re.sub(self.replacePara,"\n ",x)

x = re.sub(self.replaceBR,"\n",x)

x = re.sub(self.removeExtraTag,"",x)

return x.strip()

初始化类后,我们可以写提取正文的方法:

def getContent(self,page):

pattern = re.compile('

contents.append(content.encode('utf-8'))

return contents

四、写入文件,保存为txt

def setFileTitle(self,title):

if title is not None:

self.file = open(title + ".txt","w+")

else:

self.file = open(self.defaultTitle + ".txt","w+")

def writeData(self,contents):

for item in contents:

self.file.write(item)

五、完善代码

有了以上的方法作为基础，我们来写运行的方法:

def start(self):

indexPage = self.getPage(1)

pageNum = self.getPageNum(indexPage)

title = self.getTitle(indexPage)

self.setFileTitle(title)

if pageNum == None:

print "URL已失效,请重试"

return None

try:

print "该帖子共有" + str(pageNum) + "页"

for i in range(1,int(pageNum)+1):

print "正在写入第" + str(i) + "页数据"

page = self.getPage(i)

contents = self.getContent(page)

self.writeData(contents)

except IOError,e:

print "写入异常,原因" + e.message

finally:

print "写入任务完成"

此时完成的代码:

# -*- coding:utf-8 -*-

import urllib

import urllib2

import re

class Tool:

removeImg = re.compile('| {7}|')

removeAddr = re.compile('|')

replaceLine = re.compile('

|

|

|')

replaceTD = re.compile('

')

replacePara = re.compile('

')

replaceBR = re.compile('

')

replaceBR = re.compile('
|
')

removeExtraTag = re.compile('<.>

def replace(self,x):

x = re.sub(self.removeImg,"",x)

x = re.sub(self.removeAddr,"",x)

x = re.sub(self.replaceLine,"\n",x)

x = re.sub(self.replaceTD,"\t",x)

x = re.sub(self.replacePara,"\n ",x)

x = re.sub(self.replaceBR,"\n",x)

x = re.sub(self.removeExtraTag,"",x)

return x.strip()

class DEMO:

def __init__(self,baseUrl,seeLZ):

self.baseURL = baseUrl

self.seeLZ = '?see_lz=' +str(seeLZ)

self.tool = Tool()

def getPage(self,pageNum):

try:

url = self.baseURL+self.seeLZ + '&pn=' + str(pageNum)

request = urllib2.Request(url)

response = urllib2.urlopen(request)

return response.read().decode('utf-8')

except urllib2.URLError, e:

if hasattr(e,"reson"):

print u"链接失败,原因:",e.reason

return None

def getTitle(self,title):

page = self.getPage(1)

pattern = re.compile('

.*?(.*?)',re.S)

result = re.search(pattern,page)

if result:

return result.group(1).strip()

else:

return None

def getContent(self,page):

pattern = re.compile('

contents.append(content.encode('utf-8'))

return contents

def setFileTitle(self,title):

if title is not None:

self.file = open(title + ".txt","w+")

else:

self.file = open(self.defaultTitle + ".txt","w+")

def writeData(self,contents):

for item in contents:

self.file.write(item)

def start(self):

indexPage = self.getPage(1)

pageNum = self.getPageNum(indexPage)

title = self.getTitle(indexPage)

self.setFileTitle(title)

if pageNum == None:

print "URL已失效,请重试"

return None

try:

print "该帖子共有" + str(pageNum) + "页"

for i in range(1,int(pageNum)+1):

print "正在写入第" + str(i) + "页数据"

page = self.getPage(i)

contents = self.getContent(page)

self.writeData(contents)

except IOError,e:

print "写入异常,原因" + e.message

finally:

print "写入任务完成"

print u"请输入帖子代号"

baseURL = 'http://tieba.baidu.com/p/' + str(raw_input(u'http://tieba.baidu.com/p/'))

seeLZ = raw_input("是否只获取楼主发言,是输入1,否输入0\n")

demo = DEMO(baseURL,seeLZ)

demo.start()

好我们来运行一下试试:

到这里输入目标网页的代号:

整个运行过程:

好,这时候该目录下多了这个txt文件,打开看看

文本已经成功存到本地,到这里就成功了。