糗百利用urllib2库进行操作,需要headers验证。
url是读取的页面。
写的程序可能因为糗百的网页结构更新导致正则表达式匹配错误,从而出错或者抓取不到。
程序敬请参考,有意见和问题欢迎讨论
#!/user/bin/env python
# coding=utf-8
import urllib
import urllib2
import re
import sys
defauletencoding = 'utf-8'
class QSBK:
def __init__(self):
self.pageIndex = 1
self.user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:51.0) Gecko/20100101 Firefox/51.0"
self.headers = {"User-Agent":self.user_agent}
self.stories = []
def getPage(self, pageIndex):
try:
url = "http://www.qiushibaike.com/hot/page/" + str(pageIndex) + "?s=4955345"
request = urllib2.Request(url, headers=self.headers)
response = urllib2.urlopen(request)
context = response.read().decode("utf-8")
return context
except urllib2.URLError, e:
if hasattr(e, "reason"):
print e.reason
return None
def getPageItems(self, pageIndex):
context = self.getPage(pageIndex)
if not context:
print "加载出错"
return None
pattern = re.compile(
'<div class="article block untagged mb15".*?<h2>(.*?)</h2>.*?<div class="article.*?">(.*?)</div>.*?<span>(.*?)</span>.*?</a>(.*?)<div class="stats',
re.S)
items = re.findall(pattern, context)
pageStories = []
for item in items:
havaImg = re.search("img", item[3])
if not havaImg:
replaceBR = re.compile("<br/>")
text = re.sub(replaceBR, "\n", item[2])
store = {"name":item[0], "age":item[1], "content":text}
pageStories.append(store)
return pageStories
def start(self):
print "开始读取段子"
pageStories = self.getPageItems(1)
for page in pageStories:
print "昵称 :" + page['name']
print "年龄 :" + page['age']
print "段子 :" + page['content']
print "\n\n"
reload(sys)
sys.setdefaultencoding(defauletencoding)
spider = QSBK()
spider.start()