我的第一个爬虫

我的第一个爬虫

# coding:utf-8

#爬糗事百科等段子，有如下功能，自动获得总页数，爬取每页段子过滤掉图片段子，自动创建TXT文件，将每页段子写入对应文件
import urllib
import urllib2
import re
import sys


reload(sys)
sys.setdefaultencoding('utf-8')
# 有这个才能f.write()写入TXT文件，不知道具体原因可能事编码问题


class QSBK:
def __init__(self):
self.user_agent = 'Mozilla/4.0(compatible;MSIE 5.5;windows NT)'
self.headers = {'User-Agent':self.user_agent}
# 设置头部

def getPage(self):
url = 'http://www.qiushibaike.com/hot/page'
request = urllib2.Request(url,headers = self.headers)
resqonse = urllib2.urlopen(request)
pageCode = resqonse.read().decode('utf-8')
return pageCode

def getPageNum(self):

pageCode = self.getPage()
pagenumpattern = re.compile('<span class=\"page-numbers\">(.*?)</span>',re.S)
pagenumitems = re.findall(pagenumpattern,pageCode)
pagenum = int(pagenumitems[-1])
#获得列表最后一项即为总页码
return pagenum




def start(self):
pagenum = self.getPageNum()

for i in range(1,pagenum+1):
num = "%s.txt"%i
f = open(num,"w")
url = 'http://www.qiushibaike.com/hot/page/' + str(i)
request = urllib2.Request(url,headers = self.headers)
resqonse = urllib2.urlopen(request)
pageCode = resqonse.read().decode('utf-8')
pattern = re.compile('<div class="content".*?span>(.*?)</span>.*?<!--.*?>(.*?)</div>',re.S)
items = re.findall(pattern,pageCode)


for item in items:
haveImg = re.search("img",item[1])
if not haveImg:
f.write(item[0])
f.close



s = QSBK()
s.start()