#19.5.14

贴吧的爬虫，逻辑如下↓

#--coding:utf-8--

import urllib2

def write_file(file_name, text):
print ‘正在存储文件’ + file_name
f = open(file_name, ‘w+’)
f.write(text)
f.close()

def load_page(url):
req = urllib2.Request(url)
response = urllib2.urlopen(req)
html = response.read()
return html

def tieba_spider(url, begin_page, end_page):
user_agent =‘Mozilla/5.0 (compatible; MSIE 9.0; Windows NT6.1; Trident/5.0;’ #IE
9.0 headers = {‘User-Agent’:user_agent}

for i in range(begin_page, end_page+1):
pn = 50 * (i - 1)
html = load_page(url+str(pn))

  file_name = str(i) +'.html'
  print '正在下载第' + str(i) + '个网页'
  write_file(file_name, html)

if name == ‘main’:
bdurl = str(raw_input(‘请输入贴吧的地址, 去掉pn=后面的数字:’))
begin_page = int(raw_input(‘请输入开始的页数’))
end_page = int(raw_input(‘请输入结束的页数’))
tieba_spider(bdurl, begin_page, end_page)

------分割线------
*part1：写入本地

def write_file(file_name, text):
print ‘正在存储文件’ + file_name
f = open(file_name, ‘w+’)
f.write(text)
f.close()

*part2：爬取 （昨天的改版）

def load_page(url):
req = urllib2.Request(url)
response = urllib2.urlopen(req)
html = response.read()
return html

*part3：伪装头 伪装火狐浏览器

def tieba_spider(url, begin_page, end_page):
user_agent =‘Mozilla/5.0 (compatible; MSIE 9.0; Windows NT6.1; Trident/5.0;’ #IE
9.0 headers = {‘User-Agent’:user_agent}

*part4：核心部分（pn处理）

for i in range(begin_page, end_page+1):
pn = 50 * (i - 1)
html = load_page(url+str(pn))

------分割线------
展望：
尚未有过滤器，代码还要再熟一点。