# -*- coding: utf-8
import urllib2
import urllib
import re,os
import time
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
class Tiebar:
#初始化数据
def __init__(self,base_url,see_lz):
self.base_url = base_url
self.see_lz = '?see_lz=' + see_lz
self.page = 1
self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
self.headers = { 'User-Agent' : self.user_agent }
self.tool = Tool()
self.out_put_file = 'd:/python/test/out.txt'
#获取页面内容的方法
def get_cotent(self,page):
try:
url = self.base_url + self.see_lz + '&pn=' + str(page)
request = urllib2.Request(url,headers=self.headers)
response = urllib2.urlopen(request)
act_url = response.geturl()
print 'init url=',url,'act url=',act_url
if url == act_url:
content = response.read()
return content
else:
return None
except urllib2.URLError, e:
if hasattr(e,"reason"):
print u"连接贴吧页面失败,错误原因",e.reason
return None
#获取帖子主题
def get_titile(self):
content = self.get_cotent(1)
pattern = re.compile('<h3 .*?>(.*?)</h3>')
result = re.search(pattern,content)
if result:
return result.group(1).strip()
else:
return None
#获取帖子的页数
def get_page_num(self):
content = self.get_cotent(1)
pattern = re.compile('<li class="l_reply_num.*?.*?<span.*?>(.*?)',re.S)
result = re.search(pattern,content)
if result:
return result.group(1).strip()
else:
return None
#获取帖子内容
def get_tiebar(self,page):
content = self.get_cotent(page).decode('utf-8')
str = ''
if not content:
print "抓取完毕"
return None
patt_content = re.compile('<a data-field=.*?class="p_author_name j_user_card".*?>(.*?)</a>.*?'
+ '<div id=".*?" class="d_post_content j_d_post_content "> '
+ '(.*?)
import urllib2
import urllib
import re,os
import time
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
class Tiebar:
#初始化数据
def __init__(self,base_url,see_lz):
self.base_url = base_url
self.see_lz = '?see_lz=' + see_lz
self.page = 1
self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
self.headers = { 'User-Agent' : self.user_agent }
self.tool = Tool()
self.out_put_file = 'd:/python/test/out.txt'
#获取页面内容的方法
def get_cotent(self,page):
try:
url = self.base_url + self.see_lz + '&pn=' + str(page)
request = urllib2.Request(url,headers=self.headers)
response = urllib2.urlopen(request)
act_url = response.geturl()
print 'init url=',url,'act url=',act_url
if url == act_url:
content = response.read()
return content
else:
return None
except urllib2.URLError, e:
if hasattr(e,"reason"):
print u"连接贴吧页面失败,错误原因",e.reason
return None
#获取帖子主题
def get_titile(self):
content = self.get_cotent(1)
pattern = re.compile('<h3 .*?>(.*?)</h3>')
result = re.search(pattern,content)
if result:
return result.group(1).strip()
else:
return None
#获取帖子的页数
def get_page_num(self):
content = self.get_cotent(1)
pattern = re.compile('<li class="l_reply_num.*?.*?<span.*?>(.*?)',re.S)
result = re.search(pattern,content)
if result:
return result.group(1).strip()
else:
return None
#获取帖子内容
def get_tiebar(self,page):
content = self.get_cotent(page).decode('utf-8')
str = ''
if not content:
print "抓取完毕"
return None
patt_content = re.compile('<a data-field=.*?class="p_author_name j_user_card".*?>(.*?)</a>.*?'
+ '<div id=".*?" class="d_post_content j_d_post_content "> '
+ '(.*?)
转载于:http://blog.itpub.net/10972173/viewspace-2133595/