注意:类的方法声名时,必须加上self变量(不管方法中有没有用到,如下例中的方法getHtml())
脚本如下:
#-*- coding:utf-8 -*-
#import urllib
import urllib2
import re
import sys
import cookielib
reload(sys)
sys.setdefaultencoding("utf-8")
class getTheLine(object):
def __init__(self,url,re_compile,filePath,pages):
self.url = url
self.re_compile = re_compile
self.filePath = filePath
self.pages = pages
def getHtml(self,html_url):
cookie_support = urllib2.HTTPCookieProcessor(cookielib.CookieJar())
#opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
opener = urllib2.build_opener(cookie_support)
user_agents = ['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36']
opener.addheaders = [("User-agent", user_agents), ("Accept", "*/*"), ('Referer', 'http://www.douban.com')]
response = opener.open(html_url)
html = response.read().decode("utf-8")
return html
def getTitle(self,html_url):
#reg = u'class="note-item">.*?<a.*?href=(.*?)class="title".*?target="_blank">(.*?)</a>.*?<span>(.*?)喜欢</span>'
#titleRe = re.compile(reg,re.S)
titlelist = re.findall(self.re_compile,self.getHtml(html_url))
return titlelist
def howManyPages(self):
page_num = 0
#filePath = r'C:\Users\Administrator\tmp\DoubanTop250.txt'
while page_num < self.pages:
#html_url = 'https://www.douban.com/tag/%E8%A3%85%E4%BF%AE/article?start=' + str(page_num*15)
html_url = self.url + '?start=' + str(page_num*15)
page_num = page_num + 1
Contents = self.getTitle(html_url)
if page_num == 1:
files_work = open(self.filePath, 'w')
else:
files_work = open(self.filePath, 'a')
for Content in Contents:
if int(Content[2]) >= 1500:
#if 1:
files_work.write(Content[2] + '人喜欢' + '\r')
files_work.write('Title:' + Content[1] + '\r')
files_work.write('Link:' + Content[0] + '\r')
files_work.write('from the ' + str(page_num) + ' page' + '\r\n')
print 'Read the ' + str(page_num) + ' page successful...'
files_work.close()
reg = u'class="note-item">.*?<a.*?href=(.*?)class="title".*?target="_blank">(.*?)</a>.*?<span>(.*?)喜欢</span>'
titleRe = re.compile(reg,re.S)
filePath = r'C:\Users\Administrator\tmp\article.txt'
html_url = 'https://www.douban.com/tag/%E8%A3%85%E4%BF%AE/article'
pages = 10
getTheLine(html_url,titleRe,filePath,pages).howManyPages()