网页下载器
import requests
from http import cookiejar
import urllib
class HtmlDownloader():
def cookie():
with open('cookie.txt','r') as f:
cookies={}
for line in f.read().split(';'):
name,value=line.strip().split('=',1)
cookies[name]=value
return cookies
def download(self,url):
if url is None:
return None
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.90 Safari/537.36 2345Explorer/9.3.2.17331',
'Referer': r'http://movie.mtime.com',
'Connection': 'keep-alive'
}
cookie=cookiejar.CookieJar()
opener=urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie))
response=opener.open(url)
r=requests.get(url,headers=headers,cookies=cookie)
if r.status_code==200:
r.encoding='utf-8'
return r.text
return None
网页解析器
import re
import json
class HtmlParser():
def parser_url(self,page_url,response):
pattern=re.compile(r'(http://movie.mtime.com/(\d+)/)')
urls=pattern.findall(response)
if urls!=None:
return list(set(urls))
else:
return '没有链接了!'
def parser_json(self,page_url,response):
pattern=re.compile(r'=(.*?);')
result=pattern.findall(response)[0]
if result!=None:
value=json.loads(result)
try:
isRelease=value.get('value')
except Exception as e:
print(e)
return None
if isRelease:
if value.get('value').get('hotValue')==None:
return self._parser_release(page_url,value)
else:
return self._parser_no_release(page_url,value,isRelease=2)
else:
return self._parser_no_release(page_url,value)
def _parser_release(self,page_url,value):
try:
isRelease=1
movieRating=value.get('value').get('movieRating')
boxOffice=value.get('value').get('boxOffice')
movieTitle=value.get('value').get('movieTitle')
RPictureFinal=movieRating.get('RPictureFinal')
RStoryFinal=movieRating.get('RStoryFinal')
RDirectorFinal=movieRating.get('RDirectorFinal')
ROtherFinal=movieRating.get('ROtherFinal')
RatingFinal=movieRating.get('RatingFinal')
MovieId=movieRating.get('MovieId')
Usercount=movieRating.get('Usercount')
AttitudeCount=movieRating.get('AttitudeCount')
if boxOffice!=None:
TotalBoxOffice=boxOffice.get('TotalBoxOffice')
TotalBoxOfficeUnit=boxOffice.get('TotalBoxOfficeUnit')
TodayBoxOffice=boxOffice.get('TodayBoxOffice')
TodayBoxOfficeUnit=boxOffice.get('TodayBoxOfficeUnit')
ShowDays=boxOffice.get('ShowDays')
try:
Rank=boxOffice.get('Rank')
except Exception as e:
Rank=0
return (MovieId,movieTitle,RatingFinal,ROtherFinal,RPictureFinal,RDirectorFinal,RStoryFinal,Usercount,AttitudeCount,
TotalBoxOffice+TotalBoxOfficeUnit,TodayBoxOffice+TodayBoxOfficeUnit,Rank,ShowDays,isRelease)
else:
Rank=0
return (MovieId,movieTitle,RatingFinal,ROtherFinal,RPictureFinal,RDirectorFinal,RStoryFinal,Usercount,AttitudeCount,u'无',u'无',Rank,0,isRelease)
except Exception as e:
print(e,page_url,value)
return None
def _parser_no_release(self,page_url,value,isRelease=0):
try:
movieRating=value.get('value').get('movieRating')
movieTitle=value.get('value').get('movieTitle')
RPictureFinal=movieRating.get('RPictureFinal')
RStoryFinal=movieRating.get('RStoryFinal')
RDirectorFinal=movieRating.get('RDirectorFinal')
ROtherFinal=movieRating.get('ROtherFinal')
RatingFinal=movieRating.get('RatingFinal')
MovieId=movieRating.get('MovieId')
Usercount=movieRating.get('Usercount')
AttitudeCount=movieRating.get('AttitudeCount')
try:
Rank=value.get('value').get('hotValue').get('Ranking')
except Exception as e:
Rank=0
return (MovieId,movieTitle,RatingFinal,ROtherFinal,RPictureFinal,RDirectorFinal,RStoryFinal,Usercount,AttitudeCount,u'无',u'无',Rank,0,isRelease)
except Exception as e:
print(e,page_url,value)
return None
##数据存储器
import sqlite3
class DataOutput():
def __init__(self):
self.cx=sqlite3.connect('/home/as/test.db')
self.create_table('MTime')
self.datas=[]
def create_table(self,table_name):
values='''
id integer primary key,
MovieId integer,
MovieTitle varchar(40) NOT NULL,
RatingFinal REAL NOT NULL DEFAULT 0.0,
ROtherFinal REAL NOT NULL DEFAULT 0.0,
RPictureFinal REAL NOT NULL DEFAULT 0.0,
RDirectorFinal REAL NOT NULL DEFAULT 0.0,
RStoryFinal REAL NOT NULL DEFAULT 0.0,
Usercount integer NOT NULL DEFAULT 0,
AttitudeCount integer NOT NULL DEFAULT 0,
TotalBoxOffice varchat(20) NOT NULL,
TodayBoxOffice varchat(20) NOT NULL,
Rank integer NOT NULL DEFAULT 0,
ShowDays integer NOT NULL DEFAULT 0,
isRelease integer NOT NULL
'''
self.cx.execute('CREATE TABLE IF NOT EXISTS %s ( %s ) '%(table_name, values))
def store_data(self,data):
if data is None:
return
self.datas.append(data)
if len(self.datas)>10:
self.output_db('MTime')
def output_db(self,table_name):
for data in self.datas:
self.cx.execute('INSERT INTO %s (MovieId,MovieTitle,RatingFinal,'
'ROtherFinal,RPictureFinal,RDirectorFinal,RStoryFinal,'
'Usercount,AttitudeCount,TotalBoxOffice,TodayBoxOffice,'
'Rank,ShowDays,isRelease) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
''%table_name,data)
self.datas.remove(data)
self.cx.commit()
def output_end(self):
if len(self.datas)>0:
self.output_db('MTime')
self.cx.close()
##爬虫调度器
import time
class SpiderMan():
def __init__(self):
self.downloader=HtmlDownloader()
self.parser=HtmlParser()
self.output=DataOutput()
def crawl(self,root_url):
content=self.downloader.download(root_url)
urls=self.parser.parser_url(root_url,content)
for url in urls:
try:
t=time.strftime('%Y%-m%-d%H%M%S3282',time.localtime())
rank_url='http://service.library.mtime.com/Movie.api'\
'?Ajax_CallBack=true'\
'&Ajax_CallBackType=Mtime.Library.Services'\
'&Ajax_CallBackMethod=GetMovieOverviewRating'\
'&Ajax_CrossDomain=1'\
'&Ajax_RequestUrl=%s'\
'&t=%s'\
'&Ajax_CallBackArgument0=%s'%(url[0],t,url[1])
rank_content=self.downloader.download(rank_url)
data=self.parser.parser_json(rank_url,rank_content)
self.output.store_data(data)
except Exception as e:
print(e)
spider=SpiderMan()
spider.crawl('http://theater.mtime.com/China_Beijing/')
更多爬虫实例请见 https://blog.csdn.net/weixin_39777626/article/details/81564819