Python2.7Mac OS
抓取的是电影天堂里面最新电影的页面。链接地址: http://www.dytt8.net/html/gndy/dyzz/index.html
获取页面的中电影详情页链接
import urllib2
import os
import re
import string
# 电影URL集合
movieUrls = []
# 获取电影列表
def queryMovieList():
url = 'http://www.dytt8.net/html/gndy/dyzz/index.html'
conent = urllib2.urlopen(url)
conent = conent.read()
conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')
pattern = re.compile ('
.*?>
'(.*?)
',re.S)items = re.findall(pattern,conent)
str = ''.join(items)
pattern = re.compile ('(.*?).*?
(.*?)',re.S)news = re.findall(pattern, str)
for j in news:
movieUrls.append('http://www.dytt8.net'+j[0])
抓取详情页中的电影数据
def queryMovieInfo(movieUrls):
for index, item in enumerate(movieUrls):
print('电影URL: ' + item)
conent = urllib2.urlopen(item)
conent = conent.read()
conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')
movieName = re.findall(r'
(.*?)
if (len(movieName) > 0):
movieName = movieName[0] + ""
# 截取名称
movieName = movieName[movieName.find("《") + 3:movieName.find("》")]
else:
movieName = ""
print("电影名称: " + movieName.strip())
movieContent = re.findall(r'
pattern = re.compile('
- (.*?)', re.S)
movieDate = re.findall(pattern,movieContent[0])
if (len(movieDate) > 0):
movieDate = movieDate[0].strip() + ''
else:
movieDate = ""
print("电影发布时间: " + movieDate[-10:])
pattern = re.compile('
(.*?)
movieInfo = re.findall(pattern, movieContent[0])
if (len(movieInfo) > 0):
movieInfo = movieInfo[0]+''
# 删除
标签
movieInfo = movieInfo.replace("
","")
# 根据 ◎ 符号拆分
movieInfo = movieInfo.split('◎')
else:
movieInfo = ""
print("电影基础信息: ")
for item in movieInfo:
print(item)
# 电影海报
pattern = re.compile('', re.S)
movieImg = re.findall(pattern,movieContent[0])
if (len(movieImg) > 0):
movieImg = movieImg[0]
else:
movieImg = ""
print("电影海报: " + movieImg)
pattern = re.compile('
.*?', re.S)movieDownUrl = re.findall(pattern,movieContent[0])
if (len(movieDownUrl) > 0):
movieDownUrl = movieDownUrl[0]
else:
movieDownUrl = ""
print("电影下载地址:" + movieDownUrl + "")
print("------------------------------------------------\n\n\n")
执行抓取
if __name__=='__main__':
print("开始抓取电影数据");
queryMovieList()
print(len(movieUrls))
queryMovieInfo(movieUrls)
print("结束抓取电影数据")
总结
学好正则表达式很重要,很重要,很重要!!!! Python的语法好有感觉, 对比Java …