import requests
import re
import json
import time
def get_one_page(url): #获取页面内容
headers = {
'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0'
}
response = requests.get(url, headers=headers)
if response.status_code == 200:
return response.text
return None
def grb_message(html): #利用正则表达式获取想要的内容
ruler = '<dd.*?board-index.*?>(.*?)</i>.*?<img data-src="(.*?)".*?name.*?a.*?>(.*?)</a>'
ruler += '.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>'
pattern = re.compile(ruler, re.S)
items = re.findall(pattern, html)
return items
def parse_result(results): #整理数据,使数据更规整一些
for result in results:
yield{
'index': result[0],
'image': result[1],
'title': result[2],
'actor': result[3],
'time': result[4],
'score': result[5].strip() + result[6].strip()
}
def write_file(content): #写到文件当中
with open('result.txt', 'a', encoding='utf-8') as f:
print(type(json.dumps(content)))
f.write(json.dumps(content, ensure_ascii=False)+'\n')
def main():
top = 100; #top 100
for offset in range(10):
url = 'https://maoyan.com/board/4?offset=' + str(offset * 10) #第一页是0,第二页是10.....
html = get_one_page(url)
results = grb_message(html)
for item in parse_result(results):
write_file(item)
#读取速度过快会被屏蔽,因此休眠1s
time.sleep(1)
main()
爬取猫眼电影Top100
最新推荐文章于 2021-11-25 20:58:20 发布