import requests import re import json import time from requests.exceptions import RequestException """猫眼电影Top100爬取""" def get_one_page(url): """请求页面,若成功则返回页面的内容""" try: # 设置请求头, headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3)\ AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36' } response = requests.get(url, headers=headers) # 若请求成功,返回响应的内容 if response.status_code == 200: # print(response.text) return response.text return None # 出现异常返回None except RequestException: return None def parse_one_page(html): """解析页面,提取需要的信息""" pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name.*?a.*?>(.*?)</a>.*?' 'star.*?>(.*?)</p>.*?releasetime.*?>(.*?)</p>.*?integer.*?>' '(.*?)</i>.*?fraction.*?>(\d*)</i>.*?</dd>', re.S) # 找出所有符合正则式的信息 items = re.findall(pattern, html) for item in items: # print(item) # yield用于生成一个可迭代器 yield { 'index': item[0], 'image': item[1], 'name': item[2].strip(), 'actor': item[3].strip()[3:] if len(item[3]) > 3 else '', 'time': item[4].strip()[5:] if len(item[4]) > 5 else '', 'score': item[5].strip()+item[6].strip(), } def write_to_file(content): """写入本地文件""" with open('result.txt', 'a', encoding='utf-8') as f: f.write(json.dumps(content, ensure_ascii=False)+'\n') def main(offset): """主函数,用于调用其他函数""" url = 'http://maoyan.com/board/4?offset=' + str(offset) html = get_one_page(url) for item in parse_one_page(html): # print(item) write_to_file(item) if __name__ == '__main__': for i in range(10): main(offset=i*10) time.sleep(1)
猫眼电影Top100爬取
最新推荐文章于 2021-12-24 10:11:12 发布