爬取猫眼电影Top100

import requests
import re
import json
import time


def get_one_page(url):	#获取页面内容
    headers = {
        'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0'
    }
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        return response.text
    return None


def grb_message(html):	#利用正则表达式获取想要的内容
    ruler = '<dd.*?board-index.*?>(.*?)</i>.*?<img data-src="(.*?)".*?name.*?a.*?>(.*?)</a>'
    ruler += '.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>'
    pattern = re.compile(ruler, re.S)
    items = re.findall(pattern, html)
    return items


def parse_result(results):	#整理数据，使数据更规整一些
    for result in results:
        yield{
            'index': result[0],
            'image': result[1],
            'title': result[2],
            'actor': result[3],
            'time': result[4],
            'score': result[5].strip() + result[6].strip()
        }


def write_file(content):	#写到文件当中
    with open('result.txt', 'a', encoding='utf-8') as f:
        print(type(json.dumps(content)))
        f.write(json.dumps(content, ensure_ascii=False)+'\n')


def main():
	top = 100; #top 100 
	for offset in range(10):
		url = 'https://maoyan.com/board/4?offset=' + str(offset * 10)	#第一页是０，第二页是10.....
		html = get_one_page(url)
		results = grb_message(html)

		for item in parse_result(results):
			write_file(item)
			
		#读取速度过快会被屏蔽，因此休眠１s	
		time.sleep(1)


main()