python3 requests + 正则表达式抓取猫眼电影

最新推荐文章于 2020-02-27 09:05:37 发布

Hold2u

最新推荐文章于 2020-02-27 09:05:37 发布

阅读量258

点赞数

分类专栏： Python

本文链接：https://blog.csdn.net/Hold2u/article/details/86545598

版权

Python 专栏收录该内容

6 篇文章 0 订阅

订阅专栏

import requests
from requests.exceptions import RequestException
import re
import json
from multiprocessing import Pool
def get_one_page(url):
        try:
                response = requests.get(url)
                if response.status_code == 200:
                        return response.text
                return None
        except RequestException:
                return None
def parse_one_page(html):
        pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
                             +'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
                             +'.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>',re.S)
        items = re.findall(pattern, html)
        for item in items:
                yield {
                        'index':item[0],
                        'image':item[1],
                        'title':item[2],
                        'actor':item[3].strip()[3:],
                        'time':item[4].strip()[5:],
                        'score':item[5]+item[6]
                }
def write_to_file(content):
        with open('result.txt', 'a') as f:
                f.write(json.dumps(content, ensure_ascii= False) + '\n')
                f.close()


def main(offset):
        url = "https://maoyan.com/board/4?offset="+str(offset)
        html = get_one_page(url)
        # print(html)
        for item in parse_one_page(html):
                print(item)
                write_to_file(item)
if __name__ == '__main__':
        pool = Pool()
        pool.map(main, [i*10 for i in range(10)])