requests+BeautifulSoup爬取猫眼电影top100

首先测试一下网页是否有用

import requests,json
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
def get_one_page(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'
    }
    response=requests.get(url,headers=headers)
    try:
        if response.status_code==200:  #如果状态码为200，则返回抓取的网页文本
            return response.text
        return None
    except RequestException:
        return print("访问页面错误(●'◡'●)")
def main():
    url = 'https://maoyan.com/board/4?offset=0'
    html=get_one_page(url)
    print(html)
if __name__ == '__main__':
    main()

运行成功，返回如下text。这一段包含我们想要爬去的所有的内容。

                <dd>
                        <i class="board-index board-index-9">9</i>
    <a href="/films/1212" title="千与千寻" class="image-link" data-act="boarditem-click" data-val="{movieId:1212}">
      <img src="//ms0.meituan.net/mywww/image/loading_2.e3d934bf.png" alt="" class="poster-default" />
      <img data-src="http://p0.meituan.net/movi