爬虫入门实例（五）

爬取猫眼电影榜单

import json
import os
import requests
from bs4 import BeautifulSoup


def get_html(url):
    try:
        d = {'User Agent':"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0"}
        r = requests.get(url, params=d)
        if r.status_code == 200:
            return r.text
        return None
    except:
        return None

def parse_html(html):
    soup = BeautifulSoup(html, "html.parser")
    # 创建BeautifulSoup对象,解析HTML网页
    data = soup.body.find_all('div', {'class': 'movie-item-info'})
    # 获取soup的body部分，找到class为movie-item-info的div
    name = []
    # 建立一个空列表，储存爬取的数据
    for i in data:
        lis = i.text.replace('\n', " ")
        # 取标签文本部分，并用空格代替换行
        name.append(lis)
        # 存入列表
        t = 1
    for i in name:
        print(t, i)
        t += 1
    return name

def write_down(lis):
    with open('C:/Users/root/Desktop/py/猫眼.txt','a') as f:
        for i in lis:
            f.write(json.dumps(i,ensure_ascii=False) + '\n')
            # 注意编码false,
        f.close()

def main():
    url = 'https://maoyan.com/board/4'
    html = get_html(url)
    list = parse_html(html)
    write_down(list)

# __main__python的主函数
if __name__ == '__main__':
    main()