爬取猫眼网排行榜

爬取猫眼电影排行榜数据

这是一篇关于如何从猫眼网爬取电影排行数据的文章。
需要对requests库以及正则表达式有所了解。

所用工具库

由于该项目需依赖requests库，所以在开始前需要确保安装requests库。
安装了requests后再引入所需库文件。

import csv
import re
import requests
from requests import RequestException

完成对库文件的操作

代码解析

1.页面内容获取

def get_one_page(url,offset):
    try:
        headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64)AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'}
        response = requests.get(url=url, headers=headers,params={"offset":offset})
        if response.status_code==200:
            return response.text
            pass
        else:
            return None
            pass
        pass
    except RequestException as e:
        return None
        pass
    pass

请求一个页面返回响应内容，通过添加headers报头进行爬虫伪装，利用requests.get函数发起请求，如果网页返回码为200（成功），则return response.text，返回页面源代码。
2.页面解析

def parse_one_page(html):
    pattern = '<dd>.*?board-index.*?">(\d+)</i>.*?data-src="(.*?)".*?/>.*?movie-item-info.*?title="(.*?)".*?star">' + \
              '(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(\d+)</i>.*?</dd>'
    # re.S匹配多行
    regex = re.compile(pattern, re.S)
    items = regex.findall(html)
    for item in items:
        yield {
            'index': item[0],
            'thumb': get_large_thumb(item[1]),
            'title': item[2],
            'actors': item[3].strip()[3:],
            'release_time': get_release_time(item[4].strip()[5:]),
            'area': get_release_area(item[4].strip()[5:]),
            'score': item[5] + item[6]
        }
        pass
    pass
# 获取上映时间
def get_release_time(data):
    pattern = '^(.*?)(\(|$)'
    regex = re.compile(pattern)
    w = regex.search(data)
    return w.group(1)


# 获取上映地区
def get_release_area(data):
    pattern = '.*\((.*)\)'
    regex = re.compile(pattern)
    w = regex.search(data)
    if w is None:
        return'未知'
    return w.group(1)


# 获取封面大图
def get_large_thumb(url):
    pattern = '(.*?)@.*?'
    regex = re.compile(pattern)
    w = regex.search(url)
    return w.group(1)




# 下载封面图
def download_thumb(title,url):
    try:
        response = requests.get(url=url)
        # 获取二进制数据
        with open('thumb/'+title+'.jpg', 'wb') as f:
            f.write(response.content)
            f.close()
    except RequestException as e:
        print(e)
        pass


# 2.主调度程序
def main():
    # 起始URL
    start_url="http://maoyan.com/board/4"
    for i in range(0,100,10):
        # 获取响应文本内容
        html = get_one_page(url=start_url, offset=i)
        if html is None:
            print("链接：%s?offset=%s异常".format(start_url,i))
            continue
        pass
        for item in parse_one_page(html=html):
            store_data(item)
            download_thumb(item["title"],item["thumb"])
            pass

    pass

解析页面数据时将每个电影的信息化作一个个字典，形成结构化数据。便于后续的存储。
3.存储数据

# 存储数据
def store_data(item):
    with open('movie.csv','a',newline='',encoding='utf-8') as data_csv:
        # dialect为打开csv文件的方式，默认是excel，delimiter="\t"参数指写入的时候的分隔符
        try:
            csv_writer = csv.writer(data_csv)
            csv_writer.writerow([item['index'], item['thumb'], item['title'], item['actors'],item['release_time'],item['area'],item['score']])
        except Exception as e:
            print(e)
            print(item)

接着使用writerow()将values数据一次一行的写入csv中。在open中newline=’ '是为了防止写入数据后多出空白行。

完整代码

import csv
import re

import requests
from requests import RequestException

# 3.页面内容获取
# 请求一个页面返回响应内容
def get_one_page(url,offset):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'}
        response = requests.get(url=url, headers=headers,params={"offset":offset})
        if response.status_code==200:
            return response.text
            pass
        else:
            return None
            pass
        pass
    except RequestException as e:
        return None
        pass
    pass

# 4.页面解析
# 解析一个页面
def parse_one_page(html):
    pattern = '<dd>.*?board-index.*?">(\d+)</i>.*?data-src="(.*?)".*?/>.*?movie-item-info.*?title="(.*?)".*?star">' + \
              '(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(\d+)</i>.*?</dd>'
    # re.S匹配多行
    regex = re.compile(pattern, re.S)
    items = regex.findall(html)
    for item in items:
        yield {
            'index': item[0],
            'thumb': get_large_thumb(item[1]),
            'title': item[2],
            'actors': item[3].strip()[3:],
            'release_time': get_release_time(item[4].strip()[5:]),
            'area': get_release_area(item[4].strip()[5:]),
            'score': item[5] + item[6]
        }
        pass
    pass

# 获取上映时间
def get_release_time(data):
    pattern = '^(.*?)(\(|$)'
    regex = re.compile(pattern)
    w = regex.search(data)
    return w.group(1)


# 获取上映地区
def get_release_area(data):
    pattern = '.*\((.*)\)'
    regex = re.compile(pattern)
    w = regex.search(data)
    if w is None:
        return'未知'
    return w.group(1)


# 获取封面大图
def get_large_thumb(url):
    pattern = '(.*?)@.*?'
    regex = re.compile(pattern)
    w = regex.search(url)
    return w.group(1)

# 存储数据
def store_data(item):
    with open('movie.csv','a',newline='',encoding='utf-8') as data_csv:
        # dialect为打开csv文件的方式，默认是excel，delimiter="\t"参数指写入的时候的分隔符
        try:
            csv_writer = csv.writer(data_csv)
            csv_writer.writerow([item['index'], item['thumb'], item['title'], item['actors'],item['release_time'],item['area'],item['score']])
        except Exception as e:
            print(e)
            print(item)


# 下载封面图
def download_thumb(title,url):
    try:
        response = requests.get(url=url)
        # 获取二进制数据
        with open('thumb/'+title+'.jpg', 'wb') as f:
            f.write(response.content)
            f.close()
    except RequestException as e:
        print(e)
        pass


# 2.主调度程序
def main():
    # 起始URL
    start_url="http://maoyan.com/board/4"
    for i in range(0,100,10):
        # 获取响应文本内容
        html = get_one_page(url=start_url, offset=i)
        if html is None:
            print("链接：%s?offset=%s异常".format(start_url,i))
            continue
        pass
        for item in parse_one_page(html=html):
            store_data(item)
            download_thumb(item["title"],item["thumb"])
            pass

    pass

if __name__ == '__main__':
    main()
    print("爬取完成!")
    pass