python数据分析实战一：IMDB Top 250

This article deals with the top 250 movies in IMDB, including data scraping, data preparation, data cleaning, data analysis and visualization.

Data scraping

First, we need to scrape the data from this website.

# import package
import pandas as pd
import time
import urllib.request
from lxml.html import fromstring
from bs4 import BeautifulSoup

# download html
def download(url):
    print('Downloading:', url)
    request = urllib.request.Request(url)
    request.add_header('User-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36') #进行伪装
    resp = urllib.request.urlopen(request)
    html = resp.read().decode('utf-8')
    return html


# content to be scrape
Name = []
Year = []
Rate = []
Level = []
Directors = []
Writers = []
Stars = []
Genres = []
Runtime = []
Country = []
Language = []
Budget = []
Box_Office_USA = []
Box_Office_World = []

start_url = download('https://www.imdb.com/chart/top/?ref_=nv_mv_250')
domain = 'https://www.imdb.com/'
start_soup = BeautifulSoup(start_url)

# scrape every item
for k in range(250):
    sub_html = start_soup.find_all('tbody')[0].find_all('a')[2*k+1].get('href')
    url = download(domain + sub_html)
    time.sleep(3)   
    tree = fromstring(url)
    soup = BeautifulSoup(url)
    name = soup.find('span',{
   'id':'titleYear'}).previous_sibling
    Name.append(name.replace(name[-1],''))
    Year.append(tree.xpath('//*[@id="titleYear"]/a')[0].text_content())
    Rate.append(tree.xpath('//*[@id="title-overview-widget"]/div[1]/div[2]/div/div[1]/div[1]/div[1]/strong/span')[0].text_content())
    Level.append(soup.find('div',{
   'class':'subtext'}).span.previous_sibling.strip())
    try:
        Directors.append(soup.find(text='Director:').parent.parent.find('a').get_text())
    except AttributeError:
        directors = [k.get_text() for k in soup.find(text='Directors:').parent.parent.find_all('a')]
        Directors.append('/'.join(directors))
    try:
        writers = [k.get_text() for k in soup.find(text='Writers:').parent.parent.find_all('a')]
        Writers.append('/'.join(writers))
    except AttributeError:
        Writers.append(soup.find(text='Writer:').parent.parent.find('a').get_text())
    stars = [k.get_text()