爬取豆瓣电影top250

代码

#  Author:ZhouChuang
#  coding:utf-8

from bs4 import BeautifulSoup
import requests
import time

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3514.0 Safari/537.36',
    'Cookie':'viewed="2166211"; bid=wLwzb9b0g_A; douban-fav-remind=1; ll="118173"; __utmc=30149280; __utmc=223695111; _vwo_uuid_v2=D96C'
             '22273BD00491856812822DDB071A2|e5653604c927a32fa93d6e494419f10c; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1533806091%2C%22h'
             'ttps%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DuEf5o7h6W1QgcIPLdxBrM9-O5w1pL72KygnR1F15VN2W7NpRddrICJa95QHW8IHb%26wd%3D%26eqid%3'
             'Dd58b6e88000163dd000000045b6bfbe0%22%5D; _pk_ses.100001.4cf6=*; ps=y; ck=2wWO; __utma=30149280.1177526221.1531553567.1533803'
             '492.1533806181.4; __utmz=30149280.1533806181.4.3.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/phone/b'
             'ind; __utma=223695111.801250262.1533803492.1533803492.1533806181.2; __utmz=223695111.1533806181.2.2.utmcsr=accounts.douban.com'
             '|utmccn=(referral)|utmcmd=referral|utmcct=/phone/bind; ap=1; push_noty_num=0; push_doumail_num=0; douban-profile-remind=1; __'
             'utmv=30149280.15261; __utmb=30149280.22.10.1533806181; _pk_id.100001.4cf6=3eba8e0d5047ec4c.1533803492.2.1533806798.1533803530.;'
             ' __utmb=223695111.15.10.1533806181'
}
urls = ['https://movie.douban.com/top250?start={}&filter='.format(str(i)) for i in range(0,275,25)]
url = 'https://movie.douban.com/top250?start=0&filter='
def get(url,data=None):
    wb_data = requests.get(url,headers=headers)
    time.sleep(2)
    Soup = BeautifulSoup(wb_data.text,'lxml')
    paimings =Soup.select('#content > div > div.article > ol > li > div > div.pic > em')
    titles = Soup.select('#content > div > div.article > ol > li > div > div.pic > a > img')
    pingfens = Soup.select('#content > div > div.article > ol > li > div > div.info > div.bd > div > span.rating_num')
    jianpings = Soup.select('#content > div > div.article > ol > li > div > div.info > div.bd > p.quote > span')
    images = Soup.select('#content > div > div.article > ol > li > div > div.pic > a > img')
    # for pingfen in pingfens:
    #     print(pingfen.get_text())
    for title,pingfen,image,paiming,jianping in zip(titles,pingfens,images,paimings,jianpings):
        data = {
            '排名':paiming.get_text(),
            '名称':title.get('alt'),
            '评分':pingfen.get_text(),
            '简评':jianping.get_text(),
            '图片链接':image.get('src')
        }
        print(data)
# print(urls)
for smart in urls:
    get(smart)

结果截图

截图.png