爬取豆瓣电影数据

采集豆瓣数据时，最好加上代理，负责被封可不负责

直接上代码

# -*- coding: utf-8 -*-

import requests, json, time, re, csv, random
from lxml import etree

"""
__Author__: luoshen
__Date: 2020-06-19
"""


class Spider(object):
    header = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 '
                      'Safari/537.36'
    }

    proxy = [
        {"proxy": "", "proxy_scheme": "http"},
        {"proxy": "", "proxy_scheme": "http"},
        {"proxy": "", "proxy_scheme": "http"},
        {"proxy": "", "proxy_scheme": "http"},
        {"proxy_scheme": "https", "proxy": ""},
        {"proxy_scheme": "http", "proxy": ""},
        {"proxy_scheme": "http", "proxy": ""},
        {"proxy_scheme": "http", "proxy": ""},
        {"proxy_scheme": "http", "proxy": ""}
    ]

    scraped_urls = set()

    csv_header = ['title', 'runtime', 'ReleaseDate', 'attrs', 'stars', 'commits', 'type', 'actors', 'state', 'language']

    # def __init__(self):
    #     with open('douban.csv', 'w') as f:
    #         f_csv = csv.writer(f)
    #         f_csv.writerow(self.csv_header)

    def get_content(self, url):
        time.sleep(1)
        try:
            item = random.choice(self.proxy)
            proxy = {
                item['proxy_scheme']: item['proxy']
            }
            req = requests.get(url, headers=self.header, proxies=proxy, timeout=2)
            if req.status_code == 200:
                return req.text
            else:
                self.get_content(url)
        except:
            self.get_content(url)

    def json_to_dict(self, content):
        content_dict = json.loads(content)
        return content_dict

    def Analytic_content_get_url(self, dict):
        urllist = list()
        for item in dict['data']:
            if item['url'] in self.scraped_urls:
                pass
            else:
                urllist.append(item['url'])
        return urllist

    def Analytic_moive_page_content(self, content):
        all = list()
        html = etree.HTML(content)
        for item in html.xpath('//*[@id="content"]'):
            title = item.xpath('./h1/span[1]/text()')  # 标题
            all.append(''.join(title))
        for item in html.xpath('//div[@id="info"]'):
            runtime = item.xpath('./span[@property="v:runtime"]/text()')  # 片长
            ReleaseDate = item.xpath('./span[@property="v:initialReleaseDate"]/text()')  # 上映时间
            attrs = item.xpath('./span[1]/span[@class="attrs"]/a/text()')  # 导演
            all.append(''.join(runtime))
            all.append(','.join(ReleaseDate))
            all.append(''.join(attrs))
        stars = re.findall('property="v:average">(.*?)</strong>', content)  # 评分
        state = re.findall('<span class="pl">制片国家/地区:</span> (.*?)<br/>', content)  # 国家
        actors = re.findall('rel="v:starring">(.*?)</a>', content)  # 主演
        type = re.findall('<span property="v:genre">(.*?)</span>', content)  # 类型
        language = re.findall('<span class="pl">语言:</span> (.*?)<br/>', content)  # 语言
        commits = re.findall(r'>(全部 \d+ 条)</a>', content)[0]
        all.append(''.join(stars))
        all.append(''.join(commits))
        all.append(''.join(type))
        all.append(','.join(actors))
        all.append(','.join(state))
        all.append(','.join(language))
        print('----------------------------------------')
        for item in all:
            print(item)
        return all

    def write_to_csv(self, items):
        with open('douban.csv', 'a') as f:
            a = csv.writer(f)
            if items is None:
                pass
            else:
                a.writerow(items)
                print(items[0], 'write to csv')

    def run(self):
        for i in range(0, 100, 20):
            url = 'https://movie.douban.com/j/new_search_subjects?sort=T&range=0,' \
                  '10&tags=%E7%94%B5%E5%BD%B1&start={}'.format(i)
            content = S.get_content(url)
            content_dict = S.json_to_dict(content)
            urllist = S.Analytic_content_get_url(content_dict)
            for url in urllist:
                movie_page_content = S.get_content(url)
                movie_content = S.Analytic_moive_page_content(movie_page_content)
                # S.write_to_csv(movie_content)


if __name__ == '__main__':
    S = Spider()
    S.run()