python爬豆瓣电视剧_爬取豆瓣热门电视剧.py

import requests

from fake_useragent import UserAgent

import json

import csv

import pymongo

import random

import time

clien=pymongo.MongoClient(自己的数据库)

db=clien.dianshiju

coll=db.text

ua=UserAgent()

headers={

'User-Agent':ua.random

}

def request_json(page):

try:

url='https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start={}'.format(page)

response=requests.get(url,headers=headers)

this_is_json=json.loads(response.text)

return this_is_json

except Exception:

print('request error')

def extract_data(json):

if json:

for i in json.get('subjects'):

data = {}

data['电视图片']=i.get('cover')

data['电视名字']=i.get('title')

data['电视评分']=i.get('rate')

data['是否是新的电视剧']=i.get('is_new')

data['电视ID']=i.get('id')

coll.insert_one(data)

print(data)

def sava_csv(json):

pass

for i in range(0,1000):

page=i*20

extract_data(request_json(page))

time.sleep(random.randint(0,5))

一键复制

编辑

Web IDE

原始数据

按行查看

历史