import requests
from fake_useragent import UserAgent
import json
import csv
import pymongo
import random
import time
clien=pymongo.MongoClient(自己的数据库)
db=clien.dianshiju
coll=db.text
ua=UserAgent()
headers={
'User-Agent':ua.random
}
def request_json(page):
try:
url='https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start={}'.format(page)
response=requests.get(url,headers=headers)
this_is_json=json.loads(response.text)
return this_is_json
except Exception:
print('request error')
def extract_data(json):
if json:
for i in json.get('subjects'):
data = {}
data['电视图片']=i.get('cover')
data['电视名字']=i.get('title')
data['电视评分']=i.get('rate')
data['是否是新的电视剧']=i.get('is_new')
data['电视ID']=i.get('id')
coll.insert_one(data)
print(data)
def sava_csv(json):
pass
for i in range(0,1000):
page=i*20
extract_data(request_json(page))
time.sleep(random.randint(0,5))
一键复制
编辑
Web IDE
原始数据
按行查看
历史