爬取豆瓣电影前十页数据
# -*- coding: utf-8 -*-
# @Author : HitoChen
# @Time : 2021/9/22 6:52 下午
# @Function:
import urllib.request
import urllib.parse
import ssl
import json
ssl._create_default_https_context = ssl._create_unverified_context
def creat_request(page):
base_url = 'https://movie.douban.com/j/chart/top_list?type=17&interval_id=100%3A90&action=&'
data = {
'start' : (page-1)*20,
'limit' : 20
}
data = urllib.parse.urlencode(data)
new_url=base_url+data
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}
request = urllib.request.Request(url=new_url, headers=headers)
return request
def get_content(request):
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
return content
def get_download(page,content):
with open('douban.json','w',encoding='utf-8') as fp:
fp.write(content)
if __name__ == '__main__':
start_page = int(input('请输入起始页码'))
end_page = int(input('请输入终止页码'))
for page in range(start_page,end_page+1):
request = creat_request(page)
content = get_content(request)
get_download(page,content)
print('爬去数据成功,查看json文件')