python爬取动态页面图片_python3爬取动态网站图片

思路：

1、图片放在XXX标签中

2、利用fiddler抓包获取存放图片信息的js文件url

3、利用requests库获取html内容，然后获取其中图片id

4、利用fiddler抓取下载图片地址，结合图片id来下载图片(大文件)

# -*- coding:UTF-8 -*-

import requests, json, time

from contextlib import closing

class get_photos(object):

def __init__(self):

self.photos_id = []

self.download_server = ‘https://unsplash.com/photos/xxx/download?force=trues‘#下载图片地址，通过fiddler抓包获取

self.target = ‘https://unsplash.com/napi/photos?page=1&per_page=24‘#存有图片信息的js文件地址

self.headers = {‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36‘}

"""

函数说明:获取图片ID

"""

def get_ids(self):

req = requests.get(url=self.target, headers=self.headers, verify=False)

html = json.loads(req.text)

print(type(html), len(html))

for i in range(len(html)):

self.photos_id.append(html[i][‘id‘])#将列表html中获取id放到列表photos_id中

print(self.photos_id)

time.sleep(1)

"""

函数说明:图片下载

"""

def download(self, photo_id, filename):

self.headers = {‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36‘}

self.target = self.download_server.replace(‘xxx‘, photo_id)#将xxx换成图片id

#closing可以将任意对象转成上下文对象

with closing(requests.get(url=self.target, stream=True, verify = False, headers = self.headers)) as r:

with open(‘%d.jpg‘ % filename, ‘ab+‘) as f:

#下载大文件，r.iter_content表示获取响应原始内容

for chunk in r.iter_content(chunk_size = 1024):#边下载边存硬盘，chunk_size表示以1024比例存数据

if chunk:

f.write(chunk)

f.flush()#强行把缓冲区中的内容放到磁盘中

if __name__ == ‘__main__‘:

gp = get_photos()#类实例化

print(‘获取图片连接中:‘)

gp.get_ids()#获取图片id

print(‘图片下载中:‘)

for i in range(len(gp.photos_id)):

print(‘正在下载第%d张图片‘ % (i+1))

gp.download(gp.photos_id[i], (i+1))