fiddlers2

import requests
from  bs4 import BeautifulSoup
import json
import lxml
import re
import os
import os.path
import string
#headers中的内容要分大小写
#//nas/LargeSave/高清图像数据/httpswww.pexels.comzh-tw/
#a.关于使用request.get访问不了的情况，加上Accept  Accept-Language  Referer  User-Agent  Cookie  proxies  这些参数，并且要注意大小写,有无空格这些，要与网页中看到的一致，
#b.证书问题考虑加上verify=False  同样注意大小写
#c.其次可以在fiddlers中查看 程序请求到的网页状态是什么样的（process为python）
#d.设置断点可以查看是否应用到了设置的这些防止反爬的参数


path = 'D:/httpswwwpexelscomzh-tw/'
def dwonload_img(page):
    global count
    url=f'https://www.pexels.com/zh-tw/?format=js&seed={page}&type='
    res=requests.get(url,headers={'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
                              'Accept-Language':'zh-CN,zh;q=0.9',
                              'Referer':'https://www.pexels.com/zh-tw/',
                              'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.39',
                              'Cookie':'_ga=GA1.2.953246023.1646290358; _hjSessionUser_171201=eyJpZCI6ImI4NzRmMjRiLWRlMjQtNWZlMi1hNjRkLWUyMDA2MjRmODFlYiIsImNyZWF0ZWQiOjE2NDYyOTA5MDI2NTcsImV4aXN0aW5nIjp0cnVlfQ==; locale=zh-TW; NEXT_LOCALE=zh-TW; _gid=GA1.2.572006961.1647307445; ab.storage.sessionId.5791d6db-4410-4ace-8814-12c903a548ba=%7B%22g%22%3A%223c77dd5a-d80d-5514-6e89-a515667aaba1%22%2C%22e%22%3A1647336874054%2C%22c%22%3A1647335074055%2C%22l%22%3A1647335074055%7D; ab.storage.deviceId.5791d6db-4410-4ace-8814-12c903a548ba=%7B%22g%22%3A%22b43b9c98-d28a-dfdb-1c65-c0f58a7b3689%22%2C%22c%22%3A1646290357434%2C%22l%22%3A1647335074056%7D; _gat=1; __cf_bm=MtqVbgDKWWQPM0LD5MJ2yjHVLCK1HddcW169X4OgWPQ-1647335087-0-AZcu7uQ0AlxdR8R5ANaYX7c9fCDbKBKL4XGoxTSgYCC0zon2SKUWtTvwSNBk4TyzHLrRW0TFq2RYZY42JOW2sbMjYFx3eMvD2mZc4zNwlDH9HAaGN9/HdaiOuvGR0+ClhwtVkqHRgkLtH4jt2DpznUh7yzGGojmH8CG5MBNuWrny'
                              }, proxies={'http':'http://127.0.0.1:18888','https':'http://127.0.0.1:18888'},verify=False)

    #获取img_url
    text=res.text
    print(res.status_code)
    all_url=re.findall("data-big-src=\\\\\"(http.+?)\"",text)#正则表达式匹配
  


    #dwonload
    for img in all_url:
        pattern = re.compile(r'\d+')
        id = pattern.findall(img)[0]
        str_id = str(id)
        dwonload_url = f'https://images.pexels.com/photos/{str_id}/pexels-photo-{str_id}.jpeg'
        print(dwonload_url)

        if os.path.isdir(path + str(count)):
            pass
        else:
            os.mkdir(path + str(count))

        document_path=path + str(count)
        pic_path=document_path + '/'+str(count) + '.jpg'#这里用了‘/’来构成路径
        
        byte=requests.get(dwonload_url,headers={'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
                              'Accept-Language':'zh-CN,zh;q=0.9',
                              'Referer':'https://www.pexels.com/zh-tw/',
                              'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.39',
                              'Cookie':'_ga=GA1.2.953246023.1646290358; _hjSessionUser_171201=eyJpZCI6ImI4NzRmMjRiLWRlMjQtNWZlMi1hNjRkLWUyMDA2MjRmODFlYiIsImNyZWF0ZWQiOjE2NDYyOTA5MDI2NTcsImV4aXN0aW5nIjp0cnVlfQ==; locale=zh-TW; NEXT_LOCALE=zh-TW; _gid=GA1.2.572006961.1647307445; ab.storage.sessionId.5791d6db-4410-4ace-8814-12c903a548ba=%7B%22g%22%3A%223c77dd5a-d80d-5514-6e89-a515667aaba1%22%2C%22e%22%3A1647336874054%2C%22c%22%3A1647335074055%2C%22l%22%3A1647335074055%7D; ab.storage.deviceId.5791d6db-4410-4ace-8814-12c903a548ba=%7B%22g%22%3A%22b43b9c98-d28a-dfdb-1c65-c0f58a7b3689%22%2C%22c%22%3A1646290357434%2C%22l%22%3A1647335074056%7D; _gat=1; __cf_bm=MtqVbgDKWWQPM0LD5MJ2yjHVLCK1HddcW169X4OgWPQ-1647335087-0-AZcu7uQ0AlxdR8R5ANaYX7c9fCDbKBKL4XGoxTSgYCC0zon2SKUWtTvwSNBk4TyzHLrRW0TFq2RYZY42JOW2sbMjYFx3eMvD2mZc4zNwlDH9HAaGN9/HdaiOuvGR0+ClhwtVkqHRgkLtH4jt2DpznUh7yzGGojmH8CG5MBNuWrny'
                              }, proxies={'http':'http://127.0.0.1:18888','https':'http://127.0.0.1:18888'},verify=False).content
        fp = open(pic_path,'wb')#写入图片的话是要用二进制形式写入
        fp.write(byte)
        fp.close()
        count +=1
        print(count)


    #获取下一页
    next=re.findall(r"seed=.+?\\",text)[0]#实现的难点
    while len(next)==0:
        print('总数为：',count)
        exit()

    next=next.strip('\\')#去除转义符号
    page_next=next[5:]
    page_next=str(page_next)
    print(type(page_next))
    print(page_next)
    dwonload_img(page_next)


if __name__ == "__main__":
    page='2022-03-21%2012%3A18%3A51%20UTC'
    count=0
    dwonload_img(page)