Python爬虫：解析异步加载的图片网站（实例）

以360图片站为例：
https://image.so.com/z?ch=wallpaper#/

爬取其壁纸频道，前9页，共270张壁纸缩略图：

"""
360图片网站爬虫
网站图片爬取

"""

import requests
from urllib.parse import urlencode #把字典转化为url参数

'''
链接分析：
https://image.so.com/zjl?ch=wallpaper&sn=30&pn=30&prevsn=0
https://image.so.com/zjl?ch=wallpaper&sn=60&pn=30&prevsn=30
https://image.so.com/zjl?ch=wallpaper&sn=90&pn=30&prevsn=60
https://image.so.com/zjl?ch=wallpaper&sn=120&pn=30&prevsn=90
'''
#构建参数字典
def create_params():
    urls = []
    for i in range(1,10):
        params = {
            'ch' : 'wallpaper',
            'sn' : i*30,
            'pn' : '30',
            'prevsn' : i*30-30
        }
        url = 'https://image.so.com/zjl?'
        #把字典转化为url参数
        params_list = urlencode(params)
        #拼接URL
        url = url+params_list
        urls.append(url)
    print(urls)
    return urls

#请求网页
def request_pages(url):
    ua = {'User-Agent':'User-Agent:Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'}
    resp = requests.get(url,headers = ua)
    print("请求状态:%d"%(resp.status_code))
    #将网页内容转换为json格式
    html_json = resp.json()
    #print(html_json)
    return html_json

#提取图片链接列表
def get_image_list(html_json):
    image_list = html_json['list']
    #print(image_list[0])
    image_url_list=[]
    for item in image_list:
        temp = {'title':item['title'],'imgurl':item['qhimg_thumb']}
        image_url_list.append(temp)
    #print(image_url_list)
    return(image_url_list)

#保存图片
def save_img(image_url_list):
    for i in image_url_list:
        print(i)
        img_resp = requests.get(i['imgurl'])
        try:
            with open(r'C:\pics\%s.jpg'%(i['title']),'bw') as imgf:
                imgf.write(img_resp.content)
            if 'qhmsg' not in i['imgurl']:
                continue
        except:
            with open(r'C:\pics\%s.jpg'%(i['title'][:3]),'bw') as imgf:
                imgf.write(img_resp.content)
            if 'qhmsg' not in i['imgurl']:
                continue
    print("success")

def main_def():
    urls = create_params()
    for url in urls:
        html_json = request_pages(url)
        image_url_list = get_image_list(html_json)
        save_img(image_url_list)
    
if __name__ == '__main__':
    main_def()