图库网图片爬虫-json分析

# -*- coding:utf-8 -*-
import os
import time
import requests
import urllib
import uuid
import json
from pyquery import PyQuery

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/75.0.3770.142 Safari/537.36"
}


def href_url_download():
    # 1.填写要爬取的关键词list.txt
    keyword_list = open("list.txt", 'r', encoding='utf-8')
    lines = keyword_list.readlines()
    keyword_list.close()
    for keyword in lines:
        keyword = keyword.strip()
        url = "https://www.tukuchina.cn/index.php?r=photo/channelInfo&chid=1&" \
              "parent=1096153&id=1096251&q=" + keyword  # 获取keyword网页
        print(url)
        txt = requests.get(url, headers=headers).text  # 获取URL及headers
        doc = PyQuery(txt)
        # print(doc)
        pages = int(doc(".page a").text().split(" ")[-1])  # 获取keyword最大页数
        # print(pages)
        # 2.填写爬取图片的页数，默认所有页数：pages
        for page in range(1, pages+1):
            # print(page)
            images_url = url + "&type=&Photo_page=" + str(page)  # 获取keyword具体页网页
            # print(images_url)
            txt2 = requests.get(images_url, headers=headers).text
            doc2 = PyQuery(txt2)
            # print(doc2)
            id_list = doc2(".searchLi .pic_list_ll img").items()  # 获取类别URL
            for id_url in id_list:
                i = id_url.attr("id")
                json_url = "https://www.tukuchina.cn/index.php?r=picture/detail&photoID=" + i
                # json结构分析：F12->Network->https://www.tukuchina.cn/index.php?r=picture/detail&photoID=id
                txt3 = requests.get(json_url, headers=headers).text
                json_txt = json.loads(txt3)
                print(json_txt)
                img_url = json_txt.get("data").get("photoInfo").get("photoViewURL")
                image_download(img_url, keyword)  # 下载"图片"


def image_download(img_url, keyword):
    # 3.图片存储路径
    folder_path = "./image/" + keyword + "/"
    if not os.path.exists(folder_path):
        os.makedirs(folder_path)
        # print(folder_path)
    image = keyword + "_" + str(uuid.uuid1()) + ".jpg"  # 图片命名
    try:
        print(image)
        # urllib.request.urlretrieve(img_url, folder_path + "/" + image)
        content = requests.get(img_url, headers=headers)
        with open(folder_path + image, "wb") as f:
            f.write(content.content)
    except urllib.error.HTTPError:
        print("Internal Server Error")


if __name__ == '__main__':
    href_url_download()