python——爬百度图片

python——爬百度图片

爬虫的本质：获取网页，解析网页，获取出自己想要的数据，对数据进行保存

import urllib.request
import urllib.parse
import re
import os


class BaiduPicture(object):
    # User-Agent是必须的，伪装成浏览器进行访问
    # Referer是必须的,否则会返回403错误
    header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                            'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
              "referer": "https://image.baidu.com"}

    # 获取图片的网址
    url = "https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=" \
          "{word}&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=0&word={word}&s=&se=&tab=&width=&height=&" \
          "face=0&istype=2&qc=&nc=1&fr=&cg=girl&pn={pageNum}&rn=30&gsm=1e00000000001e&1490169411926="

    # 想要搜索的图片关键字
    keyword = '王力宏'
    # 对中文进行转码
    keyword = urllib.parse.quote(keyword, 'utf-8')

    # n用于去控制下载的图片的数量
    n = 0
    # j作为数字被加在图片的名称后面
    j = 0
    error = 0
    while (n < 300):
        n += 30
        # format() 用于对字符串的格式化
        new_url = url.format(word=keyword, pageNum=str(n))
        rep = urllib.request.Request(new_url, headers=header)
        rep = urllib.request.urlopen(rep)
        try:
            html = rep.read().decode('utf-8')
        except:
            print('wrong')
            error = 1
            print("page" + str(n))
        if (error == 1):
            continue
        # compile() 函数,将一个字符串编译为字节代码
        p = re.compile("thumbURL.*?\.jpg")
        s = p.findall(html)
        if os.path.isdir("Users/donghongyu/Downloads/picture") != True:
            os.makedirs(r"Users/donghongyu/Downloads/picture")
        with open("testPic1.txt", "w") as f:
            for i in s:
                i = i.replace("thumbURL\":\"", "")
                print(i)
                f.write(i)
                f.write("\n")
                urllib.request.urlretrieve(i, "/Users/donghongyu/Downloads/picture/pic{num}.jpg".format(num=j))
                j += 1
if __name__ == '__main__':
    BaiduPicture()