百度图片爬虫，爬取高清图片

在做深度学习研究与应用的时候，经常需要爬取样本，例如，超分辨率重建，实际的训练与产品应用中，你需要爬取一些高清的图片，下面提供一个简单的爬虫：

# coding=utf-8
import re
import sys
import urllib
import os
import requests

def get_onepage_urls(onepageurl):
    if not onepageurl:
        print('执行结束')
        return [], ''
    try:
        html = requests.get(onepageurl).text
    except Exception as e:
        print(e)
        pic_urls = []
        fanye_url = ''
        return pic_urls, fanye_url
    pic_urls = re.findall('"objURL":"(.*?)",', html, re.S)
    fanye_urls = re.findall(re.compile(r'<a href="(.*)" class="n">下一页</a>'), html, flags=0)
    fanye_url = 'http://image.baidu.com' + fanye_urls[0] if fanye_urls else ''
    fanye_url = fanye_url + "&z=9" #z=9表示爬取特大尺寸
    return pic_urls, fanye_url
 
 
def down_pic(pic_urls,localPath):

    print ("need download:%d" % len(pic_urls))
    if not os.path.exists(localPath):  #
        os.mkdir(localPath)
    """给出图片链接列表, 下载图片"""
    for i, pic_url in enumerate(pic_urls):
        try:
            pic = requests.get(pic_url, timeout=30)
            string = str(i + 1) + '.jpg'
            with open(os.path.join(localPath,'%d.jpg' % i), 'wb')as f:
                f.write(pic.content)
                print('成功下载第%s张图片: %s' % (str(i + 1), str(pic_url)))
        except Exception as e:
            print('下载第%s张图片时失败: %s' % (str(i + 1), str(pic_url)))
            print(e)
            continue
if __name__ == '__main__':
    keyword = sys.argv[1]  # 关键词, 改为你想输入的词即可
    url_init_first = r'http://image.baidu.com/search/flip?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1497491098685_R&pv=&ic=0&nc=1&z=9&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&ctd=1497491098685%5E00_1519X735&word='
    url_init = url_init_first + urllib.parse.quote(keyword, safe='/')
    all_pic_urls = []
    onepage_urls, fanye_url = get_onepage_urls(url_init)
    all_pic_urls.extend(onepage_urls)

    fanye_count = 1  # 图片所在页数，下载完后调整这里就行
    while 1:
        onepage_urls, fanye_url = get_onepage_urls(fanye_url)
        fanye_count += 1
        print('第%s页' % fanye_count)
        if fanye_url == '' and onepage_urls == []:
            break
        all_pic_urls.extend(onepage_urls)


    print ("need download:%d" % len(all_pic_urls))
    down_pic(list(set(all_pic_urls)),os.path.join('./image/'+sys.argv[1]))#保存位置也可以修改

运行环境需要python3支持。