【data processing】数据爬虫、清洗、合并图片文件夹、裁剪、去除小图和下载错误图像、重命名、去除重复图片等处理

1.爬取指定关键字图片

'''
爬取指定关键字图片
'''
import re  # 正则表达式，解析网页
import requests  # 请求网页
import traceback
import os


def dowmloadPic(html, keyword, startNum):
    headers = {
   'user-agent': 'Mozilla/5.0'}  # 浏览器伪装，因为有的网站会反爬虫，通过该headers可以伪装成浏览器访问，否则user-agent中的代理信息为python
    pic_url = re.findall('"objURL":"(.*?)",', html, re.S)  # 找到符合正则规则的目标网站
    num = len(pic_url)
    i = startNum
    subroot = root + '/' + word
    # txtpath = subroot + '/download_detail.txt'

    print('找到关键词:' + keyword + '的图片，现在开始下载图片...')

    for each in pic_url:
        a = '第' + str(i + 1) + '张图片，图片地址:' + str(each) + '\n'
        b = '正在下载' + a
        print(b)
        path = subroot + '/' + str(i + 1)
        try:
            if not os.path.exists(subroot):
                os.mkdir(subroot)
            if not os.path.exists(path):
                pic = requests.get(each, headers=headers, timeout=10)
                with open(path + '.jpg', 'wb') as f:
                    f.write(pic.content)
                    f.close()
                # with open(txtpath, 'a') as f:
                #     f.write(a)
                #     f.close()

        except:
            continue
            # traceback.print_exc()
            # print('【错误】当前图片无法下载')
            # os.remove(pic.content)
            # pass
            # continue
        i += 1

    return i


if __name__ == '__main__':

    headers = {
   'user-agent': 'Mozilla/5.0'}
    # words = ['人拉行李箱', '行李箱和人', '人托行李箱']
    # words = ['黄色行李箱','黄色包包','黄色双肩包','黄色腰包','黄色手提包','黄色单肩包','黄色手提袋']
    words = ['人拉灰色行李箱','人背灰色包包','人背灰色双肩包','人背灰色腰包','人拿灰色手提包','人背灰色单肩包','人拿灰色手提袋']
    # words = ['粉色行李箱','粉色包包','粉色双肩包','粉色腰包','粉色手提包','粉色单肩包','粉色手提袋']
    # words = [ ‘蓝色行李箱','棕色包包','棕色双肩包','棕色腰包','棕色手提包','棕色单肩包','棕色手提袋']
    # words &#