python爬虫存储csv和图片练习：汉服图爬取

本文链接：https://blog.csdn.net/liaozp88/article/details/121707935

这段代码实现了一个使用Python的requests库进行网络请求，通过模拟浏览器User-Agent来获取网页数据的爬虫。它从特定URL抓取汉服图片，并使用多线程进行下载。每个图片保存时会根据当前时间戳命名，以防止文件覆盖。同时，爬虫将图片链接写入CSV文件中。

摘要由CSDN通过智能技术生成

只做练习用，不做其他！

import os
import threading
import time
import requests
import random
from requests.packages.urllib3.exceptions import InsecureRequestWarning

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

UA_LIST = ['Mozilla/5.0 (compatible; U; ABrowse 0.6; Syllable) AppleWebKit/420+ (KHTML, like Gecko)',
           'Mozilla/5.0 (compatible; U; ABrowse 0.6;  Syllable) AppleWebKit/420+ (KHTML, like Gecko)',
           'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Acoo Browser 1.98.744; .NET CLR 3.5.30729)',
           'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Acoo Browser 1.98.744; .NET CLR   3.5.30729)',
           'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0;   Acoo Browser; GTB5; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;   SV1) ; InfoPath.1; .NET CLR 3.5.30729; .NET CLR 3.0.30618)',
           'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SV1; Acoo Browser; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Avant Browser)',
           'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1;   .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)',
           'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; GTB5; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; Maxthon; InfoPath.1; .NET CLR 3.5.30729; .NET CLR 3.0.30618)',
           'Mozilla/4.0 (compatible; Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Acoo Browser 1.98.744; .NET CLR 3.5.30729); Windows NT 5.1; Trident/4.0)',
           'Mozilla/4.0 (compatible; Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB6; Acoo Browser; .NET CLR 1.1.4322; .NET CLR 2.0.50727); Windows NT 5.1; Trident/4.0; Maxthon; .NET CLR 2.0.50727; .NET CLR 1.1.4322; InfoPath.2)',
           'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Acoo Browser; GTB6; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; InfoPath.1; .NET CLR 3.5.30729; .NET CLR 3.0.30618)']
headers={'user-agent':random.choice(UA_LIST)}
t=threading.Lock()
m=1
def get_url(n):
    url=f'https://api5.hanfugou.com/Trend/GetTrendListForHot?maxid=3383672&objecttype=album&page={n}&count=20'
    resp=requests.get(url,headers=headers,verify=False).json()
    for i in resp['Data']:
        for m in i['Images']:
            url1=m['ImgSrc']+'_700x.jpg'
            save_url(url1)
        name=i['User']['NickName']
        save_jpg(name)



def save_url(url):
    t.acquire()
    with open(f'url存储.csv','a+',encoding='ANSI',errors='ignore') as f:
        f.write(f'{url}\n')
    t.release()

def save_jpg(name):
    global m
    t.acquire()
    with open('url存储.csv','r') as f:
        url = f.readlines()
        for url1 in url:
            url2=url1.strip()
            # file=os.path.join('汉服图',name)
            resp=requests.get(url2,headers=headers,verify=False).content
            with open(f'./汉服图/{int(time.time())}.jpg','wb') as f:
                f.write(resp)
                time.sleep(0.5)#存储太快会让时间参数作为名称同时赋给多个图片而覆盖，所以延迟一下
                m+=1
    t.release()


if __name__ == '__main__':

    path = os.path.exists('汉服图')
    os.mkdir('汉服图') if not path else print('已经建立！')

    for n in range(1,2):
        get_url(n)