python下载图片、已知url_python爬取图片网站(将图片下载至本地)

【实例简介】

【实例截图】

【核心代码】

import requests

from lxml import etree

import re

import os

from multiprocessing.dummy import Pool as ThreadPool

def gethtml(url,encode): #获取网页源码

r = requests.get(url)

r.encoding = encode

return r.text

def filterFName(FName): #文件名过滤特殊字符

rstr = r"[\/\\\:\*\?\"\<\>\|]"

new_name = re.sub(rstr, "_", FName)

return new_name

def mkdir(path): #创建文件夹

path = path.strip()

isExists = os.path.exists(path)

if not isExists:

os.makedirs(path)

def geturl(url): #获取网站各个分类链接、分类名称

html = gethtml(url,'utf-8')

ehtml = etree.HTML(html)

nurl = ehtml.xpath('//*[@id="chenxing_menu"]/li/a/@href')

ntitle = ehtml.xpath('//*[@id="chenxing_menu"]/li/a/text()')

urldata=[]

for i in range(1,len(nurl)-1):

urldata.append(nurl[i] '|' ntitle[i])

return urldata

def downPic(url,savepath): #下载图片

picname=url.split('/')[-1]

print('download: ' url)

picdata = requests.get(url)

with open(savepath picname, 'wb') as file:

file.write(picdata.content)

def getdata(ulist): #根据网站分类链接、名称进一步解析 这段代码是下载图片主体代码

url = ulist.split('|')[0]

til = ulist.split('|')[1]

html = gethtml(url,'utf-8')

ehtml = etree.HTML(html)

count = ''.join(ehtml.xpath('//*[@class="more r"]/em/text()'))

page = int(count) // 20

if int(count) % 20 > 0 :

page =1

for i in range(1,page 1):#获取各分类下面所有图集链接

nurl = url 'page/' str(i)

html = gethtml(nurl, 'utf-8')

ehtml = etree.HTML(html)

lvurl = ehtml.xpath('//*[@class="img"]/@href')

for j in lvurl: #获取每个图集页数和链接

html = gethtml(j, 'utf-8')

ehtml = etree.HTML(html)

lvurl = ''.join(ehtml.xpath('//*[@id="imagecx"]/h1/span/text()')) # 获取图集页数

title = ''.join(ehtml.xpath('//*[@id="imagecx"]/h1/text()')) #获取图集名称

title = title.replace('()','')

lvurl = lvurl.replace('1/','')

path = savepath '\\' til '\\' filterFName(title) '\\'

mkdir(path)

for k in range(1,int(lvurl)): #获取每个图集页面的图片

zurl = j.replace('.html','_' str(k) '.html')

html = gethtml(zurl,'urf-8')

ehtml = etree.HTML(html)

picurl = ehtml.xpath('//*[@class="image_cx_cont"]/img/@src')

for l in picurl:

downPic(l,path) #下载图片

if __name__ == '__main__':

url = 'http://mzsock.com'

savepath='C:\\mzsock' #保存位置

urllist=geturl(url)

pool = ThreadPool(4)

results = pool.map(getdata, urllist)

pool.close()

pool.join()

print('任务全部完成~！')