# -*- coding: cp936 -*-
import urllib
import urllib2
import random
import os,sys
from sgmllib import SGMLParser
class URLLister(SGMLParser):
'''''获取html中的图片地址\url地址,装入list中'''
def reset(self):
SGMLParser.reset(self)
self.img = []
self.urls = []
def start_img(self, attrs):
img = [v for k, v in attrs if k=='src']
if img:
self.img.extend(img)
def start_a(self, attrs):
href = [v for k, v in attrs if k=='href']
if href:
self.urls.extend(href)
def get_docum(url):
#url=url+'//'
sock=urllib.urlopen(url)
file=sock.read()
sock.close()
return file
def is_img(url):
global imglenth
reqst=urllib2.Request(url)
opener=urllib2.build_opener()
try:
con=opener.open(reqst)
Type=con.headers.dict['content-type'][:5] #判断链接返回的 content-type是不是图片。
Length =int(con.headers.dict['content-length'])#判断图片大小
if Length>imglenth:
return Type
else:
return 0
except:
print sys.exc_info()[0],sys.exc_info()[1] ##一般来说这样就足够了
print '该图片无法在服务器找到或者图片地址无法识别!'
print url
def get_file_name(ospath,imgname,num):
#name = 'P'+str(random.randint(10000000,99999999))
#filepath = "%s%s.%s" % (ospath,name,(imgname.split('.'))[-1])
#保留原文件名
idx=imgname.rfind("/")
filename=imgname
if idx > -1:
filename=imgname[idx+1:]
filepath=ospath+'%d_'%num+filename
print filepath
return filepath
def get_img(rq):
parser = URLLister(); doc=get_docum(rq); parser.feed(doc); img = parser.img
parser.close()
for i in range(0,len(img)):
if img[i][0:4]!='http':#处理绝对路径
img[i]=rq+img[i]
return img
def get_url(rq):
parser = URLLister(); doc=get_docum(rq); parser.feed(doc); urls = parser.urls
parser.close()
for i in range(0,len(urls)):
if urls[i][0:4] != 'http': #处理绝对路径
urls[i] = rq+urls[i]
return urls
def depth(url,dep,ospath):
'''''三个参数分别是
url : 需要下载的网站地址
dep :需要遍历的深度
ospath:图片下载的本地文件夹
'''
global num
if dep<=0:
return 0
else:
img=get_img(url)
for j in range(0,len(img)):
if is_img(img[j]) == 'image':
filepath = get_file_name(ospath,img[j],num+1);
if (os.path.exists(filepath)):
pass
try:
urllib.urlretrieve(img[j], filepath)
print '已经下载好第%d张图片'%(num+1)
num+=1
except:
print '该图片无法下载或者图片地址无法识别!'
print img[j]
else:
pass
urls=get_url(url)
if len(urls)>0:
for url in urls:
depth(url,dep-1,ospath)
else:
return 0
return 1
if __name__ == '__main__':
num=0
print '默认存储路径为D:\Picture\GetImageFromWeb(在代码中可自行更改)\n'
adress=raw_input('请输入网址:')
imglenth=eval(raw_input('请输入最小图片大小(字节数):'))
dept=eval(raw_input('请输入链接遍历深度:'))
depth(adress,dept,"D:\\Picture\\GetImageFromWeb\\")
print '********************************搞定******************************************'
python图片抓取
最新推荐文章于 2021-03-19 16:50:17 发布