#!/usr/bin/python
# -*- coding: utf-8 -*-
import re
import urllib2,urllib
import random
import sys, threading, time
class PicThread(threading.Thread):
def __init__(self,url,path):
threading.Thread.__init__(self)
self.url=url
self.path=path
self.done=False
def run(self):
try:
fr = urllib.urlopen(self.url)
stream = fr.read()
fr.close()
fw = open(self.path, 'wb')
fw.write(stream)
fw.close()
print("Done++!")
except BaseException, e:
print("下载图片时异常信息:")
print(e)
# 有的网站的编码在返回代码的时候,会有BOM编码,这会对解码造成困扰,使用它来返回能正确解析的网页代码
def RemoveBOM(byteArray):
if byteArray[:3] == b'\xef\xbb\xbf':
return byteArray[3:]
else:
return byteArray
#这个是用来得到网页中所有的连接
def FindPictureURLs(htnlContent):
regText='http://[a-z0-9\./_]*'
pattern = re.compile(regText)
return pattern.findall(htnlContent)
#下载图片的函数
def DownImg(url, dirPath):
fr = urllib.urlopen(url)
stream = fr.read()
fr.close()
fw = open(dirPath, 'wb')
fw.write(stream)
fw.close()
#获取网页的标题,这样保存图片的时候,方便按照文件名查找
def GetTitle(self):
ptitle="<title>.*</title>"
title=re.search(ptitle,self,re.I)
try:
return title.group().lstrip("<TITLE>").rstrip("</TITLE>")
except:
return "NoTitle:"+random.choice('abcdefghighklmnopqrstuvwxyz1234567890')
根据传过来的图片的连接,得到图片名称
def GetPicName(sStr1):
spltindex = '/'
index = sStr1.rindex(spltindex)
return sStr1[index+1:]
# 下一个网页地址
def GetNexturl(content):
temp=re.search('</a></span> <span>.*</a></a></span></div>',content)
if not temp==None:
s1=temp.group()
s2='<a href=\''
startindex=s1.find(s2)
s1=s1[startindex+9:]
endindex=s1.find('\'')
s1=s1[0:endindex]
return s1
else:
return 'Not Found'
#下载图片的主要代码
def mymain(content):
myTemp=FindPictureURLs(content)
picNameGroup=GetTitle(content) #title
print picNameGroup
print 'Donwload Now!'
for str in myTemp:
try:
flag = '.'
nPos = str.rindex(flag)
endstr=str[nPos+1:]
if endstr=='jpg' or endstr=='png' or endstr=='gif':
# 获取图片名称
currentName=picNameGroup+GetPicName(str)
#
mypicname=unicode('G:\\pp\\'+currentName,'utf8')
#
t=PicThread(str,mypicname)
t.setDaemon(False)
t.start()
time.sleep(2)
except BaseException, e:
print("主程序中下载出错:"+ str+"\n")
print(e)
v=0
host='网站的第一个图片页面的地址'
print "...........Program Start..........."
while v<7:
content = urllib2.urlopen(mmurl).read()
mmurl=host+GetNexturl(content)
print(mmurl)
mymain(content)
time.sleep(6)
print v
v=v+1
print 'Done!'
一个用来爬行某个网站的图片的测试程序.