python批量下载网页文件夹_【zz】Python2 批量下载网站文件

#! encoding=utf-8

importurllib2importreimportosdefDownload(url,output):print "downloading..."+url

response=urllib2.urlopen(url)

resourceFile= open(output,"wb")

resourceFile.write(response.read())

resourceFile.close()print "downloaded"

def Action(url,ext = "pdf",output = "."):#1.domain

index = url.rfind("/");

domain= url[0:index+1];printdomain

request=urllib2.Request(url)

response=urllib2.urlopen(request)#2.content

content =response.read()#print content

#3.resource

mode = '\"([^\"]+'+ext+')\"'pattern=re.compile(mode)

strMatch=pattern.findall(content)

size=len(strMatch)print "file num:"+str(size)for i in range(0,size,1):#print strMatch[i]

one =strMatch[i]

partIndex= one.rfind('/')if not one.startswith('http://'):if -1!=partIndex:

directDir= one[0:partIndex+1]else:

directDir= ""

#print directDir

try:

os.makedirs(output+"/"+directDir)exceptException,e:passfileUrl= domain+one

fileOutput= output+"/"+oneprintfileUrlprintfileOutput

Download(fileUrl,fileOutput)else:printoneprint "........."

printone[partIndex:]

fileOutput= output+"/"+one[partIndex:]printfileOutput

Download(one,fileOutput)#5.download

if __name__=='__main__':print "download"url= "http://compgeom.cs.uiuc.edu/~jeffe/teaching/algorithms/";

Action("http://tech.qq.com/","jpg");