python 下载文件-Python下载文件的方法

通过python采集时 ，经常需要从html 中获取图片或文件的URL并下载到本地，这里列举最常用的三种模块下载的方法：urllib模块、urllib2模块、requests模块。具体代码如下：

import urllib

import urllib2

import requests

url = 'http://www.test.com/wp-content/uploads/2012/06/wxDbViewer.zip'

print "downloading with urllib"

urllib.urlretrieve(url, "code.zip")

print "downloading with urllib2"

f = urllib2.urlopen(url)

data = f.read()

with open("code2.zip", "wb") as code:

code.write(data)

print "downloading with requests"

r = requests.get(url)

with open("code3.zip", "wb") as code:

code.write(r.content)

看起来使用urllib最为简单，一句语句即可。当然你可以把urllib2缩写成：

f = urllib2.urlopen(url)

with open("code2.zip", "wb") as code:

code.write(f.read())

上面的方法中，还可以设置timeout参数，避免采集一直阻塞。除上面的介绍外，还可以使用pycurl 模块进行下载文件。

import pycurl

import StringIO

##### init the env ###########

c = pycurl.Curl()

c.setopt(pycurl.COOKIEFILE, "cookie_file_name")#把cookie保存在该文件中

c.setopt(pycurl.COOKIEJAR, "cookie_file_name")

c.setopt(pycurl.FOLLOWLOCATION, 1) #允许跟踪来源

c.setopt(pycurl.MAXREDIRS, 5)

#设置代理 如果有需要请去掉注释，并设置合适的参数

#c.setopt(pycurl.PROXY, 'http://11.11.11.11:8080')

#c.setopt(pycurl.PROXYUSERPWD, 'aaa:aaa')

########### get the data && save to file ###########

head = ['Accept:*/*','User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0']

buf = StringIO.StringIO()

curl.setopt(pycurl.WRITEFUNCTION, buf.write)

curl.setopt(pycurl.URL, url)

curl.setopt(pycurl.HTTPHEADER, head)

curl.perform()

the_page =buf.getvalue()

buf.close()

f = open("./%s" % ("img_filename",), 'wb')

f.write(the_page)

f.close()