python beautifulsoup下载_python – 使用请求和BeautifulSoup下载文件

这会将页面中包含原始文件名的所有文件写入pdfs /目录.

import requests

from bs4 import BeautifulSoup as bs

import urllib2

_ANO = '2013/'

_MES = '01/'

_MATERIAS = 'matematica/'

_CONTEXT = 'wp-content/uploads/' + _ANO + _MES

_URL = 'http://www.desconversa.com.br/' + _MATERIAS + _CONTEXT

# functional

r = requests.get(_URL)

soup = bs(r.text)

urls = []

names = []

for i, link in enumerate(soup.findAll('a')):

_FULLURL = _URL + link.get('href')

if _FULLURL.endswith('.pdf'):

urls.append(_FULLURL)

names.append(soup.select('a')[i].attrs['href'])

names_urls = zip(names, urls)

for name, url in names_urls:

print url

rq = urllib2.Request(url)

res = urllib2.urlopen(rq)

pdf = open("pdfs/" + name, 'wb')

pdf.write(res.read())

pdf.close()