#创建img目录
os.makedirs('./img/', exist_ok=True)
#1
IMAGE_URL = "https://img3.doubanio.com/view/subject/l/public/s4676930.jpg"
from urllib.request import urlretrieve
urlretrieve(IMAGE_URL, './img/image1.jpg') # whole document
#2
import requests
r = requests.get(IMAGE_URL)
with open('./img/image2.jpg', 'wb') as f:
f.write(r.content) # whole document
#3
#所以说, 如果你要下载的是大文件, 比如视频等. requests 能让你下一点, 保存一点, 而不是要全部下载完才能保存去另外的地方. 这就是一个 chunk 一个 chunk 的下载. 使用 r.iter_content(chunk_size)
来控制每个 chunk 的大小, 然后在文件中写入这个 chunk 大小的数据.
#stream=True 时刻下载
r = requests.get(IMAGE_URL, stream=True) # stream loading with open('./img/image3.png', 'wb') as f: for chunk in r.iter_content(chunk_size=32): f.write(chunk)
4例子:
#http://www.ngchina.com.cn
from bs4 import BeautifulSoup
import requests
URL = "http://www.ngchina.com.cn/photography/photo_of_the_day/"
html = requests.get(URL).text
soup = BeautifulSoup(html, 'lxml')
img_ul = soup.find_all('ul', {"class": "cf showImg-list"})
import os
os.makedirs('./img/', exist_ok=True)
for ul in img_ul:
imgs = ul.find_all('img')
for img in imgs:
url = img['src']
r = requests.get(url, stream=True)
image_name = url.split('/')[-1]
with open('./img/%s' % image_name, 'wb') as f:
for chunk in r.iter_content(chunk_size=128):
f.write(chunk)
print('Saved %s' % image_name)
5.例子:
抓取豆瓣‘编程’分类的一个连接URL里各书图片#还是只能抓一个页面,太简单了~~
import requests
from bs4 import BeautifulSoup as bs
import os
#以豆瓣‘编程’分类的一个连接URL为例子开始爬数据ID
url = 'https://book.douban.com/tag/编程?start=20&type=T'
res = requests.get(url) #发送请求
#print(res.encoding) #这个是用来查看网页编码的
#res.encoding = 'utf-8' #跟上一个结合来用,如果编码有乱码,则可以通过这个定义编码来改变
html = res.text
#print(html)
IDs = []
soup = bs(html,"html.parser") #定义一个BeautifulSoup变量
items = soup.find_all('a',attrs={'class':'nbg'})
#print(items)
os.makedirs('./img/', exist_ok=True)
for ul in items:
imgs = ul.find_all('img')
for img in imgs:
url = img['src']
r = requests.get(url, stream=True)
image_name = url.split('/')[-1]
with open('./img/%s' % image_name, 'wb') as f:
for chunk in r.iter_content(chunk_size=128):
f.write(chunk)
print('Saved %s' % image_name)