网页抓取学习（6）下载图片

import os
#创建img目录
os.makedirs('./img/', exist_ok=True)

#1
IMAGE_URL = "https://img3.doubanio.com/view/subject/l/public/s4676930.jpg"
from urllib.request import urlretrieve

urlretrieve(IMAGE_URL, './img/image1.jpg')      # whole document

#2

import requests
r = requests.get(IMAGE_URL)
with open('./img/image2.jpg', 'wb') as f:
    f.write(r.content)                      # whole document

#3

#所以说, 如果你要下载的是大文件, 比如视频等. requests 能让你下一点, 保存一点, 而不是要全部下载完才能保存去另外的地方. 这就是一个 chunk 一个 chunk 的下载. 使用 r.iter_content(chunk_size) 来控制每个 chunk 的大小, 然后在文件中写入这个 chunk 大小的数据.

#stream=True 时刻下载

r = requests.get(IMAGE_URL, stream=True)    # stream loading

with open('./img/image3.png', 'wb') as f:
    for chunk in r.iter_content(chunk_size=32):
        f.write(chunk)

4例子：

#http://www.ngchina.com.cn
from bs4 import BeautifulSoup
import requests


URL = "http://www.ngchina.com.cn/photography/photo_of_the_day/"


html = requests.get(URL).text
soup = BeautifulSoup(html, 'lxml')
img_ul = soup.find_all('ul', {"class": "cf showImg-list"})


import os


os.makedirs('./img/', exist_ok=True)
for ul in img_ul:
    imgs = ul.find_all('img')
    for img in imgs:
        url = img['src']
        r = requests.get(url, stream=True)
        image_name = url.split('/')[-1]
        with open('./img/%s' % image_name, 'wb') as f:
            for chunk in r.iter_content(chunk_size=128):
                f.write(chunk)

        print('Saved %s' % image_name)

5.例子：

抓取豆瓣‘编程’分类的一个连接URL里各书图片#还是只能抓一个页面,太简单了~~

import requests
from bs4 import BeautifulSoup as bs
import os


#以豆瓣‘编程’分类的一个连接URL为例子开始爬数据ID
url = 'https://book.douban.com/tag/编程?start=20&type=T'
res = requests.get(url)  #发送请求
#print(res.encoding)    #这个是用来查看网页编码的
#res.encoding = 'utf-8'   #跟上一个结合来用，如果编码有乱码，则可以通过这个定义编码来改变
html = res.text     
#print(html)


IDs = []
soup  = bs(html,"html.parser")     #定义一个BeautifulSoup变量
items = soup.find_all('a',attrs={'class':'nbg'})
#print(items)




os.makedirs('./img/', exist_ok=True)
for ul in items:
    imgs = ul.find_all('img')
    for img in imgs:
        url = img['src']
        r = requests.get(url, stream=True)
        image_name = url.split('/')[-1]
        with open('./img/%s' % image_name, 'wb') as f:
            for chunk in r.iter_content(chunk_size=128):
                f.write(chunk)
        print('Saved %s' % image_name)