【Class 48】【实例】python爬虫实现 下载所有 XKCD 漫画

下载所有 XKCD 漫画

#! python3
# -*- coding: utf-8 -*-

import requests, os, bs4

url = "http://xkcd.com"
os.makedirs('xkcd',exist_ok=True)

while not url.endswith('#'):

    # 下载漫画
    print('Downloading page %s...' % url)
    res = requests.get(url)
    res.raise_for_status()

    # 解析网页
    soup = bs4.BeautifulSoup(res.text,'html.parser')

    # 寻找图片链接
    comicElem = soup.select('#comic img')
    if comicElem == []:
        print('Could not find comic image.')
    else:
        comicUrl = 'http:'+comicElem[0].get('src')

    # 下载图片
    print('Downloading image %s...' % (comicUrl))
    res = requests.get(comicUrl)
    res.raise_for_status()

    # 保存图片
    imageFile = open(os.path.join('xkcd', os.path.basename(comicUrl)), 'wb')
    for chunk in res.iter_content(100000):
        imageFile.write(chunk)
    imageFile.close()

    # 获得前一页的链接
    prevLink = soup.select('a[rel="prev"]')[0]
    url = 'http://xkcd.com' + prevLink.get('href')

print('Done')

下载结果：

Downloading image http://imgs.xkcd.com/comics/light_pollution.png...
Downloading page http://xkcd.com/2120/...
Downloading image http://imgs.xkcd.com/comics/brain_hemispheres.png...
Downloading page http://xkcd.com/2119/...
Downloading image http://imgs.xkcd.com/comics/video_orientation.png...
Downloading page http://xkcd.com/2118/...
Downloading image http://imgs.xkcd.com/comics/normal_distribution.png...
Downloading page http://xkcd.com/2117/...
Downloading image http://imgs.xkcd.com/comics/differentiation_and_integration.png...
Downloading page http://xkcd.com/2116/...
Downloading image http://imgs.xkcd.com/comics/norm_normal_file_format.png...
Downloading page http://xkcd.com/2115/...
Downloading image http://imgs.xkcd.com/comics/plutonium.png...
Downloading page http://xkcd.com/2114/...
Downloading image http://imgs.xkcd.com/comics/launch_conditions.png...
Downloading page http://xkcd.com/2113/...
Downloading image http://imgs.xkcd.com/comics/physics_suppression.png...
Downloading page http://xkcd.com/2112/...
Downloading image http://imgs.xkcd.com/comics/night_shift.png...
Downloading page http://xkcd.com/2111/...
Downloading image http://imgs.xkcd.com/comics/opportunity_rover.png...
Downloading page http://xkcd.com/2110/...
Downloading image http://imgs.xkcd.com/comics/error_bars.png...