python--爬取知乎中的图片

首先，我们查看一下知乎的robots协议。

User-agent: *
Disallow: /

知乎是不允许爬取其根目录的。

但是，我们只是用于实验，而且访问频率和正常访问差距不大，所以可以爬取。

先明确目的：

1. 对手动输入的网址进行解析
2. 把爬取到的图片保存到指定目录

__author__ = '_liky'

import requests
from bs4 import BeautifulSoup
import os

def getHTMLText(url):
    try:
        kw = {'user-agent': 'Mozilla/5.0'}
        r = requests.get(url, headers=kw, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        print('Get HTML Message Successfully')
        return r.text
    except:
        return 'Get HTML Message Error'

def ProcessText(text, root):
    soup = BeautifulSoup(text, 'html.parser')
    for link in soup.find_all('img'):
        if str(link['src']).split('.')[-1] == 'jpg' or str(link['src']).split('.')[-1] == 'gif':
            path = root + str(link['src']).split('/')[-1]
            try:
                if not os.path.exists(root):
                    os.mkdir(root)
                if not os.path.exists(path):
                    with open(path, 'wb') as f:
                        f.write(requests.get(link['src']).content) #以二进制的形式写入
                        print('the file ' + str(link['src']).split('/')[-1] + ' has been crawled successfully')
                else:
                    print('the file ' + str(link['src']).split('/')[-1] + ' has existed')
            except:
                print('Process Error')

def main():
    print('请勿用于商业用途')
    url = input('URL (exa: https://www.zhihu.com/question/46508954): ')
    root = input('PATH (exa: D://Picture_ZhiHu//): ')
    text = getHTMLText(url)
    ProcessText(text, root)

if __name__ == '__main__':
    main()

这样就能实现简单的爬取了