首先,我们查看一下知乎的robots协议。
User-agent: *
Disallow: /
知乎是不允许爬取其根目录的。
但是,我们只是用于实验,而且访问频率和正常访问差距不大,所以可以爬取。
先明确目的:
- 对手动输入的网址进行解析
- 把爬取到的图片保存到指定目录
__author__ = '_liky'
import requests
from bs4 import BeautifulSoup
import os
def getHTMLText(url):
try:
kw = {'user-agent': 'Mozilla/5.0'}
r = requests.get(url, headers=kw, timeout=30)
r.raise_for_status()
r.encoding = r.apparent_encoding
print('Get HTML Message Successfully')
return r.text
except:
return 'Get HTML Message Error'
def ProcessText(text, root):
soup = BeautifulSoup(text, 'html.parser')
for link in soup.find_all('img'):
if str(link['src']).split('.')[-1] == 'jpg' or str(link['src']).split('.')[-1] == 'gif':
path = root + str(link['src']).split('/')[-1]
try:
if not os.path.exists(root):
os.mkdir(root)
if not os.path.exists(path):
with open(path, 'wb') as f:
f.write(requests.get(link['src']).content) #以二进制的形式写入
print('the file ' + str(link['src']).split('/')[-1] + ' has been crawled successfully')
else:
print('the file ' + str(link['src']).split('/')[-1] + ' has existed')
except:
print('Process Error')
def main():
print('请勿用于商业用途')
url = input('URL (exa: https://www.zhihu.com/question/46508954): ')
root = input('PATH (exa: D://Picture_ZhiHu//): ')
text = getHTMLText(url)
ProcessText(text, root)
if __name__ == '__main__':
main()
这样就能实现简单的爬取了