源码系列1:
大概的思路和上一篇差不多,不同的是不同的网站有不同的反爬策略
爬取的网站是:https://www.pexels.com/
参考:https://blog.csdn.net/qq_32511479/article/details/76050973
源码如下:
import requests
import re
import os
import time
def get_url(url):
kw = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)'}
try:
r = requests.get(url, headers=kw)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r
except:
print('error!')
def get_photourl(photo_url):
kw = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)'}
try:
r = requests.get(photo_url, headers=kw)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r
except:
return 'error'
def get_photos(url, new_fpath):
result = get_url(url)
pattern = re.compile(r'src="https://images.pexels.com/photos/(\d+)/(.*?)\?h=350&auto=compress&cs=tinysrgb"', re.S)
items = re.findall(pattern, result.text)
for item in items:
try:
photo_url = 'https://static.pexels.com/photos/' + str(item[0]) + '/' + str(item[1])
# 把图片链接中的images,改成了static
save(photo_url, item, new_fpath)
time.sleep(1)
except:
continue
def makedir(new_fpath, i, key):
E = os.path.exists(new_fpath)
if not E:
os.makedirs(new_fpath)
os.chdir(new_fpath)
print('文件夹' + key + '_page' + str(i + 1) + '创建成功!')
else:
print('文件夹已存在!')
def save(photo_url, item, new_fpath):
Final_fpath = new_fpath + '/' + str(item[0]) + str(item[1])
print('正在下载图片......')
result = get_photourl(photo_url)
if result != 'error':
print('下载成功!'