# 导入re正则库,用于正则匹配像主名和像主id
import re
# 导入requests库,用于图片抓取和页面访问
import requests
# lxml库用于解析html页面
from lxml import etree
# 导入os库用于创建文件夹,以及保存图片
import os
# 伪造通讯协议头
headers = {
'Connection': 'keep-alive',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Referer': 'http://diglweb.zjlib.cn:8081/zjtsg/zgsmcgzx/cygltotal.jsp?page=67^&channelid=91740',
'Accept-Language': 'zh-CN,zh;q=0.9',
}
# 图片下载函数
def request_download(filename, IMAGE_URL, num):
r = requests.get(IMAGE_URL)
path = f'./image/{filename}'
folder = os.path.exists(path)
# 如果文件夹已存在,则跳过此环节,如果不存在,则创建文件夹
if not folder:
os.makedirs(path)
with open(f'./image/{filename}/{num}.png', 'wb') as f:
f.write(r.content)
# 可以得到一个像主所有图片的函数
def getOnePeople(filename,keyNum):
# 提前设置一个大数组
# 每页页面有二十张图片链接,将一个像主的所有图片链接存放到一个List中,然后for循环下载
BIgImageList = []
# 通过循环和break来确保迭代所有页面
for i in range(1, 10000):
response = requests.get(f'http://diglweb.zjlib.cn:8081/zjtsg/zgsmcgzx/cypicgl.jsp?page={i}&channelid=91743&searchword=%CF%F1%D6%F7%B1%E0%BA%C5={keyNum}', headers=headers, verify=False)
html = etree.HTML(response.text)
imageList = html.xpath("//tr[@style='font-size:12px;']//img/@src")
count = len(imageList)
BIgImageList += imageList
if count < 1:
break
# 下载图片并给予编号
for n, one in enumerate(BIgImageList):
request_download(filename, one, n+1)
# 爬取ID 对应的像主,ID从1到1330
def getmap(record):
params = (
('channelid', '91741'),
('record', record),
)
# 访问对应URL
response = requests.get('http://diglweb.zjlib.cn:8081/zjtsg/zgsmcgzx/cyxl.jsp', headers=headers, params=params, verify=False)
text = response.text
# 通过正则表达式匹配这两个关键信息
keynum = re.findall('td width="285" valign="top">(.*?\d+)</td>', text)[0]
name = re.findall('<td width="285" align="left" valign="bottom">(.*?)</td>', text)[0]
print(name)
print(keynum)
# 运行关键函数
getOnePeople(name, keynum)
# 爬取所有ID,即1300个像主的所有图片
for i in range(1, 1331):
try:
getmap(i)
except:
pass
爬取中国寺庙祠观造像数据库代码
最新推荐文章于 2023-05-02 00:01:03 发布