采集雅拉伊MM图
Python
# https://www.yalayi.com/gallery/
import os
import re
import requests
from bs4 import BeautifulSoup
from access.sprider.SpriderAccess import SpriderAccess
from base.BaseConfig import BaseConfig
from base.BaseFrame import BaseFrame
from object.entity.SpriderEntity import SpriderEntity
from plugin.Tools import Tools
class Yalayi:
page_count = 1 # 每个栏目开始业务
base_url = "https://www.yalayi.com" # 采集的网址
base_hot = "https://www.yalayi.com/gallery/"
save_path = BaseConfig().CORPUS_ROOT + os.sep + "Yalayi"
def __init__(self):
# https://www.yalayi.com/gallery/210.html
# https://www.yalayi.com/gallery/203.html
pass
def get_max_pager_index(self, soup):
max_index_str = soup.find("a", text='尾页').get("href")
max_index = str(max_index_str).split(".")[0].split("_")[1]
return int(max_index)
pass
#region 采集美女图片
def sprider_mm_pic(self):
BaseFrame.__log__("开始采集youzi4网站高清图片...")
try:
first_url = self.base_hot.format(self.page_count)
response = requests.get(first_url, timeout=10)
except Exception as e:
BaseFrame.__log__("请求一级栏目出现错误,倒霉...重试()" + str(e))
#self.sprider_mm_pic() # 重新执行一下
return
response.encoding = 'UTF-8'
soup = BeautifulSoup(response.text, "html5lib")
page_end_number = self.get_max_pager_index(soup)
while self.page_count <= int(page_end_number): # 翻完停止
if self.page_count == 1: # 栏目的第一个地址
regx = "https://www\S*/\d{1,4}\.html"
images = re.findall(regx, str(soup))
for url in images:
self.sprider_image(url)
else:
try:
next_url_str = self.base_hot + "/" + "index_{0}.html"
next_url = next_url_str.format(self.page_count)
response = requests.get(next_url, timeout=10)
response.encoding = 'UTF-8'
soup = BeautifulSoup(response.text, "html5lib")
regx = "https://www\S*/\d{1,4}\.html"
images = re.findall(regx, str(soup))
for url in images:
self.sprider_image(url)
except Exception as e:
BaseFrame.__log__("请求第"+self.page_count+"页出现错误,已经跳过" + str(e))
continue
pass
self.page_count = self.page_count + 1
pass
#endregion
# region 构建网址下所有图片的地址(图片很少没有分页)
# 原创美女写真摄影作品列表 - 雅拉伊
def sprider_image(self, url):
try:
response = requests.get(url, timeout=3) # 图片地址
response.encoding = 'UTF-8'
soup = BeautifulSoup(response.text, "html5lib")
pic_title = str(soup.find("title").string).split("-")[0].strip()
BaseFrame.__log__("采集" + pic_title + "的图片..." + url)
images = soup.findAll('img', attrs={"class": 'lazy'})
image_index = 1
for image in images:
url = image.get("data-src")
mmEntity = SpriderEntity() # 依据图片执行,下载过的图片不再下载
mmEntity.sprider_base_url = self.base_url
mmEntity.create_datetime = Tools.get_current_datetime()
mmEntity.sprider_url = url
mmEntity.sprider_pic_title = pic_title
mmEntity.sprider_pic_index = str(image_index)
if SpriderAccess().query_sprider_entity_by_urlandindex(url, str(image_index)) is None:
SpriderAccess().save_sprider(mmEntity)
self.down_pic(url, pic_title, image_index)
else:
BaseFrame.__log__(url+"数据采集过因此跳过")
image_index = image_index + 1
except Exception as e:
BaseFrame.__log__("请求过程出先错误...跳过本次请求" + str(e))
return
# endregion
# region 下载图片
def down_pic(self, pic_url, pic_title, index):
try:
content = requests.get(pic_url)
if content.status_code == 200:
real_path = self.save_path+os.sep + pic_title + os.sep
if (os.path.exists(real_path) is False):
os.makedirs(real_path)
pic_title = pic_title + str(index);
pic_cun = real_path + pic_title + '.jpg'
fp = open(pic_cun, 'wb')
fp.write(content.content)
fp.close()
else:
pass
except Exception as e:
BaseFrame.__log__("下载图片过程出现错误" + str(e))
return
pass
# endregion
if __name__ == '__main__':
Yalayi().sprider_mm_pic()
pass
采集雅拉伊MM图
都是高清大图