采集雅拉伊MM图

采集雅拉伊MM图

Python

# https://www.yalayi.com/gallery/
import os
import re

import requests
from bs4 import BeautifulSoup

from access.sprider.SpriderAccess import SpriderAccess
from base.BaseConfig import BaseConfig
from base.BaseFrame import BaseFrame
from object.entity.SpriderEntity import SpriderEntity
from plugin.Tools import Tools


class Yalayi:
    page_count = 1  # 每个栏目开始业务
    base_url = "https://www.yalayi.com"  # 采集的网址
    base_hot = "https://www.yalayi.com/gallery/"
    save_path = BaseConfig().CORPUS_ROOT + os.sep + "Yalayi"

    def __init__(self):
        # https://www.yalayi.com/gallery/210.html
        # https://www.yalayi.com/gallery/203.html
        pass

    def get_max_pager_index(self, soup):
        max_index_str = soup.find("a", text='尾页').get("href")
        max_index = str(max_index_str).split(".")[0].split("_")[1]
        return int(max_index)
        pass

    #region 采集美女图片
    def sprider_mm_pic(self):
        BaseFrame.__log__("开始采集youzi4网站高清图片...")
        try:
            first_url = self.base_hot.format(self.page_count)
            response = requests.get(first_url, timeout=10)
        except Exception as e:
            BaseFrame.__log__("请求一级栏目出现错误，倒霉...重试()" + str(e))
            #self.sprider_mm_pic()  # 重新执行一下
            return
        response.encoding = 'UTF-8'
        soup = BeautifulSoup(response.text, "html5lib")
        page_end_number = self.get_max_pager_index(soup)
        while self.page_count <= int(page_end_number):  # 翻完停止
            if self.page_count == 1:  # 栏目的第一个地址
                regx = "https://www\S*/\d{1,4}\.html"
                images = re.findall(regx, str(soup))
                for url in images:
                    self.sprider_image(url)
            else:
                try:
                    next_url_str = self.base_hot + "/" + "index_{0}.html"
                    next_url = next_url_str.format(self.page_count)
                    response = requests.get(next_url, timeout=10)
                    response.encoding = 'UTF-8'
                    soup = BeautifulSoup(response.text, "html5lib")
                    regx = "https://www\S*/\d{1,4}\.html"
                    images = re.findall(regx, str(soup))
                    for url in images:
                        self.sprider_image(url)
                except Exception as e:
                    BaseFrame.__log__("请求第"+self.page_count+"页出现错误，已经跳过" + str(e))
                    continue
                pass
            self.page_count = self.page_count + 1
        pass
    #endregion

    # region 构建网址下所有图片的地址（图片很少没有分页）
    # 原创美女写真摄影作品列表 - 雅拉伊
    def sprider_image(self, url):
        try:
            response = requests.get(url, timeout=3)  # 图片地址
            response.encoding = 'UTF-8'
            soup = BeautifulSoup(response.text, "html5lib")
            pic_title = str(soup.find("title").string).split("-")[0].strip()
            BaseFrame.__log__("采集" + pic_title + "的图片..." + url)
            images = soup.findAll('img', attrs={"class": 'lazy'})
            image_index = 1

            for image in images:
                url = image.get("data-src")
                mmEntity = SpriderEntity()  # 依据图片执行，下载过的图片不再下载
                mmEntity.sprider_base_url = self.base_url
                mmEntity.create_datetime = Tools.get_current_datetime()
                mmEntity.sprider_url = url
                mmEntity.sprider_pic_title = pic_title
                mmEntity.sprider_pic_index = str(image_index)
                if SpriderAccess().query_sprider_entity_by_urlandindex(url, str(image_index)) is None:
                    SpriderAccess().save_sprider(mmEntity)
                    self.down_pic(url, pic_title, image_index)
                else:
                    BaseFrame.__log__(url+"数据采集过因此跳过")
                image_index = image_index + 1
        except Exception as e:
            BaseFrame.__log__("请求过程出先错误...跳过本次请求" + str(e))
            return

    # endregion

    # region 下载图片
    def down_pic(self, pic_url, pic_title, index):
        try:
            content = requests.get(pic_url)
            if content.status_code == 200:
                real_path = self.save_path+os.sep + pic_title + os.sep
                if (os.path.exists(real_path) is False):
                    os.makedirs(real_path)
                pic_title = pic_title + str(index);
                pic_cun = real_path + pic_title + '.jpg'
                fp = open(pic_cun, 'wb')
                fp.write(content.content)
                fp.close()
            else:
                pass
        except Exception as e:
            BaseFrame.__log__("下载图片过程出现错误" + str(e))
            return
        pass
    # endregion


if __name__ == '__main__':
    Yalayi().sprider_mm_pic()
pass

采集雅拉伊MM图

都是高清大图