Python 采集某网站的壁纸

一、运行环境

pycharm 2020 社区版

python 3.7

beautifulsoup4 4.11.1

二、实际代码

import os
import re

import requests
from bs4 import BeautifulSoup

from access.sprider.SpriderAccess import SpriderAccess
from base.BaseConfig import BaseConfig
from base.BaseFrame import BaseFrame
from object.entity.SpriderEntity import SpriderEntity
from plugin.Tools import Tools


class Netbian:
    page_count = 1  # 每个栏目开始业务
    base_url = "http://pic.netbian.com"  # 采集的网址 http://pic.netbian.com/e/search/result/?searchid=2543
    save_path = BaseConfig().CORPUS_ROOT + os.sep + "Netbian"  # "/Users/zhangyu/Pictures/Wallpaper/"
    second_url = ("4kmeinv","4kfengjing",  "4kbeijing", "4kyouxi")

    def __init__(self):
        pass

    def sprider_wall_paper(self):
        BaseFrame.__log__("开始采集首页彼岸图网网站的图片...")
        for column in self.second_url:
            url = self.base_url + "/" + column + "/"
            response = requests.get(url)
            response.encoding = 'gbk'
            soup = BeautifulSoup(response.text, "html5lib")
            try:
                page_end_url = soup.find('span', attrs={"class": 'slh'}).find_next_siblings()
                page_end_num = page_end_url[0].text
                while self.page_count <= int(page_end_num):  # 翻完停止
                    if self.page_count == 1:  # 栏目的第一个地址
                        list_url = soup.find_all('a', attrs={"target": '_blank'})
                        regx = "tupian/\d{1,5}\.html"
                        images_url = re.findall(regx, str(list_url))
                        for iurl in images_url:
                            image_full_url = self.base_url + "/" + iurl
                            response = requests.get(image_full_url)
                            response.encoding = 'gbk'
                            soup = BeautifulSoup(response.text, "html5lib")
                            for image_obj in soup.find('a', attrs={"id": 'img'}).children:
                                pic_url = self.base_url + image_obj.get("src")
                                pic_title = image_obj.get("alt")
                                BaseFrame.__log__("采集" + pic_title + "的图片..." + url)
                                mmEntity = SpriderEntity()  # 依据图片执行，下载过的图片不再下载
                                mmEntity.sprider_base_url = self.base_url
                                mmEntity.create_datetime = Tools.get_current_datetime()
                                mmEntity.sprider_url = url
                                mmEntity.sprider_pic_title = pic_title
                                mmEntity.sprider_pic_index = str(self.page_count)
                                if SpriderAccess().query_sprider_entity_by_urlandtitle(pic_url, pic_title) is None:
                                    SpriderAccess().save_sprider(mmEntity)
                                    self.down_pic(pic_url, pic_title, column)
                                else:
                                    BaseFrame.__log__("下载过已经跳过。")
                    else:
                        next_pager_url = self.base_url + "/" + column + "/index_" + str(self.page_count) + ".html"
                        try:
                            response = requests.get(next_pager_url)
                            response.encoding = 'gbk'
                            soup = BeautifulSoup(response.text, "html5lib")
                            list_url = soup.find_all('a', attrs={"target": '_blank'})
                            regx = "tupian/\d{1,5}\.html"
                            images_url = re.findall(regx, str(list_url))
                            for iurl in images_url:
                                image_full_url = self.base_url + "/" + iurl
                                response = requests.get(image_full_url)
                                response.encoding = 'gbk'
                                soup = BeautifulSoup(response.text, "html5lib")
                                for image_obj in soup.find('a', attrs={"id": 'img'}).children:

                                    pic_url = self.base_url + image_obj.get("src")
                                    pic_title = image_obj.get("alt")
                                    BaseFrame.__log__("采集" + pic_title + "的图片..." + url)
                                    mmEntity = SpriderEntity()  # 依据图片执行，下载过的图片不再下载
                                    mmEntity.sprider_base_url = self.base_url
                                    mmEntity.create_datetime = Tools.get_current_datetime()
                                    mmEntity.sprider_url = url
                                    mmEntity.sprider_pic_title = pic_title
                                    mmEntity.sprider_pic_index = str(self.page_count)
                                    if SpriderAccess().query_sprider_entity_by_urlandtitle(pic_url, pic_title) is None:
                                        SpriderAccess().save_sprider(mmEntity)
                                        self.down_pic(pic_url, pic_title, column)
                                    else:
                                        BaseFrame.__log__("下载过已经跳过。")
                        except Exception as e:
                            BaseFrame.__log__("请求站点过程发生错误..." + e)
                            continue
                        pass
                    self.page_count = self.page_count + 1  # 翻页要不一致第一页
            except Exception as e:
                BaseFrame.__err__(str(e))
                continue
        pass

    # region 下载图片
    def down_pic(self, pic_url, pic_title, second_path):
        try:
            headers = {"Referer": pic_url,
                       'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
                                     '(KHTML, like Gecko)Chrome/62.0.3202.94 Safari/537.36'}
            content = requests.get(pic_url, headers=headers)

            real_path = self.save_path + os.sep + second_path + os.sep
            if (os.path.exists(real_path) is False):
                os.makedirs(real_path)
            if content.status_code == 200:
                pic_cun = real_path + pic_title + '.jpg'
                fp = open(pic_cun, 'wb')
                fp.write(content.content)
                fp.close()
        except Exception as e:
            BaseFrame.__err__("下载图片出现错误" + str(e))
        pass
    # endregion


if __name__ == '__main__':
    Netbian().sprider_wall_paper()
pass