python项目_Scrapy_爬取图片???

Main Codes

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

count = 0


class BeautySpider(CrawlSpider):
    name = 'beauty'
    # allowed_domains = ['www.meinv.hk/']  # allowed_domain 中加入http 报错
    # scrapy genspider beauty "http://www.meinv.hk/",然后start_urls 第一个元素有自动添加http://
    start_urls = ['http123://www.meinv1213.hk/']  
    rules = (
        # 因为在要提取的链接后头加入/,出错
        Rule(LinkExtractor(
            # 在正则中?:匹配0个或1个由前面的正则表达式定义的片段，非贪婪方式,利用反斜杠表示?本身
            allow='http://www.meinv.hk/\?p=\d+'),  # r 有啥用,?使用正斜杠/无法爬取
            callback='parse_item',
            ),  # 当加入follow=True 后爬虫仍然会爬取解析到的网页中符合规则的url
    )

    def parse_item(self, response):
        item = {}
        global count
        item['star_name'] = response.xpath('//h1[@class="title"]/text()').get()
        print(item['star_name'])
        item['image_urls'] = response.xpath('//div[@class="post-content"]//img/@src').extract()
        print(item['image_urls'])
        count += 1
        print(count)
        return item  # return 和yield 有啥区别,返回回来的图片张数一样,27


"""
提取首页中的美丽图片规则
"""
# item['star_name'] = response.xpath('//div[@class="posts-default-title"]/h2/a/text()').get()
# print('--------')
# print(item['star_name'])
# item['image_urls'] = response.xpath('//div[@class="posts-default-img"]//img/@src').extract()
# print(item['image_urls'])
# print('---------------')
# # item['description'] = response.xpath('//div[@id="description"]').get()
# return item

# 规则来源
"""
2020-04-27 17:18:04 [scrapy.core.scraper] DEBUG: Scraped from <200 http://www.meinv.hk/?p=2701>
{'star_name': '嘘', 'image_urls': ['http://www.meinv.hk/wp-content/uploads/2018/02/2018020721314999.jpeg', 'http
://www.meinv.hk/wp-content/uploads/2018/02/2018020721314456.jpg']}

"""