Scrapy 简单爬取厨房网站菜谱清单，并将结果保存为csv文件

链接：http://www.xiachufang.com/explore/

from scrapy import Request
from scrapy.spiders import Spider

class HotSaleSpoider(Spider):
    name = 'hot'
    qidian_headers = {"user-agent":'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46'}
    def start_requests(self):
        url = 'https://www.xiachufang.com/explore/'
        yield Request(url, headers=self.qidian_headers, callback=self.parse)
    def parse(self, response):
        list_selector = response.xpath("//div[@class='info pure-u']")
        for one in list_selector:
            str1 = "https://www.xiachufang.com/"
            name = one.xpath("p[@class='name']/a/text()").extract()[0]
            composition = one.xpath("p[@class='ing ellipsis']/a/text()").extract()
            url = one.xpath("p[@class='name']/a/@href").extract()[0]
            url = "".join(url)
            url = str1+url
            print(url)
            composition = ",".join(composition)
            hot_dict = {"菜名": name,
                        "原材料": composition,
                        'url': url}

            yield hot_dict

效果：

 

 

运行命令：scrapy crawl hot -o hot.csv