爬虫爬取人民网

最新推荐文章于 2024-03-04 16:28:45 发布

可我不想做饼干

最新推荐文章于 2024-03-04 16:28:45 发布

阅读量338

点赞数

文章标签： 1024程序员节

本文链接：https://blog.csdn.net/weixin_63403986/article/details/134020519

版权

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy_readbook.items import ScrapyReadbookItem


class ReadSpider(CrawlSpider):
    name = "read"
    output_file = "output.txt"
    # allowed_domains = ["www.dushu.com"]
    # start_urls = ["https://www.dushu.com/book/1188_1.html"]
    allowed_domains = ["finance.people.com.cn"]
    start_urls = ["http://finance.people.com.cn/n1/2023/1024/c1004-40101769.html"]
    # rules = (Rule(
    #     # LinkExtractor(allow=r"/book/1188_\d+.html"),
    #     LinkExtractor(allow=r".*"),
    #               callback="parse_item",
    #               follow=False),)
    rules = (
        Rule(
            LinkExtractor(allow=r"http://finance\.people\.com\.cn/n1/2023/1024/c1004-40101769\.html"),
            callback="parse_item",
            follow=False
        ),
    )

    def parse_item(self, response):
        # img_list = response.xpath('//div[@class="bookslist"]//img')
        # img_list = response.xpath('//div[@class="layout rm_txt cf"]')
        # print('++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')
        # for img in img_list:
        #     title = img.xpath('.//h1/text()').extract_first()
        #     content = img.xpath('.//a/text()').extract_first()
        #     print(title)
        #     print(content)

        print('++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')
        title = response.xpath('/html/body/div[1]/div[7]/div[1]/h1/text()').extract_first()
        # content = response.xpath('/html/body/div[1]/div[7]/div[1]/div[3]//p/text()').extract_first()
        content_list = response.xpath('//p/text()').extract()
        content = '\n'.join(content_list)
        date = response.xpath('/html/body/div[1]/div[7]/div[1]/div[2]/div[1]/text()').extract_first()
        source = response.xpath('/html/body/div[1]/div[7]/div[1]/div[2]/div[1]/a/text()').extract()
        # book = ScrapyReadbookItem(title=title, content=content)
        #
        # yield book
        print(title)
        print(content)
        print(date)
        print(source)
        with open(self.output_file, "a", encoding="utf-8") as file:
            file.write(title + "\n")
            file.write(content + "\n")
            file.write(date + "\n")
            file.write(source + "\n")