scrapy爬取电竞新闻

进入CMD，cd进要创建项目的文件夹，创建项目 scrapy startproject uuu9。再cd进入项目所在的文件夹，创建爬虫 scrapy genspider -t crawl uuu9_spider “uuu9.com” 引号内的是域名，引号前面的是爬虫名。
setting.py 中机器人协议改为False,加入下载延迟2s,加入User-Agent。

BOT_NAME = 'uuu9'

SPIDER_MODULES = ['uuu9.spiders']
NEWSPIDER_MODULE = 'uuu9.spiders'
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 2
DEFAULT_REQUEST_HEADERS = {
   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
   'Accept-Language': 'en',
   'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
ITEM_PIPELINES = {
    'uuu9.pipelines.Uuu9Pipeline': 300,
}

在爬虫项目所有文件，建立start.py ,可以不用在命令行执行。

from scrapy import cmdline
cmdline.execute("scrapy crawl uuu9_spider".split())

在uuu9_spider.py 中获取所有想要爬取页面的分类，在Rule中用正则模糊匹配，callback使用解析爬取内容的方法 。follow是跟进，true 可以爬取相关的链接，false反之。
电竞和平台游戏标签一致，使用parse_new()方法。dota2是使用另外的标签格式，使用parse_dota2解析。

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from uuu9.items import Uuu9Item

class Uuu9SpiderSpider(CrawlSpider):
    name = 'uuu9_spider'
    allowed_domains = ['uuu9.com']
    start_urls = ['http://uuu9.com/']

    rules = (
        Rule(LinkExtractor(allow=r'.+esports.+shtml'), callback='parse_news', follow=True),
        Rule(LinkExtractor(allow=r'.+pcgame.+shtml'), callback='parse_news', follow=True),
        Rule(LinkExtractor(allow=r'dota2.uuu9.com/.+shtml'), callback='parse_dota2', follow=True), #根据需要加入所求的新闻的分类
    )

    def parse_news(self, response):
        item = {}
        #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
        #item['name'] = response.xpath('//div[@id="name"]').get()
        #item['description'] = response.xpath('//div[@id="description"]').get()
        title = response.xpath('//div[@class="detail"]/h1/text()').get().strip()
        year = response.xpath('//div[@class="article-time-year"]/span/text()').get().strip()
        month = response.xpath('//div[@class="article-time-month"]/span/text()').get().strip()
        content_p = response.xpath('//div[@class="textdetail"]//text()').getall()
        content = "".join(content_p).strip()
        item = Uuu9Item(title=title,year=year,month=month,content=content)
        yield item
    def parse_dota2(self, response):
        item = {}
        title = response.xpath('//div[@class="detail content"]/h1/text()').get().strip()
        time = response.xpath('//div[@class="detail content"]/h3/text()').get().strip()
        content_p = response.xpath('//div[@class="textdetail"]//text()').getall()
        content = "".join(content_p).strip()
        item = Uuu9Item(title=title,time=time,content=content)
        yield item

items.py 定义信息到item，传递到Json中

import scrapy
class Uuu9Item(scrapy.Item):
	title = scrapy.Field()
	year = scrapy.Field()
	time = scrapy.Field()
	content = scrapy.Field()
	month = scrapy.Field()

pipelines.py 中逐行下载写入json

from itemadapter import ItemAdapter
from scrapy.exporters import JsonLinesItemExporter

class Uuu9Pipeline:
	def __init__(self):
		self.fp = open('uuu9.json','wb')
		self.exporter = JsonLinesItemExporter(self.fp,ensure_ascii=False,encoding='utf-8')
	def process_item(self, item, spider):
		self.exporter.export_item(item)
		return item
	def close_spider(self,spider):
		self.fp.close()

使用start.py 执行就可以下载到json数据