spiders --- top17173

# -*- coding: utf-8 -*-
import scrapy
from demo1.items import GameItem

class Top17173Spider(scrapy.Spider):
    name = 'top17173'
    allowed_domains = ['top17173.com']
    start_urls = ['http://top.17173.com/list-0-0-0-0-0-0-0-0-0-0-1.html','http://top.17173.com/list-0-0-4-0-0-0-0-0-0-0-1.html','http://top.17173.com/vr-index.shtml']

    def parse(self, response):
        name = response.xpath('//div[@class="item-in"]/div[@class="c2"]//a/text()').extract()
        poll = response.xpath('//div[@class="item-in"]/div[@class="c3"]/text()').extract()
        polls = []
        names =  []
        for p in poll:
            p = ''.join(p.split())
            polls.append(p)

        for n in name:
            n = ''.join(n.split())
            names.append(n)

        # print(polls)
        # print(names)
        item = GameItem()
        for name,poll in zip(names,polls):
            item['name']=name
            item['poll']=poll

            if response.url.find('list-0-0-0-0-0-0-0-0-0-0-1.html')!=-1:
                item['type']='网游'

            elif response.url.find('list-0-0-4-0-0-0-0-0-0-0-1.html')!=-1:

                item['type']='手游'

            else:
                item['type']='VR'

            yield item

 

 

#创建item对象类

items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class GameItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field()
    poll = scrapy.Field()
    type = scrapy.Field()

 

 

 

 

#保存文件

pipelines.py

 

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json

class Demo1Pipeline(object):
    def open_spider(self,spider):
        self.file = open('game.txt','a',encoding='utf-8')
    def process_item(self, item, spider):
        # if item['type']=='网游':
        #     with open('wangyou.txt','w',encoding='utf-8')
        self.file.write(json.dumps(dict(item),ensure_ascii=False)+'\n')

        print(item)
        self.file.flush()

        return item

    def close_spider(self,spider):
        self.file.close()