爬虫zjzx.py

import scrapy
import re
import requests
from ..items import ZjzxItem
def getURL(No):
    for i in range(No):
        url="http://finance.stockstar.com/finance/macrodata/gdplist.aspx?page={}&order=1&by=1".format(i)
        yield url
class Zjzx1Spider(scrapy.Spider):
    name = 'zjzx1'
    allowed_domains = ['finance.stockstar.com/finance/macrodata/gdplist.aspx']
    start_urls = [url  for url in getURL(4)]
    
    def parse(self, response):
#        pagelist=response.xpath("//div[@class='pager right']//a/text()").extract()
#        page=pagelist[:-1]
        trlist =response.xpath("//div[@class='mainFrame']//tr")
        trlist =trlist[2:]
        for i in range(0,len(trlist)):
            tdlist =trlist[i].xpath("td/text()").extract()
            spanlist=trlist[0].xpath("td/span[@class='red_color']/text()").extract()
            shujuriqi = tdlist[0]
            gnsczzjde = tdlist[1]
            dycyjde =tdlist[2]
            decyjde = tdlist[3]
            dscyjde =tdlist[4]
                
            gnsczztbzj = spanlist[0]
            dycytbzj = spanlist[1]
            decytbzj = spanlist[2]
            dscytbzj = spanlist[3]
            yield ZjzxItem( shujuriqi = shujuriqi , gnsczzjde= gnsczzjde,gnsczztbzj=gnsczztbzj,dycyjde=dycyjde,dycytbzj=dycytbzj,decyjde=decyjde,
                    decytbzj= decytbzj, dscyjde = dscyjde , dscytbzj=dscytbzj )
        for i in range(2,5):
            url="http://finance.stockstar.com/finance/macrodata/gdplist.aspx?page={}&order=1&by=1".format(i)
            yield scrapy.http.Request(url,self.parse,meta={"stockcode":i})