import scrapy
from scrapy.spider import Spider
from scrapy.selector import Selector
from scrapy.http import Request
from scrapy import Item, Field
class NItems(Item):
creat = Field()
class NSpider(Spider):
def parse(self,response):
item = NItems()
for da in response.xpath('//div[@class="section-stream-content"]'):
item['link'] = da.xpath('//tr/td[2]/div[1]/h2/a/@href').extract()
for con in item['link']:
request = scrapy.Request(con, callback = self.parseDate)
request.meta['item'] = item
yield request
def parseDate(self, response):
item = response.meta['item']
hxs = Selector(response)
cr = hxs.xpath('//*[@class="byline"]/text() | //*[@id="yui_3_16_0_1