Scrapy spider代码片段

from  scrapy.spider import  BaseSpider

from  scrapy.selector  import  HtmlXPathSelector

class  MySpider(BaseSpider):

    name  =  'myspider'

    start_urls  =  (

         'http://' ,

         'http://' ,

        )

     def  parse( self , response):

         # collect `item_urls`

         for  item_url  in  item_urls:

             yield  Request(url = item_url, callback = self .parse_item_list)

     def  parse_item_list( self , response):

        item  =  MyItem()

         # populate `item` fields

         yield  Request(url = item_details_url, meta = { 'item' : item},

            callback = self .parse_details)

     def  parse_details( self , response):

        item  =  response.meta[ 'item' ]

         # populate more `item` fields

         return  item

from  scrapy.contrib.linkextractors.sgml  import  SgmlLinkExtractor

class  MininovaSpider(CrawlSpider):

    name  =  ''

    allowed_domains  =  [ '' ]

    start_urls  =  [ '' ]

    rules  =  [Rule(SgmlLinkExtractor(allow = [ '/tor/\d+' ])),

             Rule(SgmlLinkExtractor(allow = [ '/abc/\d+' ]), 'parse_torrent' )]

     def  parse_torrent( self , response):

        x  =  HtmlXPathSelector(response)

        torrent  =  TorrentItem()

        torrent[ 'url' ]  =  response.url

        torrent[ 'name' ]  =  x.select( "//h1/text()" ).extract()

        torrent[ 'description' ]  = x.select( "//div[@id='description']" ).extract()

        torrent[ 'size' ]  =  x.select( "//div[@id='info-left']/p[2]/text()[2]" ).extract()

         return  torrent