from
scrapy.spider import
BaseSpider
from
scrapy.selector
import
HtmlXPathSelector
class
MySpider(BaseSpider):
name
=
'myspider'
start_urls
=
(
'http://'
,
'http://'
,
)
def
parse(
self
, response):
# collect `item_urls`
for
item_url
in
item_urls:
yield
Request(url
=
item_url, callback
=
self
.parse_item_list)
def
parse_item_list(
self
, response):
item
=
MyItem()
# populate `item` fields
yield
Request(url
=
item_details_url, meta
=
{
'item'
: item},
callback
=
self
.parse_details)
def
parse_details(
self
, response):
item
=
response.meta[
'item'
]
# populate more `item` fields
return
item
from
scrapy.contrib.linkextractors.sgml
import
SgmlLinkExtractor
class
MininovaSpider(CrawlSpider):
name
=
''
allowed_domains
=
[
''
]
start_urls
=
[
''
]
rules
=
[Rule(SgmlLinkExtractor(allow
=
[
'/tor/\d+'
])),
Rule(SgmlLinkExtractor(allow
=
[
'/abc/\d+'
]),
'parse_torrent'
)]
def
parse_torrent(
self
, response):
x
=
HtmlXPathSelector(response)
torrent
=
TorrentItem()
torrent[
'url'
]
=
response.url
torrent[
'name'
]
=
x.select(
"//h1/text()"
).extract()
torrent[
'description'
]
=
x.select(
"//div[@id='description']"
).extract()
torrent[
'size'
]
=
x.select(
"//div[@id='info-left']/p[2]/text()[2]"
).extract()
return
torrent