原http://www.dmoz.org已经重定向到dmoztools.net,html也有所改变,所以不能照搬参考文档。
Scrapy的安装和使用请参考http://docs.pythontab.com/scrapy/scrapy0.24/intro/tutorial.html。
源码:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from scrapy.spider import Spider
from scrapy.selector import Selector
from tutorial.items import DmozItem
class DmozSpider(Spider):
name = "dmoz"
allowed_domains = ["dmoztools.net"]
start_urls = [
"http://dmoztools.net/Computers/Programming/Languages/Python/Books/",
"http://dmoztools.net/Computers/Programming/Languages/Python/Resources/"
]
#http://dmoztools.net/Computers/Programming/Languages/Python/Resources/
#http://dmoztools.net/Computers/Programming/Languages/Python/Books/
def parse(self,response):
""" demo 1
#filename = response.url.split("/")[-2]
#open(filename,'wb').write(response.body)
"""
""" demo 2
sel = Selector(response)
sites = sel.xpath('//div[@class="title-and-desc"]')
for site in sites:
#title = site.xpath('a/div[@class="site-title"]/text()').extract()
#link = site.xpath('a/@href').extract()
desc = site.xpath('div[@class="site-descr "]/text()').extract()
#print title
#print link
print desc
"""
sel = Selector(response)
sites = sel.xpath('//div[@class="title-and-desc"]')
items = []
for site in sites:
item = DmozItem()
item['title'] = site.xpath('a/div[@class="site-title"]/text()').extract()
item['link'] = site.xpath('a/@href').extract()
item['desc'] = site.xpath('div[@class="site-descr "]/text()').extract()
items.append(item)
return items