1 使用命令创建savedb 工程 : scrapy startproject savedb
然后到savedb目录上查看目录结构、如下:
--spiders
__init__.py
items.py
__init__.py
pipelines.py
settings.py
2确认过目录结构没有问题后编写自己的spider
# -*- coding:utf-8 -*-
from scrapy.spider import BaseSpider
from scrapy.selector.lxmlsel import HtmlXPathSelector
from scrapy.item import Item, Field
class SavedbItem(Item):
# define the fields for your item here like:
# name = Field()
title =Field()
link =Field()
class bbsSpider(BaseSpider):
name ='boat'
allow_domains=["http://book.douban.com/tag/编程?type=S"]
start_urls = ["http://book.douban.com/tag/编程?type=S"]
def parse(self,response):
hxs = HtmlXPathSelector(response)
items = []
item = SavedbItem()
item['title']=hxs.select('//ul/li[position()>0]/div[2]/h2/a/@title').extract()
item['link']=hxs.select('//ul/li[position()>0]/div[2]/h2/a/@href').extract()
items.append(item)
return items
此处我把item类也放到同一文件中,保存文件到savedb.py。
3编写保存到数据库的pipelines
#-*- coding:utf-8 -*-
import sys
import MySQLdb
import hashlib
from scrapy.exceptions import DropItem
from scrapy.http import Request
class MySqlStorePipeline(object):
def __init__(self):
self.conn = MySQLdb.connect(user='jbpm', passwd ='jbpm', db='test', host='localhost', charset="utf8", use_unicode=True)
self.cursor = self.conn.cursor()
def process_item(self, item, spider):
try:
if item.get('title'):
for i in range(len(item['title'])):
self.cursor.execute("INSERT INTO book (title, link) VALUES (%s, %s)",(item['title'][i], item['link'][i]))
self.conn.commit()
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
return item
保存文件到dbpipelines.py
4修改settings.py文件
加入此配置
ITEM_PIPELINES = ['savedb.dbpipelines.MySqlStorePipeline']
5确认使用的数据库表与【3】中配置一致
6执行scrapy crawl boat就可以实现了
参考文章:http://blog.sina.com.cn/s/blog_6a1e0bf301019yzh.html,由于原文章我没有运行成功,查看其它文章后更改运行成功。故将此记录下来。
以供参考。如有问题请随时指摘。