安装pip和scrapy教程
环境:win10+python2.7+wamp(这个提供mysql环境)+navicat for mysql(这个管理mysql)
scrapy第一个例子(爬取糗百段子)
1.安装pymysql:pip install PyMySQL
2.用navicat for mysql或者相同软件新建数据库“news”和表“qiubai”
CREATE TABLE `qiubai` (
`name` varchar(255) DEFAULT NULL,
`news_id` varchar(20) DEFAULT '',
`url` varchar(255) DEFAULT NULL,
`text_content` varchar(255) DEFAULT NULL,
`has_image` varchar(1) DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
3.创建爬虫(在桌面cmd执行):scrapy startproject tutorial
4.桌面会出现tutorial文件夹
tutorial/
scrapy.cfg
tutorial/
__init__.py
items.py
pipelines.py
settings.py
spiders/
__init__.py
...
这些文件分别是:
scrapy.cfg: 项目的配置文件
tutorial/: 该项目的python模块。
tutorial/items.py: 项目中的item文件.
tutorial/pipelines.py: 项目中的pipelines文件.
tutorial/settings.py: 项目的设置文件.
tutorial/spiders/: 放置spider代码的目录.
5.定义爬取的item:
编辑 tutorial 目录中的 items.py 文件:
#items.py
import scrapy
class TutorialItem(scrapy.Item):
# define the fields for your item here like:
name = scrapy.Field()
news_id=scrapy.Field()
url=scrapy.Field()
#title = scrapy.Field()
text_content = scrapy.Field()
#key_words = scrapy.Field()
has_image=scrapy.Field()
6.添加UserAgent
编辑 tutorial 目录中的 middlewares.py 文件(在文件末尾增加如下代码):
import random
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
class RotateUserAgentMiddleware(UserAgentMiddleware):
def __init__(self, user_agent=''):
self.user_agent = user_agent
def process_request(self, request, spider):
ua = random.choice(self.user_agent_list)
if ua:
print ua, '-----------------'
request.headers.setdefault('User-Agent', ua)
# the default user_agent_list composes chrome,I E,firefox,Mozilla,opera,netscape
# for more user agent strings,you can find it in http://www.useragentstring.com/pages/useragentstring.php
user_agent_list = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]
7.将item插入数据库
编辑 tutorial 目录中的 pipelines.py 文件(在文件末尾增加如下代码):
import pymysql
def dbHandle():
conn = pymysql.connect(
host='localhost',
port=3306,
db='news',
user='root',
passwd='',
charset='utf8',
)
return conn
class TutorialPipeline(object):
def __init__(self):
self.dbObject = dbHandle()
self.cursor = self.dbObject.cursor()
def process_item(self, item, spider):
sql = 'insert into qiubai(name,news_id,url,text_content,has_image) values (%s,%s,%s,%s,%s)'
try:
self.cursor.execute(sql, (item['name'].encode("utf-8"), (item['news_id'].encode("utf-8"))[7:15], ("http://www.qiushibaike.com"+item['url'].encode("utf-8")), item['text_content'].encode("utf-8"), item['has_image'].encode("utf-8")))
self.dbObject.commit()
except Exception, e:
print e
self.dbObject.rollback()
return item
def __del__(self):
self.cursor.close()
self.dbObject.close()
8.注册
编辑 tutorial 目录中的settings.py 文件(在文件末尾增加如下代码):
ITEM_PIPELINES = {
'tutorial.pipelines.TutorialPipeline': 300,
}
DOWNLOADER_MIDDLEWARES = {
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':None,
'tutorial.middlewares.RotateUserAgentMiddleware':400,
}
FEED_EXPORT_ENCODING = 'utf-8'
9.建立爬虫
在tutorial/spider 目录中新建myspider.py 文件并添加如下代码:
# encoding=utf8
import scrapy
from tutorial.items import TutorialItem
class DmozSpider(scrapy.Spider):
name = "qiubai"
start_urls = [
"http://www.qiushibaike.com/"
]
def parse(self, response):
qiubai = TutorialItem()
for item in response.xpath('//div[@id="content-left"]/div[@class="article block untagged mb15"]'):
name = item.xpath('./div[@class="author clearfix"]/a[2]/h2/text()').extract()
if name:
qiubai['name'] = name[0]
news_id= item.xpath('./div[@class="author clearfix"]/a[1]/@href').extract()
if news_id:
qiubai['news_id']=news_id[0]
url = item.xpath('./a[@class="contentHerf"]/@href').extract()
if url:
qiubai['url'] = url[0]
text_content = item.xpath('./a[@class="contentHerf"]/div/span/text()').extract()
if text_content:
qiubai['text_content'] = text_content[0]
has_image = item.xpath('./div[@class="thumb"]/a/img[@src]').extract()
if has_image:
qiubai['has_image'] = '1'
else:
qiubai['has_image'] = '0'
yield qiubai
到此爬虫就写完了。关于运行有两种方式:
方式一:在tutorial用cmd执行scrapy crawl qiubai
或者 scrapy crawl dmoz -o items.json
(前者只执行,后者吧数据保存为items.json)
(qiubai为第9步中name,是唯一区别爬虫的,请勿重复)
方式二:对于用pycharm的朋友可以在tutorial新建文件runspider.py并填入下面代码:
# encoding=utf8
from scrapy import cmdline
cmdline.execute("scrapy crawl qiubai".split())
就可以直接用pycharm跑爬虫
一些说明:添加UserAgent 是防止网站有反爬,最好再加上header;
items.py文件中的name = scrapy.Field()
name对应你需要爬的每一个项目名;
settings.py文件是用于注册自己写的一些class;
myspider.py文件中的可以用start_urls
, 也可以重写def start_requests(self)
(二选一);
response.xpath()
给几个简单说明:
/html/head/title: 选择HTML文档<head>元素下面的<title> 标签。
/html/head/title/text(): 选择前面提到的<title> 元素下面的文本内容
//td: 选择所有 <td> 元素
//div[@class="mine"]: 选择所有包含 class="mine"属性的div 标签元素
更多请点击这里。
推荐是用火狐浏览器的插件firebug,方便看网页标签
PS:以上代码部分来源网络,感谢分享
增加部分:
上面代码只能爬去第一页,现在增加爬取指定页数的代码:
1.将myspider.py改为下面代码:
# encoding=utf8
import scrapy
from scrapy.http.request import Request
from tutorial.items import TutorialItem
class DmozSpider(scrapy.Spider):
allowed_domains = ["qiushibaike.com"]
name = "qiubai"
start_urls = [
"http://www.qiushibaike.com/"
]
def start_requests(self):
for url in self.start_urls:
yield Request(url,callback=self.parse)
def parse(self, response):
qiubai = TutorialItem()
for item in response.xpath('//div[@id="content-left"]/div[@class="article block untagged mb15"]'):
name = item.xpath('./div[@class="author clearfix"]/a[2]/h2/text()').extract()
if name:
qiubai['name'] = name[0]
news_id= item.xpath('./div[@class="author clearfix"]/a[1]/@href').extract()
if news_id:
qiubai['news_id']=news_id[0]
url = item.xpath('./a[@class="contentHerf"]/@href').extract()
if url:
qiubai['url'] = url[0]
text_content = item.xpath('./a[@class="contentHerf"]/div/span/text()').extract()
if text_content:
qiubai['text_content'] = text_content[0]
has_image = item.xpath('./div[@class="thumb"]/a/img[@src]').extract()
if has_image:
qiubai['has_image'] = '1'
else:
qiubai['has_image'] = '0'
yield qiubai
for x in range(2,10):
page="http://www.qiushibaike.com/8hr/page/"+str(x)
yield Request(page, callback=self.sub_parse)
def sub_parse(self, response):
qiubai = TutorialItem()
for item in response.xpath('//div[@id="content-left"]/div[@class="article block untagged mb15"]'):
name = item.xpath('./div[@class="author clearfix"]/a[2]/h2/text()').extract()
if name:
qiubai['name'] = name[0]
news_id = item.xpath('./div[@class="author clearfix"]/a[1]/@href').extract()
if news_id:
qiubai['news_id'] = news_id[0]
url = item.xpath('./a[@class="contentHerf"]/@href').extract()
if url:
qiubai['url'] = url[0]
text_content = item.xpath('./a[@class="contentHerf"]/div/span/text()').extract()
if text_content:
qiubai['text_content'] = text_content[0]
has_image = item.xpath('./div[@class="thumb"]/a/img[@src]').extract()
if has_image:
qiubai['has_image'] = '1'
else:
qiubai['has_image'] = '0'
yield qiubai
2.在settings.py增加如下代码:
DOWNLOAD_DELAY = 1#爬取间隔时间