1.安装好scrapy框架
2.运行cmd 建立项目文件 scrapy startproject qsauto(qsauto是项目名)
3.输入scrapy genspider -t crawl weisuen qiushibaike.com (crawl是自动爬虫类型,weisuen是爬虫名,qiushibaike.com是域名)
4.把setting.py中lines 22 代码前得注释去掉,把True改为False
ROBOTSTXT_OBEY = False
5.把setting.py中lines 12,14,15代码的前的注释都去掉、
BOT_NAME = 'qsauto'
SPIDER_MODULES = ['qsauto.spiders']
NEWSPIDER_MODULE = 'qsauto.spiders'
6.在setting.py中将这行lines 19 的代码的注释去掉,并加上User-Agent信息,模拟人工浏览器登陆,为后续爬虫持续运行做基础
USER_AGENT="Mozilla/5.0 (Windows NT 6.1;WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"
其他完整代码如下:
items.py(容器)
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class QsautoItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
content=scrapy.Field()
link=scrapy.Field()
weisuen.py(爬虫文件)
# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.http import Request
from qsauto.items import QsautoItem
class WeisuenSpider(CrawlSpider):
name = 'weisuen'
allowed_domains = ['qiushibaike.com']
'''
start_urls = ['http://qiushibaike.com/']
'''
rules = (
Rule(LinkExtractor(allow=r'article'), callback='parse_item', follow=True),
) #callback指定回调函数 follow 是否继续跟进 allow爬取规格 根据网页规格来爬
def start_requests(self):
header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
yield Request("http://qiushibaike.com/",headers=header)
def parse_item(self, response): #回转方法
i=QsautoItem()
i["content"] = response.xpath('//div[@class="content"]/text()').extract()
i["link"] = response.xpath('//link[@rel="canonicalz"]/@href').extract()
#item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
#item['name'] = response.xpath('//div[@id="name"]').get()
#item['description'] = response.xpath('//div[@id="description"]').get()
print(i["content"])
print(i["link"])
return i
pipelines.py(信息处理)
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
class QsautoPipeline(object):
def process_item(self, item, spider):
with open('qiushibaike.txt','w') as f:
f.write(i["content"])
return item
最后,使用scrapy crawl weisuen 运行爬虫 就完成了