- scrapy startproject 项目名称
- scrapy genspider -t crawl spiderName www.xxx.com
- 生成的文件
-
import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule class SpiderhouxueSpider(CrawlSpider): name = 'spiderHouxue' allowed_domains = ['www.houxue.com'] start_urls = ['http://www.houxue.com/'] rules = ( #修改正则匹配规则 Rule(LinkExtractor(allow=r'peixun86.*'), callback='parse_item', follow=True), ) def parse_item(self, response): item = {} #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get() #item['name'] = response.xpath('//div[@id="name"]').get() #item['description'] = response.xpath('//div[@id="description"]').get() return item
修改管道文件 pipelines.py
-
class HouxuePipeline: def __init__(self): self.fp = None def open_spider(self, spider): print('开始爬虫') self.fp = open('houxuedata.txt', 'a+', encoding='utf-8') def process_item(self, item, spider): # 将爬虫文件提交的item写入文件进行持久化存储{ # "url": response.url, # "title": title, # "keywords": keywords, # "description": description, # } self.fp.write(str(item) + '\n') print('------------') return item def close_spider(self, spider): print('结束爬虫') self.fp.close()
修改配置文件 settings.py
20行 修改为false
ROBOTSTXT_OBEY = False
66行-68行放开注释
ITEM_PIPELINES = { 'houxue.pipelines.HouxuePipeline': 300, }
启动程序
scrapy crawl spiderName #有日志
scrapy crawl spiderName --nolog #无日志