创建crawl—scrapy

创建项目文件夹

scrapy startproject myproject

进入 myproject

scrapy genspider -t crawl  crawl_baidu baidu.com

创建成功后在mproject文件夹中创建 run_baidu.py 运行文件

在spider文件夹中找到 crawl_baidu.py 进行编写代码

例：以拉勾网为例

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from myproject.items import LagouItem


class LagouSpider(CrawlSpider):
    name = 'lagou'
    allowed_domains = ['lagou.com']
    start_urls = ['http://lagou.com/']
   #此处写所需要的url的正则，它会匹配所有符合需求的url，callbacl是将结果返回给parse_item函数，follow为True时继续在此网页中寻找符合的url，False则终止寻找
    rules = (
        # Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
        Rule(LinkExtractor(allow=r'https://www\.lagou\.com/zhaopin/\w+/'), follow=True),
        Rule(LinkExtractor(allow=r'https