用scrapy爬取网页

使用xpath定位相关视频的标题位置信息

可以得到

但是还不够，需要进一步定位各个元素可得到元素排列如下

Scrapy的源码如下

import scrapy
from scrapy import Request
from scrapy.spiders import Rule

from mySpider import items


class BibiliSpider(scrapy.Spider):
    name = 'bibili'
    allowed_domains = ['bilibili.com']
    start_urls = [
        'https://search.bilibili.com/all?keyword=%E7%89%B9%E6%9C%97%E6%99%AE&from_source=nav_suggest_new%3D2%3D2&page'

    ]
    # 链接处理规则
    def parse(self, response