OpenAccessLibrary 网站简单爬取

目标

目标网站：Open Access Library
地址：https://www.oalib.com/
目标爬取内容
https://www.oalib.com/journal/3174/1 岩石力学与工程学报内容

代码

import requests
import time
from scrapy import Selector

class OalibSpider:
    """
    1、构造分页的url
    https://www.oalib.com/journal/3174/1
    https://www.oalib.com/journal/3174/2

    2、抓取每一个分页中论文的url
    3、抓取每一篇论文，解析论文内容
    """
    def __init__(self):
        self.page_urls = ['https://www.oalib.com/journal/3174/{}'.format(i) for i in range(1,5)]
    def parse_page(self, page_url):
        """
        解析每一个分页，获取当前分页中论文的url
        :param page_url:
        :return:
        """
        resp = requests.get(page_url)
        root = Selector(