目标
目标网站:Open Access Library
地址:https://www.oalib.com/
目标爬取内容
https://www.oalib.com/journal/3174/1 岩石力学与工程学报内容
代码
import requests
import time
from scrapy import Selector
class OalibSpider:
"""
1、构造分页的url
https://www.oalib.com/journal/3174/1
https://www.oalib.com/journal/3174/2
2、抓取每一个分页中论文的url
3、抓取每一篇论文,解析论文内容
"""
def __init__(self):
self.page_urls = ['https://www.oalib.com/journal/3174/{}'.format(i) for i in range(1,5)]
def parse_page(self, page_url):
"""
解析每一个分页,获取当前分页中论文的url
:param page_url:
:return:
"""
resp = requests.get(page_url)
root = Selector(