python爬虫爬取文献数据

技术大白嘉嘉子

已于 2023-05-25 15:00:59 修改

阅读量4.4k

点赞数 6

文章标签： python selenium 爬虫大数据

于 2023-05-25 13:57:34 首次发布

本文链接：https://blog.csdn.net/m0_66526403/article/details/130864126

版权

本文利用python，安装Selenium工具爬取某文献网站数据。

本文偏向实践，其原理可能了解的不深，解释的并不是很好，望见谅。

如果我们在生活中有批量获取文献数据的需求的话，在这里可以提供两种方法：

一、有些网站本身自带的批量导出（以某大型文献网站为例）

1.批量选择需要信息的文章（最多可选择500篇）

点击文章上方的导出与分析--导出文献--自定义

2.勾选需要的信息后，可选择word或excel导出。

3.结果展示

二、利用python爬虫爬取

1.安装Selenium工具

在python中使用pip install Selenium命令安装

（Selenium工具我理解是模拟人进行滑动、点击操作等，实现电脑自动化操作，可支持多个浏览器）

2.下载浏览器驱动（浏览器驱动可以理解为自动化脚本与浏览器之间的媒介）

常见浏览器驱动下载：（推荐使用谷歌，因为兼容性更好）


#Firefox浏览器驱动：
https://link.zhihu.com/?target=https%3A//github.com/mozilla/geckodriver/releases

#Chrome浏览器驱动：
https://registry.npmmirror.com/binary.html?path=chromedriver/

#IE浏览器驱动：IEDriverServer
https://link.zhihu.com/?target=http%3A//selenium-release.storage.googleapis.com/index.html

#Edge浏览器驱动：MicrosoftWebDriver
https://link.zhihu.com/?target=https%3A//developer.microsoft.com/en-us/microsoft-edge/tools/webdriver

3.进行浏览器的初始化

from selenium import webdriver

browser = webdriver.Chrome()
browser = webdriver.Firefox()
browser = webdriver.Edge()
browser = webdriver.Safari()

4.进行网页爬取

注：引号内（需要更改的，即自己需要的）为xpath，xpath获取路径在代码下方

代码内部附注解

import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from urllib.parse import urljoin


def open_page(driver, theme):
    
# 打开知网页面
    driver.get("https://www.cnki.net")
    
#点击搜索类型
    WebDriverWait(driver, 100).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div[2]/div[2]/div/div[1]/div/div[1]"))).click()
    
#选择搜索类型
    WebDriverWait(driver, 100).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div[2]/div[2]/div/div[1]/div/div[2]/ul/li[9]"))).click()

# 传入关键字，关键字写在theme里，在下方输入
    WebDriverWait(driver, 100).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div[2]/div[2]/div/div[1]/input[1]"))).send_keys(theme)

# 点击搜索
    WebDriverWait(driver, 100).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div[2]/div[2]/div/div[1]/input[2]"))).click()
    time.sleep(3)

    # 点击切换中文文献
#     WebDriverWait(driver, 100).until(
#         EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[1]/div/div/div/a[1]"))).click()
#     time.sleep(3)

    # 获取总文献数和页数
    res_unm = WebDriverWait(driver, 100).until(EC.presence_of_element_located(
        (By.XPATH, "/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[1]/span[1]/em"))).text

    # 去除千分位里的逗号
    res_unm = int(res_unm.replace(",", ''))
    page_unm = int(res_unm / 20) + 1
    print(f"共找到 {res_unm} 条结果, {page_unm} 页。")
    return res_unm


def crawl(driver, papers_need, theme):
    # 赋值序号, 控制爬取的文章数量
    count = 1
    
    # 当爬取数量小于需求时，循环网页页码
    while count <= papers_need:
        # 等待加载完全，休眠3S
        time.sleep(3)

        title_list = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "fz14")))
        # 循环网页一页中的条目
        for i in range(len(title_list)):
            try:
                if (count % 20) != 0:
                    term = count % 20  # 本页的第几个条目
                else:
                    term = 20
                title_xpath = f"/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[{term}]/td[2]"#获取文章题目
                author_xpath = f"/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[{term}]/td[3]"#获取文章作者
                source_xpath = f"/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[{term}]/td[4]"#获取文章来源
                date_xpath = f"/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[{term}]/td[5]"#获取文章时间
                database_xpath = f"/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[{term}]/td[6]"
                title = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, title_xpath))).text
                authors = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, author_xpath))).text
                source = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, source_xpath))).text
                date = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, date_xpath))).text
                database = WebDriverWait(driver, 10).until(
                    EC.presence_of_element_located((By.XPATH, database_xpath))).text

                
#获取点开文章之后的摘要等信息             
                
                # 点击条目
                title_list[i].click()

                # 获取driver的句柄
                n = driver.window_handles

                # driver切换至最新生产的页面
                driver.switch_to.window(n[-1])
                time.sleep(3)

                # 开始获取页面信息
                title = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH ,"/html/body/div[2]/div[1]/div[3]/div/div/div[3]/div/h1"))).text
                authors = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH ,"/html/body/div[2]/div[1]/div[3]/div/div/div[3]/div/h3[1]"))).text
                institute = WebDriverWait(driver, 10).until(EC.presence_of_element_located(
                    (By.XPATH, "/html/body/div[2]/div[1]/div[3]/div/div/div[3]/div/h3[2]"))).text
                abstract = WebDriverWait(driver, 10).until(
                    EC.presence_of_element_located((By.CLASS_NAME, "abstract-text"))).text
                try:
                    keywords = WebDriverWait(driver, 10).until(
                        EC.presence_of_element_located((By.CLASS_NAME, "keywords"))).text[:-1]
                    cssci = WebDriverWait(driver, 10).until(
                        EC.presence_of_element_located((By.XPATH, "/html/body/div[2]/div[1]/div[3]/div/div/div[1]/div[1]/a[2]"))).text
                except:
                    keywords = '无'
                    cssci = 'NULL'
                url = driver.current_url

                res = f"{count}\t{title}\t{authors}\t{cssci}\t{institute}\t{date}\t{source}\t{database}\t{keywords}\t{abstract}\t{url}".replace(
                        "\n", "") + "\n"
                print(res)
                
                #写入文件
                import os
                #文件保存在python文件默认路径中
                with open(f'CNKI_{theme}.xls', 'a', encoding='gbk') as f:
                    f.write(res)

            except:
                print(f" 第{count} 条爬取失败\n")
                # 跳过本条，接着下一个
                continue
            finally:
                # 如果有多个窗口，关闭第二个窗口， 切换回主页
                n2 = driver.window_handles
                if len(n2) > 1:
                    driver.close()
                    driver.switch_to.window(n2[0])
                # 爬完一篇计数加 1
                count += 1

                if count > papers_need:
                    break
                    
        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//a[@id='PageNext']"))).click()


if __name__ == "__main__":
    print("开始爬取！")

    # get直接返回，不再等待界面加载完成
    desired_capabilities = DesiredCapabilities.CHROME
    desired_capabilities["pageLoadStrategy"] = "none"

    # 设置驱动器的环境
    options = webdriver.EdgeOptions()

    # 设置chrome不加载图片，提高速度
    options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})

    # 设置不显示窗口
    options.add_argument('--headless')

    # 创建一个浏览器驱动器
    driver = webdriver.Edge(options=options)

    # 输入需要搜索的关键词
    # theme = input("请输入你要搜索的关键词：")
    theme = "医学"

    # 设置所需篇数
    # papers_need = int(input("请输入你要爬取篇数："))
    papers_need = 2

    res_unm = int(open_page(driver, theme))

    # 判断所需是否大于总篇数
    papers_need = papers_need if (papers_need <= res_unm) else res_unm
    crawl(driver, papers_need, theme)

    print("爬取完毕！")

    # 关闭浏览器
    driver.close()