2023最新 - 谷歌学术文献Bibtex批量获取脚本

首先，自行解决网络访问问题，保证能访问到谷歌学术，否则下面可免看

第一步：安装 selenium

python 安装 selenium

pip install selenium

第二步：安装 Chrome 浏览器

http://chorm.com.cn/

第三步：根据 Chrome 版本，下载 Chrome Driver

在设置 - 帮助 - 关于 GoogleChrome 查看版本

然后在这里找对应版本的driver（版本一定要对应）

https://registry.npmmirror.com/binary.html?path=chromedriver/

第四步：整理需要引用的文献

将需要获取的文献题目添加到一个新建的txt文本中：（一行一篇），我新建的文件名字是：pn.txt

第五步：执行如下脚本

注意在if __name__ == "__main__":下修改成你自己的路径 driver_path，input_file_path，output_file_path

driver_path：改成你下载的chrome driver的位置（最好别有中文）
input_file_path：输入文件路径，默认是pn.txt
output_file_path：输出文件路径，默认是output.txt

脚本代码自取：

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from urllib import parse
import math
from time import sleep

class GG_Bibtex(object):
    def __init__(self, driver_path, gg_search_url):
        self.driver = None
        self.paper_names = []
        self.gg_search_url = gg_search_url
        self.driver_path = driver_path
        self.reset(driver_path)

    def reset(self, driver_path):
        self.service = Service(driver_path) 
        option = webdriver.ChromeOptions()
        # option.add_argument('headless')  # no show window
        self.driver = webdriver.Chrome(service=self.service, options=option)
        self.driver.set_window_size(800,800)
        
    def get_bib_text(self, paper_title):
        elements_xpath = {
            'qoute_btn':'/html/body/div/div[10]/div[2]/div[3]/div[2]/div/div/div[3]/a[2]/span',
            'bibtex_btn':'/html/body/div/div[4]/div/div[2]/div/div[2]/a[1]',
            'bib_text':'/html/body/pre'
        }
        strto_pn=parse.quote(paper_title)
        url = self.gg_search_url + strto_pn
        self.driver.get(url)
        qoute_btn = WebDriverWait(self.driver, 15, 0.1).until(
                            EC.presence_of_element_located((By.XPATH, elements_xpath['qoute_btn']))
                            )
        qoute_btn.click()

        bibtex_btn = WebDriverWait(self.driver, 15, 0.1).until(
                            EC.presence_of_element_located((By.XPATH, elements_xpath['bibtex_btn']))
                            )
        bibtex_btn.click()

        bib_text = WebDriverWait(self.driver, 15, 0.1).until(
                            EC.presence_of_element_located((By.XPATH, elements_xpath['bib_text']))
                            )
        bib_text = bib_text.text
        return bib_text
    
    def _quit_driver(self, ):
        self.driver.quit()
        self.service.stop()
    
    def results_writter(self, results, output_file_path = 'output.txt'):
        wtf = []
        for re_key in results.keys():
            context = results[re_key]
            # wtf.append(re_key + '\n')
            wtf.append(context + '\n\n')
        with open(output_file_path, 'w') as f:
            f.writelines(wtf)

    def run(self, paper_names, output_file_path, reset_len = 10):
        """
        @params:
            paper_names: [LIST], your paper names.
            reset_len: [INT], for avoid the robot checking, you need to reset the driver for more times, default is 10 papers
        """
        self.paper_names = paper_names
        paper_len = len(paper_names)
        rest = paper_len % reset_len
        task_packs = []
        if paper_len > reset_len:
            groups_len = int(math.floor(paper_len / reset_len)) # 21/20 = 1
            for i in range(groups_len):
                sub_names = paper_names[(i)*reset_len:(i+1)*reset_len]
                task_packs.append(sub_names)
        
        task_packs.append(paper_names[-1*rest:])
        results = {}
        for ti in task_packs:
            for pn in ti:
                if len(pn) < 3:
                    continue
                print('\n---> Searching paper: {} ---> \n'.format(pn))
                bibtex = self.get_bib_text(pn)
                print(bibtex)
                results[pn] = bibtex
            self._quit_driver()
            sleep(1)
            self.reset(self.driver_path)
            print('-'*10+'\n Reset for avoiding robot check')
        self.results_writter(results, output_file_path)
        return results
        
            

if __name__ == "__main__":
    driver_path = r"C:/Users/jt/Downloads/chromedriver_win32/chromedriver.exe"
    input_file_path = 'pn.txt'
    output_file_path = 'output.txt'


    gg_search_url = r'https://scholar.google.com.hk/scholar?hl=zh-CN&as_sdt=0%2C5&q='
    ggb = GG_Bibtex(driver_path = driver_path, gg_search_url = gg_search_url)
    paper_names = []
    with open(input_file_path, 'r') as f:
        paper_names = f.readlines()
        paper_names = [pn.replace('\n', '') for pn in paper_names]
    results = ggb.run(paper_names = paper_names, output_file_path = output_file_path)

输出结果文件：

修改一下后缀名，就能直接作为bib文件使用了。