基于selenium的小说爬取

以  https://www.beqege.cc 为目标

爬取的是 小说《死人经》

from selenium.webdriver import Chrome#实例化对象
from selenium.webdriver import ChromeOptions#实例化对象
from selenium.webdriver.common.by import By#定位元素
from lxml import etree
import time


options = ChromeOptions()
options.add_experimental_option('detach', True) #避免因程序运行完或报错导致浏览器自动关闭
driver = Chrome(options=options)
driver.get(r'https://www.beqege.cc/32073/')


def get_text(link):
    subdriver = Chrome()
    js = f'window.open("{link}")'
    subdriver.execute_script(js)
    handles = subdriver.window_handles
    subdriver.switch_to.window(handles[-1])
    time.sleep(1)
    tree = etree.HTML(subdriver.page_source)
    ps = tree.xpath('//*[@id="content"]/p')
    for p in ps:
        try:
            text = p.xpath('.//text()')[0]
            time.sleep(0.5)
            with open(title, 'a', encoding='utf8') as fp:
                fp.write(text)
                fp.write("\n")
        except:
            pass
    subdriver.close()


tree=etree.HTML(driver.page_source)
dds=tree.xpath('//*[@id="list"]//dd')
for dd in dds:
    text=dd.xpath('.//text()')[0]
    href=dd.xpath('./a/@href')[0]
    title=f'./小说/死人经/{text}.txt'
    link=f'https://www.beqege.cc{href}'
    get_text(link)

driver.quit()