Python 下载百度文库

Python 下载百度文库

下面展示一些 内联代码片。

import requests
from selenium import webdriver
from lxml import etree
import re
from selenium.webdriver.common.keys import Keys
import time
from PIL import Image
import os
from bs4 import BeautifulSoup
import bs4
from docx import Document
import sys

def getHTMLText(url):
    header = {'User-agent': 'Googlebot'}
    try:
        r = requests.get(url, headers = header, timeout = 30)
        r.raise_for_status()
        r.encoding = 'gbk'
        # r.encoding = r.apparent_encoding
        return r.text
    except:
        return ''

def parse_type(content):
    return re.findall(r"docType.*?\:.*?\'(.*?)\'\,", content)[0]

def parse_txt(html): #解析网页源代码
    plist = []
    soup = BeautifulSoup(html, "html.parser")
    plist.append(soup.title.string)
    for div in soup.find_all('div', attrs={"class": "bd doc-reader"}):
        plist.extend(div.get_text().split('\n'))
    plist = [c.replace(' ', '') for c in plist]
    plist = [c.replace('\x0c', '') for c in plist]
    return plist

def print_docx(plist, filename):  #读取文档
    file = open(filename + '.txt', 'w',encoding='utf-8')
    for str in plist:
        file.write(str)
        file.write('\n')
    file.close()
    with open(filename + '.txt', encoding='utf-8') as f:
        docu = Document()
        docu.add_paragraph(f.read())
        docu.save(filename + '.docx')

def parse_doc(url, folderPath):
    driver = webdriver.Chrome(r'./src/chromedriver.exe')
    driver.get(url)
    # 找到‘继续阅读’按钮  定位至<span class="moreBtn goBtn"><span>还剩35页未读，</span><span class="fc2e">继续阅读</span></span>
    button = driver.find_element_by_xpath("//*[@id='html-reader-go-more']/div[2]/div[1]/span")
    # 按下按钮
    driver.execute_script("arguments[0].click();", button)
    time.sleep(1)
    source = re.compile(r'<span class="page-count">/(.*?)</span>')
    number = int(source.findall(driver.page_source)[0])
    # 获取页码数
    # number = total[1]
    time.sleep(1)
    for i in range(2,number):
        driver.find_element_by_class_name("page-input").clear()
        driver.find_element_by_class_name("page-input").send_keys(f'{i}')
        driver.find_element_by_class_name("page-input").send_keys(Keys.ENTER)
        time.sleep(1)
        html=etree.HTML(driver.page_source)
        # 找到picture容器
        links=html.xpath("//div[@class='reader-pic-item']/@style")
        # 找到图片对应的url
        part = re.compile(r'url[(](.*?)[)]')
        qa="".join(links)
        z=part.findall(qa)
        if i == 2:
            for m in range(3):
                pic = requests.get(z[m]).content
                with open(f'./照片/{m+1}.jpg','wb') as f:
                    f.write(pic)
                    f.close()
        else:
            pic = requests.get(z[2]).content
            with open(f'./照片/{i+1}.jpg','wb') as f:
                f.write(pic)
                f.close()
        time.sleep(1)
    driver.quit()

def parse_other(url, folderPath):
    driver = webdriver.Chrome(r'./src/chromedriver.exe')
    driver.get(url)
    # 找到‘继续阅读’按钮  定位至<span class="moreBtn goBtn"><span>还剩35页未读，</span><span class="fc2e">继续阅读</span></span>
    button = driver.find_element_by_xpath("//*[@id='html-reader-go-more']/div[2]/div[1]/span")
    # 按下按钮
    driver.execute_script("arguments[0].click();", button)
    time.sleep(1)
    source = re.compile(r'<span class="page-count">/(.*?)</span>')
    number = int(source.findall(driver.page_source)[0])
    # 获取页码数
    # number = total[1]
    time.sleep(1)
    # 获取图片
    for i in range(2,number):
        driver.find_element_by_class_name("page-input").clear()
        driver.find_element_by_class_name("page-input").send_keys(f'{i}')
        driver.find_element_by_class_name("page-input").send_keys(Keys.ENTER)
        time.sleep(1)
        html=etree.HTML(driver.page_source)
        # 找到picture容器"//div[@class='reader-pic-item']/@style"
        z=html.xpath('//div[@class="ppt-image-wrap"]/img/@src')
        # print(z)
        # 保存图片
        if i == 2:
            for m in range(3):
                pic = requests.get(z[m]).content
                with open(folderPath + f'/{m + 1}.jpg','wb') as f:
                    f.write(pic)
                    f.close()
        else:
            pic = requests.get(z[i]).content
            with open(folderPath + f'/{i + 1}.jpg','wb') as f:
                f.write(pic)
                f.close()
        time.sleep(1)
    driver.quit()


def print_pdf(folderPath, filename):
    files = os.listdir(folderPath)
    jpgFiles = []
    sources = []
    for file in files:
        if 'jpg' in file:
            jpgFiles.append(file)
    tep = []
    for i in jpgFiles:
        ex = i.split('.')
        tep.append(int(ex[0]))
    tep.sort()
    jpgFiles=[folderPath +'/'+ str(i) + '.jpg' for i in tep]
    output = Image.open(jpgFiles[0])
    jpgFiles.pop(0)
    for file in jpgFiles:
        img = Image.open(file)
        img = img.convert("P")
        sources.append(img)
    output.save(f"{filename}.pdf","PDF",save_all=True,append_images=sources)

from datetime import datetime
def main(url, istxt):
    try:
        ticks = time.time() # 获取时间（用于命名文件夹）
        filepath = './照片' + str(ticks) # 保存爬取的图片
        filename = './爬取结果' + str(ticks) # 爬取生成的文件名

        ticks = datetime.now().strftime('%F')
        filepath = os.getcwd()+'./照片' + str(ticks) # 保存爬取的图片
        filename = os.getcwd()+'./爬取结果' + str(ticks) # 爬取生成的文件名

        if not os.path.exists(filepath): # 新建文件夹
            os.mkdir(filepath)
        html = getHTMLText(url) # requests库爬取
        type = parse_type(html) # 获取文库文件类型：ppt, pdf, docx

        # 当你要爬取文档的文本时，打开下列注释
        if(istxt == "1"):
            type = 'txt'
    
        if type == 'txt' :
            plist = parse_txt(html)
            print_docx(plist, filename)
        elif type == 'doc' or type == 'pdf':
            parse_doc(url, filepath)
            print_pdf(filepath , filename)
        else:
            parse_other(url, filepath)
            print_pdf(filepath, filename)
        print('1')
    except:
        print('0')

if __name__ == '__main__':
    #main(sys.argv[1],sys.argv[2])
    url = 'https://wenku.baidu.com/view/b659fbd0e55c3b3567ec102de2bd960590c6d981.html?fr=search'
    istxt = "1"
    main(url,istxt)