python爬虫实战：selenium下载百度文库文档

利用selenium进行页面操作

草草地写了一下，文字是爬下来了，但是还有格式等问题，遇到复杂的文档可能不通用，有点蛋疼。
用安卓模拟登陆，让页面简单一点，电脑网页有点复杂。
之后有空再来改进

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
from selenium import webdriver

options = webdriver.ChromeOptions()
options.add_argument('user-agent="Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19"')
driver = webdriver.Chrome(chrome_options = options)
driver.get('https://wenku.baidu.com/view/aa31a84bcf84b9d528ea7a2c.html')
gap = driver.find_element_by_xpath("//div[@class='foldpagewg-text-con']")
driver.execute_script("arguments[0].scrollIntoView();", gap)
go_on = driver.find_element_by_xpath("//div[@class='foldpagewg-text']")
go_on.click()
# gap2 = driver.find_element_by_xpath()

html = driver.page_source
bf1 = BeautifulSoup(html, 'lxml')
result = bf1.find_all("div", class_="content singlePage wk-container")

for para in result:
    plist = para.find_all("p", class_="txt")
    for item in plist:
        strings = item.stripped_strings
        for string in strings:
            print(string)

在webdriver里设置ip方便很多，但是貌似打不开。。。不清楚为什么

from selenium import webdriver

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('user-agent="Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) '
                            'AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19"')
chrome_options.add_argument("--proxy-server=http://183.62.22.220:3128")
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get("https://wenku.baidu.com/view/aa31a84bcf84b9d528ea7a2c.html")