python集成代码实现八爪鱼爬取知乎的所有功能+数据预处理

链接: link.
上一篇文章（链接在上面）是对每一部分进行较为详细的说明，这一篇文章将介绍爬取的集成好的代码块，以及对爬取来的数据进行预处理。

1.python集成代码实现八爪鱼爬取知乎的所有功能

```python
#!/usr/bin/env python
# coding: utf-8
import os
import pandas as pd
from selenium import webdriver
from lxml import etree
import time
import jieba
import re
import numpy as np


url1 = input("请输入您所需要爬取的网页（知乎）")

browser = webdriver.Chrome("/Users/apple/Downloads/chromedrivermac")
browser.get(url1)



try:
    #点击问题全部内容
    button1 = browser.find_elements_by_xpath("""//div[@class= "QuestionHeader-detail"]
    //button[contains(@class,"Button") and contains(@class,"QuestionRichText-more") 
    and contains(@class , "Button--plain")
    ]""")[0]
    button1.click()
except:
    print('这个问题比较简单，并没有问题的全部内容哦！')


#此网页就属于异步加载的情况
#那么我们就需要多次下滑
for i in range(20):
    browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
    time.sleep(0.5)
    print(i)

#点击知乎的登陆弹窗
button2 = browser.find_elements_by_xpath("""//button[@aria-label = '关闭']""")[0]
button2.click()    

#点击知乎的“查看全部回答”按钮
button3 = browser.find_elements_by_xpath("""//div[@class = 'Question-main']
//a[contains(@class,"ViewAll-QuestionMainAction")    and contains(@class , "QuestionMainAction")  ]""")[1]
button3.click()


final_end_it = browser.find_elements_by_xpath("""//button[contains(@class,"Button") 
and contains(@class ,'QuestionAnswers-answerButton')
and contains(@class ,'Button--blue')

and contains(@class ,'Button--spread')
]""")
while final_end_it == [