易班已发推文数据爬取

"""

本代码用于爬取易班已发表数据
(由于写这篇代码的时候就一页，就没设置翻页）
用到的库：selenium、csv、pandas、bs4、time、re
最终成果：形成易班推文数据csv格式
作者：UPC.故里
注：有成功概率，因为易班有时候有登录验证有时候没有

"""


import time
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from bs4 import BeautifulSoup

"""
The By implementation.
"""
class By(object):
    """
    Set of supported locator strategies.
    """
    ID = "id"
    XPATH = "xpath"
    LINK_TEXT = "link text"
    PARTIAL_LINK_TEXT = "partial link text"
    NAME = "name"
    TAG_NAME = "tag name"
    CLASS_NAME = "class name"
    CSS_SELECTOR = "css selector"

#模拟进入易班
url= 'https://www.yiban.cn/Org/orglistShow/type/forum/puid/5370538#'  #初始网站页面
driver = webdriver.Firefox()
driver.get(url)
time.sleep(1)

#模拟输入账号密码  账号密码用自己的或者自己学院组织的！
stu_number = ''
stu_password = ''
'''
根据xpath查看账号和密码的id
stu_number_xpath = '//*[@id="account-txt"]'  #填写账号的位置的xpath --> id是un
stu_password_xpath = '//*[@id="password-txt"]'  #填写密码的位置的xpath --> id是pd
'''
driver.find_element_by_id('account-txt').send_keys(stu_number)  #模拟输入账号
time.sleep(1)
driver.find_element_by_id('password-txt').send_keys(stu_password)  #模拟输入密码
time.sleep(1)
sign_in_xpath = '//*[@id="login-btn"]'
ActionChains(driver).click(driver.find_element_by_xpath(sign_in_xpath)).perform()  #模拟鼠标点击确定
time.sleep(5)

#转到我发布的
ActionChains(driver).click(driver.find_element_by_xpath('/html/body/main/div/div[2]/div[2]/ul/li[3]/a')).perform()  #模拟鼠标点击确定我发布的
time.sleep(2)

a_text_list = []
span_text_list = []
for j in range(6): #翻几页就是数字级
    xTable = '/html/body/main/div/div[2]/div[3]'
    table = driver.find_element_by_xpath(xTable).get_attribute('innerHTML')
    soup = BeautifulSoup(table,'html.parser') #解析表单
    a = soup.find_all('a') #查找表内a标签
    for row in a:
        row_a_text = row.text
        a_text_list.extend([row_a_text])
    span = soup.find_all('span') #查找表内span标签
    for row in span:
        row_a_text = row.text
        span_text_list.extend([row_a_text])
    time.sleep(2)
    next_page = '/html/body/main/div/div[2]/div[4]/div/div/a[2]'
    element = driver.find_element_by_xpath(next_page)
    driver.execute_script("arguments[0].click();", element)
    time.sleep(2)

data = {'活动院系':span_text_list[::6],
        '推文标题':a_text_list[1::9],
        '推送板块':span_text_list[1::6],
        '推送时间':span_text_list[2::6],
        '阅读量':span_text_list[3::6],
        '点赞量':span_text_list[4::6],
        '评论量':span_text_list[5::6]}
dataframe = pd.DataFrame(data)
dataframe.to_csv('Yiban.csv',index=False, sep=',',encoding='utf-8-sig')

driver.close()