石大第二课堂可参与活动爬取

"""

本代码用于爬取石光可参与活动列表
(由于写这篇代码的时候就一页，就没设置翻页）
用到的库：selenium、csv、pandas、bs4、time、re
最终成果：形成可参与活动列表csv格式
作者：UPC.故里
注：此代码运行时需要连接UPC校园网或VPN，使用前需要装好响应的库以及下载相应的文件

"""
import re
import time
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from bs4 import BeautifulSoup

#进入石光代码
url= 'http://dekt.upc.edu.cn'  #初始网站页面
time.sleep(1)
driver = webdriver.Firefox()
driver.get(url)

#账号密码用自己的！
stu_number = ''
stu_password = ''

'''
根据xpath查看账号和密码的id
stu_number_xpath = '//*[@id="un"]'  #填写学号的位置的xpath --> id是un
stu_password_xpath = '//*[@id="pd"]'  #填写密码的位置的xpath --> id是pd
'''
driver.find_element_by_id('un').send_keys(stu_number)  #模拟输入学号
time.sleep(1)
driver.find_element_by_id('pd').send_keys(stu_password)  #模拟输入密码
time.sleep(1)
sign_in_xpath = '/html/body/form/div[3]/div[2]/div/div[3]/span/input'
ActionChains(driver).click(driver.find_element_by_xpath(sign_in_xpath)).perform()  #模拟鼠标点击确定
time.sleep(2)

#找到可参与活动页面代码
act_xpath = '/html/body/div[1]/div[2]/div[1]/ul/li[2]/div/a'  #校园活动的xpath
ActionChains(driver).click(driver.find_element_by_xpath(act_xpath)).perform()  #模拟鼠标点击
time.sleep(2)
can_act_xpath = '/html/body/div[2]/div[2]/div/div[2]/a[1]'  #可参与活动的xpath
ActionChains(driver).click(driver.find_element_by_xpath(can_act_xpath)).perform()  #模拟鼠标点击
time.sleep(2)

xTable = '/html/body/div[2]/div[2]/div/div[3]/div[1]' #数据表
table = driver.find_element_by_xpath(xTable).get_attribute('innerHTML')
soup = BeautifulSoup(table,'html.parser') #解析表单

#活动名称
a = soup.find_all('a') #查找表内a标签
a_text_list_bg = []
for row in a:
    row_a_text = row.text
    a_text_list_bg.extend([row_a_text])
a_text_list = a_text_list_bg[1::3]

#活动具体
p_text_list_bg = []
p = soup.find_all('p') #查找表内p标签
for row in p:
    row_p_text = row.text
    p_text_list_bg .append([row_p_text])
p_text_list = []
for i in p_text_list_bg:
    p_text_list.append([''.join(i)[re.search('：',''.join(i)).start()+1:]])

data = {'活动名称':a_text_list,'报名时间':p_text_list[0::4], '活动时间':p_text_list[1::4], '分类':p_text_list[2::4], '地点':p_text_list[3::4]}
dataframe = pd.DataFrame(data)
dataframe.to_csv('dekt.csv',index=False, sep=',',encoding='utf-8-sig')

driver.close()

最终效果：