Datawhale爬虫-Task4（完）

接上：Task1、Task2、Task3

写在前面

总算到大作业了，实现了腾讯新闻的爬虫，但是没实现知乎的（反爬还是强，以后再试试），爬虫还是很实用的，只要把控类人行为这一点，可以为个人省很多事。

实践

腾讯新闻爬虫

import time
import random
from selenium import webdriver
from lxml import etree

driver = webdriver.Chrome()
driver.get('https://news.qq.com/')

# 因为是随机滑动，所以无法控制爬取的条数
for i in range(20):
    height = random.randint(800,1000)
    driver.execute_script(f'window.scrollBy(0,{height})')
    time.sleep(random.random()+0.2) #[0.2,1.2)

html = driver.page_source
tree = etree.HTML(html)
infos = tree.xpath('//ul[@class="list"]/li/div[@class="detail"]/h3/a')

infolist = []
count = 0

for i,info in enumerate(infos):
    title = info.xpath('text()')[0]
    href = info.xpath('@href')[0]
    # print(i+1,title,href)
    infolist.append([i+1,title,href])

# 保存为csv
import pandas as pd
name = ['序号','新闻标题','新闻链接']
# 多于50条只保存前50条
try:
    df = pd.DataFrame(columns=name, data=infolist[:50])
except:
    df = pd.DataFrame(columns=name, data=infolist)

# 注意此处的编码，不设置会出现乱码
df.to_csv('腾讯新闻热点.csv',index=False,encoding='utf_8_sig')
df

部分结果图：

自动登录163邮箱

import time
from selenium import webdriver
from selenium.webdriver.common.by import By
"""
使用selenium进行模拟登陆
1.初始化ChromDriver
2.打开163登陆页面
3.找到用户名的输入框，输入用户名
4.找到密码框，输入密码
5.提交用户信息
"""
# 以下需要修改为自己的账号、密码！
name = '*'
passwd = '*'

driver = webdriver.Chrome()
driver.get('https://mail.163.com/')
# 将窗口调整最大
driver.maximize_window()
# 休息1s
time.sleep(1)
current_window_1 = driver.current_window_handle
print(current_window_1)
button = driver.find_element_by_id('lbNormal')
button.click()
driver.switch_to.frame(driver.find_element_by_xpath("//iframe[starts-with(@id, 'x-URS-iframe')]"))
email = driver.find_element_by_name('email')
#email = driver.find_element_by_xpath('//input[@name="email"]')
email.send_keys(name)
password = driver.find_element_by_name('password')
# password = driver.find_element_by_xpath("//input[@name='password']")
password.send_keys(passwd)
submit = driver.find_element_by_id("dologin")
time.sleep(1)
submit.click()
# time.sleep(10)
# print(driver.page_source)
# driver.quit()

zju grs登陆

以下是活学活用用来登陆浙大研究生教务网，可直接跳往“我的课程”，节省大量点击。

from selenium import webdriver

name = '*'
passwd = '*'
driver = webdriver.Chrome()
driver.get('https://zjuam.zju.edu.cn/cas/oauth2.0/authorize?response_type=code&client_id=yckFoYFlgWxnu7Bn1N&redirect_uri=http%3A%2F%2Fgrs.zju.edu.cn%2Fallogene%2Fpage%2Fhome.htm')
# 将窗口调整最大
driver.maximize_window()
username = driver.find_element_by_name('username')
username.send_keys(name)
password = driver.find_element_by_name('password')
password.send_keys(passwd)
submit = driver.find_element_by_id("dl")
submit.click()
driver.get('http://grs.zju.edu.cn/py/page/student/grkcgl.htm')