为什么要获取cookie?
因为有的页面爬取的时候,需要登录后才能爬,比如知乎,如何判断一个页面是否已经登录,通过判断是否含有cookies就可以,我们获取到cookie后就可以携带cookie来访问需要登录后的页面了。
方式一使用session
这里的session并不是django中的session,而是requests中的session
import requests
url = 'https://www.processon.com/login'
login_email = '283867@qq.com'
login_password = 'ZZZ0'
创建一个session,作用会自动保存cookie
session = requests.session()
data = {
'login_email': login_email,
'login_password': login_password
}
使用session发起post请求来获取登录后的cookie,cookie已经存在session中
response = session.post(url = url,data=data)
用session给个人主页发送请求,因为session中已经有cookie了
index_url = 'https://www.processon.com/diagrams'
index_page = session.get(url=index_url).text
print(index_page)
把cookie保存在本地,并判断用户是否已经登录
import requests
from http import cookiejar
创建一个session,作用会自动保存cookie
session = requests.session()
指定cookie保存的路径
session.cookies = cookiejar.LWPCookieJar(filename="cookies.txt")
try:
session.cookies.load(ignore_discard=True) # 加载cookie文件,ignore_discard = True,即使cookie被抛弃,也要保存下来
except:
print('cookie未能加载')
def login_save_cookie():
"""
登录并保存cookie到本地
:return:
"""
url = 'https://www.processon.com/login'
login_email = '*****@qq.com'
login_password = '****1391'
data = {
'login_email': login_email,
'login_password': login_password
}
# 使用session发起post请求来获取登录后的cookie,cookie已经存在session中
response = session.post(url=url, data=data)
# 把cookie保存到文件中
session.cookies.save()
def read_cookie():
"""
读取cookie进入登录后的页面
:return:
"""
index_url = 'https://www.processon.com/diagrams'
index_page = session.get(url=index_url).text
print(index_page)
def login_y_n():
"""
判断用户是否已经登录,我们这里使用的方法是:随便找一个登陆后页面的url,如果我们访问它时不发生重定向,我们就可以
判断该用户应经登录了
:return:
"""
url = 'https://www.processon.com/diagrams/new#template'
response = session.get(url = url,allow_redirects=False) # allow_redirects =False不允许重定向到登录页面
if response != 200:
return False
else:
return True
read_cookie()
方法二 使用selenium获取cookies
from selenium import webdriver
import json
browser = webdriver.Chrome(executable_path=r"E:\爬虫视频\day04\chromedriver_win32_2.46\chromedriver.exe")
def get_cookies():
"""
通过selenium获取cookie保存在文件中
:return:
"""
url = 'https://www.processon.com/login'
browser.get(url=url)
browser.find_element_by_id('login_email').send_keys('286867@qq.com')
browser.find_element_by_id('login_password').send_keys('ZZZ0391')
browser.find_element_by_id('signin_btn').click()
# 获取cookie,这里得到的是一个列表
cookies_list = browser.get_cookies()
browser.close()
with open("cookies.txt", "w") as fp:
json.dump(cookies_list, fp) # 这里切记,如果我们要使用json.load读取数据,那么一定要使用json.dump来写入数据,
# 不能使用str(cookies)直接转为字符串进行保存,因为其存储格式不同。这样我们就将cookies保存在文件中了。
def read_cookie():
"""
读取cookie,添加到browser中
:return:
"""
url = 'https://www.processon.com/diagrams'
browser.get(url=url) # 这里必须先访问一次否则页面不能打开
with open('./cookies.txt','r') as fp:
cookies_list = json.load(fp)
for cookies in cookies_list:
browser.add_cookie(cookies)
browser.get(url)
read_cookie()
注意用selenium来获取的cookie是一个列表,列表中有很多字典,字典中有domain、expiry、name、value、path等key,但是在我们真正的浏览器中就只有一个字典,字典中只有name 键对应的值和value对应的值,所以在使用的时候
还需要转换一下:
复制代码
[{"domain": ".processon.com", "expiry": 1560351255.689168, "httpOnly": false, "name": "_sid", "path": "/", "secure": false, "value": "796afe66ce2a6002828ab3ca281f96fb"},
{"domain": ".processon.com", "httpOnly": true, "name": "JSESSIONID", "path": "/", "secure": false, "value": "EBDACE1BDAB1464A2CCBBFFB7048A238.jvm1"},
{"domain": ".processon.com", "expiry": 1586703257, "httpOnly": false, "name": "zg_did", "path": "/", "secure": false, "value": "%7B%22did%22%3A%20%2216a173113351bb-062c441b2e33b7-7a1437-144000-16a173113362e%22%7D"},
{"domain": ".processon.com", "expiry": 1560351255.689117, "httpOnly": false, "name": "processon_userKey", "path": "/", "secure": false, "value": "59f7fba9e4b0edf0e25cd413"},
{"domain": ".processon.com", "expiry": 1555167313, "httpOnly": false, "name": "_gat", "path": "/", "secure": false, "value": "1"},
{"domain": ".processon.com", "expiry": 1555253657, "httpOnly": false, "name": "_gid", "path": "/", "secure": false, "value": "GA1.2.1345294219.1555167253"},
{"domain": ".processon.com", "expiry": 1618239257, "httpOnly": false, "name": "_ga", "path": "/", "secure": false, "value": "GA1.2.555498451.1555167253"},
{"domain": ".processon.com", "expiry": 1586703257, "httpOnly": false, "name": "zg_3f37ba50e54f4374b9af5be6d12b208f", "path": "/", "secure": false,
"value": "%7B%22sid%22%3A%201555167253312%2C%22updated%22%3A%201555167257424%2C%22info%22%3A%201555167253326%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform
%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%2C%22cuid%22%3A%20%2259f7fba9e4b0edf0e25cd413%22%7D"}]
复制代码
Cookie: zg_did=%7B%22did%22%3A%20%2216a16fc24ab1e7-08f589794c6e4d-7a1437-144000-16a16fc24ac76a%22%7D; _ga=GA1.2.1095343087.1555163784; _gid=GA1.2.545489346.1555163784; processon_userKey=59f7fba9e4b0edf0e25cd413; _sid=796afe66ce2a6002828ab3ca281f96fb; zg_3f37ba50e54f4374b9af5be6d12b208f=%7B%22sid%22%3A%201555163784372%2C%22updated%22%3A%201555165807015%2C%22info%22%3A%201555163784376%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%2C%22cuid%22%3A%20%2259f7fba9e4b0edf0e25cd413%22%7D; JSESSIONID=685AABAF6B8D70AF25E501C7E9E67A32.jvm1