爬虫：模拟登录、会话维持、cookie登录、代理、禁止重定向、超时处理、重试

目录

模拟登录

会话维持

cookie登录

代理

禁止重定向

超时处理

重试

---

模拟登录

import requests

login_url = 'https://ptlogin.4399.com/ptlogin/login.do?v=1'

headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
}

data = {
    'loginFrom': 'uframe',
    'postLoginHandler': 'default',
    'layoutSelfAdapting': 'true',
    'externalLogin': 'qq',
    'displayMode': 'popup',
    'layout': 'vertical',
    'bizId': '',
    'appId': 'www_home',
    'gameId': '',
    'css': '',
    'redirectUrl': '',
    'sessionId': '',
    'mainDivId': 'popup_login_div',
    'includeFcmInfo':' false',
    'level': '0',
    'regLevel': '4',
    'userNameLabel': '4399用户名',
    'userNameTip': '请输入4399用户名',
    'welcomeTip': '欢迎回到4399',
    'sec': '1',
    'password':'qwe123',
    'username':'ddxt0112',
    'autoLogin':'on'

}

response = requests.post(url=login_url,headers=headers,data=data)
# print(response.text)


profile_url = 'https://u.4399.com/profile/'
resp = requests.get(url=profile_url,headers=headers)
print(resp.text)

会话维持

import requests

login_url = 'https://ptlogin.4399.com/ptlogin/login.do?v=1'
profile_url = 'https://u.4399.com/profile/'
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
}

data = {
    'loginFrom': 'uframe',
    'postLoginHandler': 'default',
    'layoutSelfAdapting': 'true',
    'externalLogin': 'qq',
    'displayMode': 'popup',
    'layout': 'vertical',
    'bizId': '',
    'appId': 'www_home',
    'gameId': '',
    'css': '',
    'redirectUrl': '',
    'sessionId': '',
    'mainDivId': 'popup_login_div',
    'includeFcmInfo':' false',
    'level': '0',
    'regLevel': '4',
    'userNameLabel': '4399用户名',
    'userNameTip': '请输入4399用户名',
    'welcomeTip': '欢迎回到4399',
    'sec': '1',
    'password':'qwe123',
    'username':'ddxt0112',
    'autoLogin':'on'

}
# session：保存服务器发送过来的cookie，并保存着下次用
session = requests.session()

response = session.post(url=login_url,headers=headers,data=data)
# print(response.text)
resp = session.get(url=profile_url,headers=headers)
print(resp.text)

cookie登录

import requests

login_url = 'https://ptlogin.4399.com/ptlogin/login.do?v=1'
profile_url = 'https://u.4399.com/profile/'
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
}

data = {
    'loginFrom': 'uframe',
    'postLoginHandler': 'default',
    'layoutSelfAdapting': 'true',
    'externalLogin': 'qq',
    'displayMode': 'popup',
    'layout': 'vertical',
    'bizId': '',
    'appId': 'www_home',
    'gameId': '',
    'css': '',
    'redirectUrl': '',
    'sessionId': '',
    'mainDivId': 'popup_login_div',
    'includeFcmInfo':' false',
    'level': '0',
    'regLevel': '4',
    'userNameLabel': '4399用户名',
    'userNameTip': '请输入4399用户名',
    'welcomeTip': '欢迎回到4399',
    'sec': '1',
    'password':'qwe123',
    'username':'ddxt0112',
    'autoLogin':'on'

}
# session：保存服务器发送过来的cookie，并保存着下次用
session = requests.session()

response = session.post(url=login_url,headers=headers,data=data)
# print(response.text)
resp = session.get(url=profile_url,headers=headers)
print(resp.text)

代理

import requests

url = 'https://httpbin.org/get'
proxies = {
    'http':''
}

response = requests.get(url=url,proxies=proxies)
print(response)

禁止重定向

import requests

url = 'https://h5.pipix.com/s/JFfPN5J/'
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
}

# allow_redirects赋值为False就是禁止重定向
response = requests.get(url=url,headers=headers,allow_redirects=False)
print(response.url)
print(response.request.url)

超时处理

import requests

url = 'https://www.instagram.com/'
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
}


# response = requests.get(url=url,headers=headers,timeoue=5)
response = requests.get(url=url,headers=headers,timeoue=(5,10)) #第一个代表请求发过去的时间 第二个代表响应返回来的时间

print(response.url)
print(response.request.url)

重试

#pip install retrying -i https://pypi.tuna.tsinghua.edu.cn/simple

import requests
from retrying import retry


url = 'https://www.instagram.com/'
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
}

# @retry # 如果报错就重新执行，不会显示报错信息
@retry(stop_max_attempt_number=3) #最多重试几次
@retry(stop_max_delay=3000) # 在这个时间内可以重试无数次，单位是毫秒
@retry(wait_fixed=3000) # 每次失败间隔多久，重试一次，单位是毫秒
@retry(wait_random_min=1000,wait_random_man=6000) # 随机间隔时间，重试一次，单位是毫秒

def res():
    print(1)
    response = requests.get(url=url,headers=headers,timeoue=5)
    print(response.url)

res()