免费代理
#encoding:utf-8
import requests
#1.请求url
url = "http://www.baidu.com"
headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"
}
#免费代理书写形式 {"http":"IP:port"}
free_proxy= {"http":"27.17.45.90:43411"}
response = requests.get(url,headers = headers,proxies =free_proxy )
print(response.status_code)
忽略证书认证SSL的操作
#encoding:utf-8
import requests
#1.请求url
url = "http://www.baidu.com"
headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"
}
#免费代理书写形式 {"http":"IP:port"}
free_proxy= {"http":"27.17.45.90:43411"}
response = requests.get(url,headers = headers,proxies =free_proxy )
print(response.status_code)
request的cookie请求
手动抓包获取cookie时,可以将cookie放在header里面,也可以在发送get请求时加上cookies的字典参数。注意:接受的参数是字典类型的,这就需要我们将cookie字符串转化为字典形式。手动转化或者正则都可。
方法一:cookie放在header里面
#encoding:utf-8
import requests
#请求数据url
member_url = 'https://www.yaozh.com/member/'
headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
"Cookie":"_ga=GA1.2.453819300.1605698635; _gid=GA1.2.1545737704.1605698635; _gat=1; Hm_lpvt_65968db3ac154c3089d7f9a4cbb98c94=1605699991; yaozh_uidhas=1; acw_tc=2f624a2c16056986318106268e7b5c5780592359df428fea4543999c74f920; Hm_lvt_65968db3ac154c3089d7f9a4cbb98c94=1605698635%2C1605699868%2C1605699919; UtzD_f52b_saltkey=TuAyAS9a; UtzD_f52b_lastvisit=1605697511; _ga=GA1.1.1391703708.1605701117; _gid=GA1.1.363878918.1605701117; UtzD_f52b_ulastactivity=1605699987%7C0; UtzD_f52b_creditrule=%E6%AF%8F%E5%A4%A9%E7%99%BB%E5%BD%95; PHPSESSID=vjeitstu34vqng73oargrnm487; yaozh_mylogin=1605744810; UtzD_f52b_creditnotice=0D0D2D0D0D0D0D0D0D833314; UtzD_f52b_creditbase=0D0D0D0D0D0D0D0D0; yaozh_userId=1008154; UtzD_f52b_lastact=1605760658%09uc.php%09; _gat=1"
}
#需要的是 字典类型
#方法一:手动拼 方法二:正则
response = requests.get(member_url,headers =headers )
data = response.content.decode("utf-8")
with open('04cook.html','w',encoding = 'utf-8') as f:
f.write(data)
方法二:手动转化为字典形式
#encoding:utf-8
import requests
#请求数据url
member_url = 'https://www.yaozh.com/member/'
# cookie 的字符串
#cookies = '_ga=GA1.2.453819300.1605698635; _gid=GA1.2.1545737704.1605698635; _gat=1; Hm_lpvt_65968db3ac154c3089d7f9a4cbb98c94=1605699991; yaozh_uidhas=1; acw_tc=2f624a2c16056986318106268e7b5c5780592359df428fea4543999c74f920; Hm_lvt_65968db3ac154c3089d7f9a4cbb98c94=1605698635%2C1605699868%2C1605699919; UtzD_f52b_saltkey=TuAyAS9a; UtzD_f52b_lastvisit=1605697511; _ga=GA1.1.1391703708.1605701117; _gid=GA1.1.363878918.1605701117; UtzD_f52b_ulastactivity=1605699987%7C0; UtzD_f52b_creditrule=%E6%AF%8F%E5%A4%A9%E7%99%BB%E5%BD%95; PHPSESSID=vjeitstu34vqng73oargrnm487; yaozh_mylogin=1605744810; UtzD_f52b_creditnotice=0D0D2D0D0D0D0D0D0D833314; UtzD_f52b_creditbase=0D0D0D0D0D0D0D0D0; yaozh_userId=1008154; UtzD_f52b_lastact=1605760658%09uc.php%09; _gat=1'
headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
}
#需要的是 字典类型
#方法一:手动拼 方法二:正则
cook_dict = {
"_ga":"GA1.2.453819300.1605698635",
"_gid":"GA1.2.1545737704.1605698635",
"_gat":"1",
"Hm_lpvt_65968db3ac154c3089d7f9a4cbb98c94":"1605699991",
"yaozh_uidhas":"1",
"acw_tc":"2f624a2c16056986318106268e7b5c5780592359df428fea4543999c74f920",
"Hm_lvt_65968db3ac154c3089d7f9a4cbb98c94":"1605698635%2C1605699868%2C1605699919",
"UtzD_f52b_saltkey":"TuAyAS9a",
"UtzD_f52b_lastvisit":"1605697511",
"_ga":"GA1.1.1391703708.1605701117",
"_gid":"GA1.1.363878918.1605701117",
"UtzD_f52b_ulastactivity":"1605699987%7C0",
"UtzD_f52b_creditrule":"%E6%AF%8F%E5%A4%A9%E7%99%BB%E5%BD%95", "PHPSESSID":"vjeitstu34vqng73oargrnm487",
"yaozh_mylogin":"1605744810",
"UtzD_f52b_creditnotice":"0D0D2D0D0D0D0D0D0D833314",
"UtzD_f52b_creditbase":"0D0D0D0D0D0D0D0D0",
"yaozh_userId":"1008154" ,
"UtzD_f52b_lastact":"1605760658%09uc.php%09",
"_gat":"1"
}
response = requests.get(member_url,headers =headers,cookies = cook_dict)
data = response.content.decode("utf-8")
with open('04cook.html','w',encoding = 'utf-8') as f:
f.write(data)
获取cookie字典类型方法三:分隔+for循环
#cookie需要的是字典类型
cook_dict = {}
cookies_list = cookies.split('; ')
for cookie in cookies_list:
#dict[key] = value
cook_dict[cookie.split('=')[0]] = cookie.split('=')[1]
获取cookie字典类型方法四:字典推导式
#cookie需要的是字典类型
cook_dict = {}
cook_dict = {
cookie.split('=')[0]:cookie.split('=')[1] for cookie in cookies.split('; ')
}
自动带着cookie去请求
#encoding:utf-8
import requests
# 请求数据url
member_url = 'https://www.yaozh.com/member/'
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
}
# session 类 可以自动保存cookies === cookiesJar
session = requests.session()
# 1.代码登录
login_url = 'https://www.yaozh.com/login'
login_form_data = {
"username": "wsssssang",
"pwd": "wss7622",
"formhash": "1F3337010D",
"backurl": "https%3A%2F%2Fwww.yaozh.com%2F"
}
login_response = session.post(login_url,data=login_form_data,headers=headers)
print(login_response.content.decode())
# 2.登录成功之后 带着 有效的cookies 访问 请求目标数据
data = session.get(member_url,headers=headers).content.decode(encoding = "utf-8")
with open('05-cookie2.html','w',encoding = "utf-8") as f:
f.write(data)