数据挖掘--requests库

数据挖掘----requests库

安居客网页请求案例

import requests
r = requests.get("https://beijing.anjuke.com/?")
print(r)
#返回text类型数据
print(r.text)
#返回二进制类型数据
print(r.content)

import random
import requests

#headers应该是一个字典，通常用于欺骗
useragent1 = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"
useragent2 = " Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"
ulist = [useragent1,useragent2]
ug = random.choice(ulist)
header = {"User-Agent":ug}

wd = {"wd":"和润临湖美景"}

r = requests.get("https://Zhengzhou.anjuke.com/?",params=wd,headers = header)
print(r)
#返回text类型数据
print(r.text)

#如果有异常情况：
try:
    print(r.text)
except ConnectionError:
    print("停止输出")

模拟post请求

get请求只有请求头和请求行，而post请求相较之多了请求体。

import requests
url = "https://www.iqianyue.com/mypost"
formdata = {"name":"hero","pass":"12345676"}
res = requests.post(url,data =formdata)
print(res.text)

模拟Cookies进行登录

import requests

url = "https://www.douban.com/"
header = {"Cookie":'ll="108288"; bid=yxuDz4uONcc; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1605147282%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3Dt9Ux05LfPTe4wqPQe5ibd5m6fSiCydDkTQtXuHv3AUyiSmQYMi_erxHsI9DJJMSK%26wd%3D%26eqid%3D8f4e6e9600066aa8000000035fac9a88%22%5D; _pk_ses.100001.8cb4=*; __utma=30149280.600627785.1605147284.1605147284.1605147284.1; __utmc=30149280; __utmz=30149280.1605147284.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; ap_v=0,6.0; push_noty_num=0; push_doumail_num=0; __utmv=30149280.22644; __yadk_uid=n6dMyTGcbnjUglQrZ9VDtD68zhIzT0Gg; __utmt=1; __gads=ID=d107ec3fda8832c4:T=1605147888:S=ALNI_MZR0IvsxAR6714_prmrmUS7oVFUGQ; _pk_id.100001.8cb4=fa399abba7497abc.1605147282.1.1605147994.1605147282.; __utmb=30149280.13.10.1605147284; dbcl2="226449379:tlzDv8q45BA"'}
res = requests.get(url,headers = header)
print(res.text)

获取Cookie对象

url = "https://www.douban.com/"
res = requests.get(url)
#获取返回的cookies对象
cookiesjar = res.cookies
#将cookiejar转为字典
cookiesdic = requests.utils.dict_from_cookiejar(cookiesjar)
print(cookiesdic)

使用session进行访问

import requests

url = "https://www.douban.com/"
#创建session对象
session = requests.session()
#构造登录所需要的参数
data = {"username":"15291105932 ","password":"19980805"}
#提交数据，获取cookie
session.post(url,data)
#让session再次进行请求
res = session.get(url)
print(res.text)