爬虫中，Requests的基本使用

Requests的基本使用

Request官网文档：https://docs.python-requests.org/zh_CN/latest/

import requests

url = 'https://www.baidu.com'

res = requests.get(url=url)

# 一个类型和六个属性
# 它的Response类型
print(type(res))

# 设置响应的编码格式
res.encoding = 'utf-8'

# 一字符串形式，返回网页源码
print(res.text)

# 返回一个url地址
print(res.url)

# 返回二进制的数据
print(res.content)

# 返回响应的状态码
print(res.status_code)

# 返回响应头
print(res.headers)

Requests_get请求

import requests

url = 'https://www.baidu.com/s?'

head = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36'
}

data = {
    'wd':'北京'
}

res = requests.get(url=url,params=data,headers=head)
content = res.text

print(content)


# 参数使用params传递，参数无需urlencode编码
# 不需要请求对象的定制
# 请求资源路径中的 ? 可以加也可以不加

Requests_post请求

import requests
import json

url =  'https://fanyi.baidu.com/v2transapi?from=zh&to=en'
head ={
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
}
data = {
    'query': '眼镜'
}

res = requests.post(url=url,data=data,headers=head)
content = res.text

obj = json.loads(content)
print(obj)


# （1）、post请求不需要编解码
# （2）、post请求的参数是data
# （3）、不需要请求对象的定制

Requests 代理

import requests

url =  'https://www.baidu.com/s?'
head ={
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
}
data = {
    'wd': 'ip'
}
proxy = {
    'https':'212.129.250.55:16816'
}

res = requests.get(url=url,params=data,headers=head,proxies=proxy)
content = res.text

with open('daili.html','w',encoding='utf-8') as fp:
    fp.write(content)

Requests_cookie登录

通过登陆 然后进入到主页面

'''
__VIEWSTATE: 4xNKK4WTIZERi2yXW2RFFwsK6D+/AYZvM+D2YbmJnRXH+xUcVJQFsj2JrDKFkJ7yCMQaxmElcJjge7uP93ho3ms/QilejHP0X+uUSK3w3Clx5g5hTpFT7HqJZtE=
__VIEWSTATEGENERATOR: C93BE1AE
from: http://so.gushiwen.cn/user/collect.aspx
email: 123123123@qq.com
pwd: nidaye
code: MOIM
denglu: 登录
'''


import requests
from bs4 import BeautifulSoup
import urllib.request

url = 'https://so.gushiwen.cn/user/login.aspx?from=http://so.gushiwen.cn/user/collect.aspx'
head = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
}
# 获取登录页面的源码
res = requests.get(url=url,headers=head)
con = res.text


# 解析页面源码
soup = BeautifulSoup(con,'lxml')

# 获取 __VIEWSTATE 和 __VIEWSTATEGENERATOR
# 这两个隐藏域是反爬手段之一，每次登录都不同，所以要去获取
VIEWSTATE = soup.select('#__VIEWSTATE')[0].attrs.get('value')
VIEWSTATEGENERATOR = soup.select('#__VIEWSTATEGENERATOR')[0].attrs.get('value')


# 获取验证码图片
code = soup.select('#imgCode')[0].attrs.get('src')
code_url = 'https://so.gushiwen.cn' + code
# 验证码不能这么下载。这方法下载的验证码是下面登录时的前一个，因为服务请求了两次
# urllib.request.urlretrieve(url=code_url,filename='code.jpg')

session = requests.session()
# 验证码的url内容
res_code = session.get(code_url)
content_code = res_code.content
# 下载之后若pycharm目录栏没有这个文件，则去本地磁盘查看
with open('code.jpg','wb') as fp:
    fp.write(content_code)


code_name = input('请输入验证码：')


# 点击登录
url_post = 'https://so.gushiwen.cn/user/login.aspx?from=http%3a%2f%2fso.gushiwen.cn%2fuser%2fcollect.aspx'

data = {
    '__VIEWSTATE': VIEWSTATE,
    '__VIEWSTATEGENERATOR': VIEWSTATEGENERATOR,
    'from': 'http://so.gushiwen.cn/user/collect.aspx',
    'email': '595165358@qq.com',
    'pwd': 'action',
    'code': code_name,
    'denglu': '登录',
}


resquest = session.post(url=url_post,headers=head,data=data)
content = resquest.text
with open('gushiwen.html','w',encoding='utf-8') as f:
    f.write(content)

Cookie登录附件

通过超级鹰，识别图片验证码。这个是收费的，不同类型的验证码价格不一样。

import requests
from hashlib import md5

class Chaojiying_Client(object):

    def __init__(self, username, password, soft_id):
        self.username = username
        password =  password.encode('utf8')
        self.password = md5(password).hexdigest()
        self.soft_id = soft_id
        self.base_params = {
            'user': self.username,
            'pass2': self.password,
            'softid': self.soft_id,
        }
        self.headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
        }

    def PostPic(self, im, codetype):
        """
        im: 图片字节
        codetype: 题目类型 参考 http://www.chaojiying.com/price.html
        """
        params = {
            'codetype': codetype,
        }
        params.update(self.base_params)
        files = {'userfile': ('ccc.jpg', im)}
        r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
        return r.json()

    def ReportError(self, im_id):
        """
        im_id:报错题目的图片ID
        """
        params = {
            'id': im_id,
        }
        params.update(self.base_params)
        r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
        return r.json()


if __name__ == '__main__':
	chaojiying = Chaojiying_Client('超级鹰用户名', '超级鹰用户名的密码', '96001')	#用户中心>>软件ID 生成一个替换 96001
	im = open('code.jpg', 'rb').read()													#本地图片文件路径 来替换 a.jpg 有时WIN系统须要//
	print(chaojiying.PostPic(im, 1902))											#1902 验证码类型  官方网站>>价格体系 3.4+版 print 后要加()