网络爬虫学习记录【WS01-03】

requests库

1. 安装requests库

pip install requests

2. 测试

import requests
r = requests.get("http://baidu.com")
print(r.status_code)
r.text

3. Requests库的7个主要方法
   
   
   **kwargs:控制访问的参数，均为可选项
   params:字典或字节序列，作为参数增加到URL中
   data:字典、字节序列或文件对象，作为request的内容
   json:JSON格式的数据，作为request的内容
   headers:字典，HTTP定制头
   cookies:字典或cookiejar，request中的cookie
   auth:元祖，支持HTTP认证功能
   files：字典类型，传输文件
   timeout:设定超时时间，单位为秒
   proxies:字典类型，设定访问代理服务器，可以增加登录认证
   allow_redirects
   stream
   verify
   cert

>>> kv={'k1':'v1','k2':'v2'}
>>> r = requests.request('GET','http://python123.io/ws',params=kv)
>>> print(r.url)
https://python123.io/ws?k1=v1&k2=v2
>>> hd={‘user-agent’:'Chrome/10'}
>>> r = requests.request('POST','http://python123.io/ws',headers=hd)
>>> fs={‘file’:open('data.xls','rb')}
>>> r = requests.request('POST','http://python123.io/ws',files=fs)

4. 爬取网页的通用代码框架

import requests
def getHTMLText(url):
		try:
			r = requests.get(url,timeout=30)
			r,raise_for_status() #不是200会引发异常
			r.encoding = r.apparent_encoding
			return r.text
		except:
			return '产生异常'
if __name__=="__main__":
	url = "http://baidu.com"
	print(getHTMLText(url))

if __name__的理解

5. Requests库的head()方法

>>> r = requests.get("http://www.baidu.com")
>>> r1=requests.head("http://www.baidu.com")
>>> r1.headers
{'Cache-Control': 'private, no-cache, no-store, proxy-revalidate, no-transform', 'Connection': 'keep-alive', 'Content-Encoding': 'gzip', 'Content-Type': 'text/html', 'Date': 'Fri, 16 Apr 2021 12:18:25 GMT', 'Last-Modified': 'Mon, 13 Jun 2016 02:50:26 GMT', 'Pragma': 'no-cache', 'Server': 'bfe/1.0.8.18'}
>>> r1.text
''
>>> r.headers #get方法也可以？
{'Cache-Control': 'private, no-cache, no-store, proxy-revalidate, no-transform', 'Connection': 'keep-alive', 'Content-Encoding': 'gzip', 'Content-Type': 'text/html', 'Date': 'Fri, 16 Apr 2021 12:17:47 GMT', 'Last-Modified': 'Mon, 23 Jan 2017 13:27:56 GMT', 'Pragma': 'no-cache', 'Server': 'bfe/1.0.8.18', 'Set-Cookie': 'BDORZ=27315; max-age=86400; domain=.baidu.com; path=/', 'Transfer-Encoding': 'chunked'}

6. Post()方法

>>> payload = {'key1':'v1','k2':'v2'}
>>> r = requests.post('http://httpbin.org/post'.data = payload)
SyntaxError: expression cannot contain assignment, perhaps you meant "=="?
>>> r = requests.post("http://httpbin.org/post",data = payload)
"form": {
    "k2": "v2", 
    "key1": "v1"
  }, #向URL post一个字典，自动编码为form表单
  #可以直接post字符串，自动编码为data
 >>>  r = requests.post("http://httpbin.org/post",data =‘ABC’)
  "data": "abc", 

Robots协议

在网站根目录下的robots.txt文件

实例

爬取京东商品页面，亚马逊同

import requests
url = "https://item.jd.com/2967929.html"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
    "cookie": "自己浏览器的信息（要先登入）"
}
r = requests.get(url, headers=headers)  # 因为京东啊，亚马逊都有反爬，所以要加入头部信息
r.raise_for_status()
r.encoding = r.apparent_encoding
print(r.text[:1001])

cookie信息登录后在Chrome浏览器F12Console,输入指令 document.cookie获取

百度搜索关键词提交

import requests
# 带上浏览器信息
headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"
}
r = requests.get("https://www.baidu.com/s?wd=python", headers=headers)
r.encoding = r.apparent_encoding
print(r.status_code)
print(r.text)

网络图片的爬取

import requests
import os
url = "https://ss1.bdstatic.com/70cFvXSh_Q1YnxGkpoWK1HF6hhy/it/u=3864420560,2390067063&fm=26&gp=0.jpg"
root =  "C:/Users/Dell/Desktop/答疑/"  # 保存路径
path = root + url.split('/')[-1]  # 新建文件名为root路径之后加上地址最后以“/”分割的部分
try:
    if not os.path.exists(root):  # 判断括号里的文件是否存在的意思，括号内的可以是文件路径
        os.mkdir(root)  # 不存在则创建目录
    if not os.path.exists(path):  # 文件不存在则开始爬取保存
        r = requests.get(url)
        with open(path, 'wb') as f:# w 是写入模式 r是读取模式 wb 是二进制写入
            f.write(r.content)#保存为二进制格式
            f.close()
            print("文件保存成功")
    else:
        print("文件已经存在")
except:
    print("爬取失败")

IP地址归属地的自动查询

import requests
text = "202.204.80.112"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.100 Safari/537.36",
}
url = "https://m.ip138.com/iplookup.asp?ip={}".format(text)
html = requests.get(url, headers=headers)
html.encoding = html.apparent_encoding
print(html.text)