python的requests库和xpath的入门使用

requests库

requests 库底层封装urllib
安装：
pip install requests

import requests
import chardet
url = "http://www.sina.com.cn"
#响应头
response = requests.get(url=url)
#获取响应头的编码
print(response.encoding)
#新浪的编码
print(chardet.detect(response.content))

ISO-8859-1 -------- ladin1编码

import requests
import chardet
url = "http://www.sina.com.cn"
#响应头
response = reque
#获取响应头的编码
# print(response.encoding)
#新浪的编码
# print(chardet.detect(response.content))
# print(response.text)
charset = chardet.detect(response.content).get("encoding")
print(charset)
response.encoding = charset
print(response.text)
with open("sina.html","w",encoding=charset) as f:
f.write(response.text)

需求：爬取新浪页面的图片

import re
import requests
import chardet
url = "http://www.sina.com.cn"
#响应头
response = requests.get(url=url)
#获取响应头的编码
# print(response.encoding)
#新浪的编码
# print(chardet.detect(response.content))
# print(response.text)
charset = chardet.detect(response.content).get("encoding")
print(charset)
response.encoding = charset
# print(response.text)
html = response.text
images = re.findall(r"src=\"(.*?jpg|png|gif|jepg)\"",html)
print(images)
print(len(images))
# with open("sina.html","w",encoding="utf-8") as f:
# f.write(response.text)
#enumerate
for