爬虫学习day1

1 .UA反爬

#使用学习urllib (爬取百度首页源码)
#    UA反爬
import urllib.request

#百度加密网址
url_page = 'https://www.baidu.com/'

#网站UA
headers = {'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.203'}

#生成请求对象
requests = urllib.request.Request(url=url_page,headers=headers)

#模拟浏览器访问页面
response = urllib.request.urlopen(requests)

#读取源码与解码
content = response.read().decode('utf-8')

print(response.getcode())

2. 编解码——1.get请求方式:urllib.parse.quote()

#使用学习urllib
#编解码——1.get请求方式:urllib.parse.quote()
import urllib.request
import urllib.parse

#'https://www.baidu.com/s?wd=周杰伦'
url_page = 'https://www.baidu.com/s?wd='
headers = {'User-Agent':
               'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
               ' Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.203'}

# 依赖urllib.parse.quote把‘周杰伦’变成 Unicode 编码格式
name = urllib.parse.quote('周杰伦')
url_page = url_page + name

requests = urllib.request.Request(url=url_page, headers=headers)

response = urllib.request.urlopen(requests)

content = response.read().decode('utf-8')

print(content)

#未能通过百度的检测导致爬取失败

返回结果未能通过百度的检测导致爬取失败