爬虫学习-1

 最简单的爬虫程序

from urllib.request import urlopen

url="http://www.baidu.com"
#发送请求
response=urlopen(url)
#读取
info=response.read()
#打印
print(info.decode())

request封装

from urllib.request import urlopen
from urllib.request import Request

url="http://www.baidu.com"
#request封装
headers={
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 FS"
}
#对比两次封装
url_1=Request(url)
url_2=Request(url,headers=headers)
print(url_1.get_header('User-agent'))
print(url_2.get_header('User-agent'))
#发送请求
response=urlopen(url_2)
#读取
info=response.read()
#打印
print(info.decode())

User-Agent中的内容可以从浏览器中的开发者模式中获取

程序运行结果：

每次访问用不同的headers（动态User-Agent）(手动版)

from urllib.request import urlopen
from urllib.request import Request
from random import choice

url="http://www.baidu.com"
user_agents=["Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
             "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv 11.0) like Gecko",
             "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
             "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 FS"]
#用choice,从列表中随机选择user-agent进行一个输出
#request封装
headers={
    "User-Agent":choice(user_agents)
}
url_2=Request(url,headers=headers)
print(url_2.get_header('User-agent'))
#发送请求
response=urlopen(url_2)
#读取
info=response.read()
#打印
print(info.decode())

每次访问用不同的headers（动态User-Agent）(自动版)

from fake_useragent import UserAgent
ua=UserAgent()#此处相当于申明类的实例
print(ua.random)#随机选择一个UA
print(ua.chrome)#随机选择一个chrome类的UA
print(ua.firefox)#随机选择一个firefox类的UA

把ua.random替换上方代码中的choice(user-agents)即可完成使用