最简单的爬虫程序
from urllib.request import urlopen url="http://www.baidu.com" #发送请求 response=urlopen(url) #读取 info=response.read() #打印 print(info.decode())
request封装
from urllib.request import urlopen from urllib.request import Request url="http://www.baidu.com" #request封装 headers={ "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 FS" } #对比两次封装 url_1=Request(url) url_2=Request(url,headers=headers) print(url_1.get_header('User-agent')) print(url_2.get_header('User-agent')) #发送请求 response=urlopen(url_2) #读取 info=response.read() #打印 print(info.decode())
User-Agent中的内容可以从浏览器中的开发者模式中获取
程序运行结果:
每次访问用不同的headers(动态User-Agent)(手动版)
from urllib.request import urlopen from urllib.request import Request from random import choice url="http://www.baidu.com" user_agents=["Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11", "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv 11.0) like Gecko", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 FS"] #用choice,从列表中随机选择user-agent进行一个输出 #request封装 headers={ "User-Agent":choice(user_agents) } url_2=Request(url,headers=headers) print(url_2.get_header('User-agent')) #发送请求 response=urlopen(url_2) #读取 info=response.read() #打印 print(info.decode())
每次访问用不同的headers(动态User-Agent)(自动版)
from fake_useragent import UserAgent ua=UserAgent()#此处相当于申明类的实例 print(ua.random)#随机选择一个UA print(ua.chrome)#随机选择一个chrome类的UA print(ua.firefox)#随机选择一个firefox类的UA
把ua.random替换上方代码中的choice(user-agents)即可完成使用