bs4的BeautifulSoup使用

基本语法

from bs4 import BeautifulSoup
#实例化BeautifulSoup对象，然后把即将被解析的页面源码数据加载到了该对象中
soup = BeautifulSoup(respone,'lxml') #respone，表示html源码，可以读取本地文件，也可以读取网络请求

标签定位

# 标签定位,只会得到第一个匹配到的数据
# 语法：soup.标签名

import requests
from bs4 import BeautifulSoup
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'}
url = 'https://www.jd.com/'
respone = requests.get(url=url,headers=headers)
respone.encoding = 'utf-8'
respone= respone.text
soup = BeautifulSoup(respone,'lxml') 
print(soup.title)
print(soup.link) 


返回结果：
<title>京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！</title>
<link href="//static.360buyimg.com" rel="dns-prefetch"/>

属性定位

# find('tagName',attrName='attrValue')：find只会定位到满足要的第一个标签
# findAll('tagName',attrName='attrValue')：可以定位到满足要求的所有标签
# attrName:除class属于要写成class_,其他的属性可以直接在源码那里复制，如：role="serachbox"

import requests
from bs4 import BeautifulSoup
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'}
url = 'https://www.jd.com/'
respone = requests.get(url=url,headers=headers)
respone.encoding = 'utf-8'
respone= respone.text
soup = BeautifulSoup(respone,'lxml')
print(soup.find('div',class_='dt cw-icon'))
print(soup.findAll('div',class_='dt cw-icon'))

返回结果：
<div class="dt cw-icon"><a href="//home.jd.com/" target="_blank">我的京东</a><i class="iconfont"></i><i class="ci-right"><s>◇</s></i></div>

[<div class="dt cw-icon"><a href="//home.jd.com/" target="_blank">我的京东</a><i class="iconfont"></i><i class="ci-right"><s>◇</s></i></div>, <div class="dt cw-icon">客户服务<i class="iconfont"></i><i class="ci-right"><s>◇</s></i></div>, <div class="dt cw-icon">网站导航<i class="iconfont"></i><i class="ci-right"><s>◇</s></i></div>]

选择器定位

# 选择器定位：
# soup.select("标签名") 得到所有标签,返回list
# soup.select('[attr = "attrVlaue"]') 得到attr属性的属性值为attrVlaue的标签
# soup.select('div[attr = "attrVlaue"]')，得到div标签且div标签的属性attr = "attrVlaue"的数据，标签不加空格，表示同级，加空格则表示，div下一级的属性attr = "attrVlaue"的数据
# soup.select('.tang > ul > li') 定位到了class为tang下面的ul下面所有的li标签
# soup.select('.tang  ul') 定位到了class为tang下面包含ul的标签

import requests
from bs4 import BeautifulSoup
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'}
url = 'https://www.jd.com/'
respone = requests.get(url=url,headers=headers)
respone.encoding = 'utf-8'
respone= respone.text
soup = BeautifulSoup(respone,'lxml')
tag1=soup.select('div')
tag2=soup.select('[class="search_logo_lk"]')
tag3=soup.select('a[class="search_logo_lk"]')
tag4=soup.select('div [class="search_logo_lk"]')
tag5=soup.select('div a[class="search_logo_lk"]')
tag6=soup.select('[role="serachbox"]>ul') #定位到了role="serachbox"属性下面的ul

提取标签中的内容

#tag.string:只可以提取到标签中直系的文本内容
#tag.text:可以提取到标签中所有的文本内容
#tag['src']) 提取标签的src的属性值


import requests
from bs4 import BeautifulSoup
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'}
url = 'https://www.jd.com/'
respone = requests.get(url=url,headers=headers)
respone.encoding = 'utf-8'
respone= respone.text
soup = BeautifulSoup(respone,'lxml')
tag=soup.select('[href="//cart.jd.com/cart.action"]')[0]
print(tag.string)
print(tag.text)
print(tag['target'])

返回结果：
我的购物车
我的购物车
_blank