BeautifulSoup库使用
BeautifulSoup库是python的一个HTML解析库,可以利用它来提取网页中的数据
from bs4 import BeautifulSoup
# BeautifulSoup依赖第三方解释器,在初始化BeautifulSoup时第二个参数改为lxml,即用lxml解释器
soup=BeautifulSoup('<p>Hello</p>','lxml')
print(soup.p.string)
Hello
节点选择器
直接调用节点名称就可以选择节点
html = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title" name="dromouse"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1"><!-- Elsie --></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
from bs4 import BeautifulSoup
# 初始化BeautifulSoup对象,第一个参数是html文本,第二个参数是依赖的解析器,在初始化过程中会进行自动格式更正
soup=BeautifulSoup(html,'lxml')
print(soup.prettify()) # 方法prettify()将要解释的文本按标准的格式输出
print("--------------------")
print(soup.title.string) # 选出title节点,调用属性string输出文本内容
<html>
<head>
<title>
The Dormouse's story
</title>
</head>
<body>
<p class="title" name="dromouse">
<b>
The Dormouse's story
</b>
</p>
<p class="story">
Once upon a time there were three little sisters; and their names were
<a class="sister" href="http://example.com/elsie" id="link1">
<!-- Elsie -->
</a>
,
<a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>
and
<a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>
;
and they lived at the bottom of a well.
</p>
<p class="story">
...
</p>
</body>
</html>
--------------------
The Dormouse's story
选择元素
html = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title" name="dromouse"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1"><!-- Elsie --></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
print(type(soup.title))
print(soup.title)
print(soup.title.string)
print(soup.p) # 在所构造的对象后面直接加上节点名称,但这样的选择方式只能选择到第一个节点
<class 'bs4.element.Tag'>
<title>The Dormouse's story</title>
The Dormouse's story
<p class="title" name="dromouse"><b>The Dormouse's story</b></p>
提取信息
1.获取名称
# <title>The Dormouse's story</title>
print(soup.title.name) # name属性用于获取节点名称
title
2.获取属性
# <p class="title" name="dromouse"><b>The Dormouse's story</b></p>
print(soup.p.attrs) # 调用attrs用来获取标签的所有属性及其属性值,返回一个字典的格式
print(soup.p.attrs['name']) # 因此可以根据字典的方法获取某个属性的值
print(soup.p['name']) # 也可以直接在节点后面加中括号,里面传入要获取的属性名称
{'class': ['title'], 'name': 'dromouse'}
dromouse
dromouse
3.获取内容
# <p class="title" name="dromouse"><b>The Dormouse's story</b></p>
print(soup.p.string) # 属性string用于获取节点内的文本内容
The Dormouse's story
嵌套选择
在某个结点的基础上选择其内部的节点(内部所有节点中选,同名节点不止一个,用第一个)
from bs4 import BeautifulSoup
html='''<html><head><title>The Dormouse's story</title></head>'''
soup=BeautifulSoup(html,'lxml')
print(soup.head.title)
print(type(soup.head.title))
print(soup.head.title.string)
<title>The Dormouse's story</title>
<class 'bs4.element.Tag'>
The Dormouse's story
关联选择
1.子节点和子孙节点
contents属性将节点所有的子节点以列表的格式返回
html = """
<html>
<head>
<title>The Dormouse's story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">
<span>Elsie</span>
</a>
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
and they lived at the bottom of a well.
</p>
<p class="story">...</p>
"""
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
print(soup.p.contents) # 以列表的形式返回p节点的所有子节点
['\n Once upon a time there were three little sisters; and their names were\n ', <a class="sister" href="http://example.com/elsie" id="link1">
<span>Elsie</span>
</a>, '\n', <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>, ' \n and\n ', <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>, '\n and they lived at the bottom of a well.\n ']
children属性也可以用来获取节点所有子节点,但是它返回的是一个生成器,需要用循环来遍历
print(soup.p.children)
for i,child in enumerate(soup.p.children):
print(i,child)
<list_iterator object at 0x0000010DBF577C18>
0
Once upon a time there were three little sisters; and their names were
1 <a class="sister" href="http://example.com/elsie" id="link1">
<span>Elsie</span>
</a>
2
3 <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>
4
and
5 <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>
6
and they lived at the bottom of a well.
descendants属性用于获取节点所有的子孙节点,同样返回的的一个生成器,循环来遍历
print(soup.p.descendants)
for i,child in enumerate(soup.p.descendants):
print(i,child)
<generator object descendants at 0x0000010DBF5619E8>
0
Once upon a time there were three little sisters; and their names were
1 <a class="sister" href="http://example.com/elsie" id="link1">
<span>Elsie</span>
</a>
2
3 <span>Elsie</span>
4 Elsie
5
6
7 <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>
8 Lacie
9
and
10 <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>
11 Tillie
12
and they lived at the bottom of a well.
2.父节点和祖先节点
parent属性用来获取节点的父节点及其内部内容
html = """
<html>
<head>
<title>The Dormouse's story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">
<span>Elsie</span>
</a>
</p>
<p class="story">...</p>
"""
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, 'lxml')
print(soup.a.parent)
print(type(soup.a.parent))
<p class="story">
Once upon a time there were three little sisters; and their names were
<a class="sister" href="http://example.com/elsie" id="link1">
<span>Elsie</span>
</a>
</p>
<class 'bs4.element.Tag'>
parents属性用来获取节点的所有祖先节点及其内部内容,返回的是一个生成器
html = """
<html>
<body>
<p class="story">
<a href="http://example.com/elsie" class="sister" id="link1">
<span>Elsie</span>
</a>
</p>
"""
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, 'lxml')
print(type(soup.a.parents))
print(list(enumerate(soup.a.parents)))
<class 'generator'>
[(0, <p class="story">
<a class="sister" href="http://example.com/elsie" id="link1">
<span>Elsie</span>
</a>
</p>), (1, <body>
<p class="story">
<a class="sister" href="http://example.com/elsie" id="link1">
<span>Elsie</span>
</a>
</p>
</body>), (2, <html>
<body>
<p class="story">
<a class="sister" href="http://example.com/elsie" id="link1">
<span>Elsie</span>
</a>
</p>
</body></html>), (3, <html>
<body>
<p class="story">
<a class="sister" href="http://example.com/elsie" id="link1">
<span>Elsie</span>
</a>
</p>
</body></html>)]
3.兄弟节点
next_sibling:返回节点的下一个兄弟元素
previous_siblings:返回节点的上一个兄弟元素
next_siblings:返回节点前面所有兄弟元素,返回一个生成器
previous_siblings:返回节点后面所有的兄弟元素,返回一个生成器
html = """
<html>
<body>
<p class="story">
Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">
<span>Elsie</span>
</a>
Hello
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
and they lived at the bottom of a well.
</p>
"""
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
print('Next Sibling',soup.a.next_sibling)
print('Prev Sibling',soup.a.previous_sibling)
print('Next Siblings',list(enumerate(soup.a.next_siblings)))
print('Prev Siblings',list(enumerate(soup.a.previous_siblings)))
Next Sibling
Hello
Prev Sibling
Once upon a time there were three little sisters; and their names were
Next Siblings [(0, '\n Hello\n '), (1, <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>), (2, ' \n and\n '), (3, <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>), (4, '\n and they lived at the bottom of a well.\n ')]
Prev Siblings [(0, '\n Once upon a time there were three little sisters; and their names were\n ')]
4.提取信息
html = """
<html>
<body>
<p class="story">
Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Bob</a><a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
</p>
"""
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
print('Next Sibling:')
print(type(soup.a.next_sibling))
print(soup.a.next_sibling)
print(soup.a.next_sibling.string)
print('Parent:')
print(type(soup.a.parents))
print(list(soup.a.parents)[0])
print(list(soup.a.parents)[0].attrs['class'])
Next Sibling:
<class 'bs4.element.Tag'>
<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>
Lacie
Parent:
<class 'generator'>
<p class="story">
Once upon a time there were three little sisters; and their names were
<a class="sister" href="http://example.com/elsie" id="link1">Bob</a><a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>
</p>
['story']
方法选择器
find_all()
find_all( name, attrs, recursive, text, **kwargs)
查询所有符合条件的元素,给它传入一些属性和文本就能得到所有符合条件的元素
1.name
html='''
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</li>
<li class="element">Bar</li>
</ul>
</div>
</div>
'''
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
print(soup.find_all(name='ul')) # 查询名为ul的节点,返回一个包含所有ul节点的列表
print(type(soup.find_all(name='ul')[0])) #
print('--------------------------------------')
for ul in soup.find_all(name='ul'): # 由于元素类型为bs4.element.Tag类型,因此可以继续查询
print(ul.find_all(name='li'))
for li in ul.find_all(name='li'):
print(li.string)
[<ul class="list" id="list-1">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>, <ul class="list list-small" id="list-2">
<li class="element">Foo</li>
<li class="element">Bar</li>
</ul>]
<class 'bs4.element.Tag'>
--------------------------------------
[<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay</li>]
Foo
Bar
Jay
[<li class="element">Foo</li>, <li class="element">Bar</li>]
Foo
Bar
2.attrs
html='''
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1" name='elements'>
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</li>
<li class="element">Bar</li>
</ul>
</div>
</div>
'''
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
# 传入attrs属性的类型是字典类型,key为属性名,value为属性值
print(soup.find_all(attrs={'id':'list-1'}))
print(soup.find_all(attrs={'name':'elements'}))
print("----------------------------------")
# 对于常用属性id,class...我们可以不用attrs来查询
print(soup.find_all(id='list-1'))
print(soup.find_all(class_='element')) # class在python中是关键字,后跟下划线
[<ul class="list" id="list-1" name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>]
[<ul class="list" id="list-1" name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>]
----------------------------------
[<ul class="list" id="list-1" name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>]
[<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay</li>, <li class="element">Foo</li>, <li class="element">Bar</li>]
3.text
# text可以用来匹配节点的文本,传入形式可以是字符也可以是正则表达式对象
html='''
<div class="panel">
<div class="panel-body">
<a>Hello, this is a link</a>
<a>Hello, this is a link, too</a>
</div>
</div>
'''
from bs4 import BeautifulSoup
import re
soup=BeautifulSoup(html,'lxml')
print(soup.find_all(text=re.compile('link'))) # 返回列表形式
['Hello, this is a link', 'Hello, this is a link, too']
find()
与find_all()方法相似,不过返回的是第一个查询到的单个元素
html='''
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</li>
<li class="element">Bar</li>
</ul>
</div>
</div>
'''
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
print(soup.find(name='ul'))
print(type(soup.find(name='ul')))
print(soup.find(class_='list'))
<ul class="list" id="list-1">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>
<class 'bs4.element.Tag'>
<ul class="list" id="list-1">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>
其他方法
用法与find_all和find完全相同但查询范围不同
find_parents():返回所有祖先节点
find_parent():返回所有直接父节点
find_next_siblings():返回后面所有兄弟节点
find_next_sibling():返回后面第一个兄弟节点
find_previous_siblings():返回前面所有兄弟节点
find_previous_sibling():返回前面第一个兄弟节点
find_all_next():返回节点后面所有符合条件的节点
find_next():返回节点后面第一个符合条件的节点
find_all_previous():返回节点前面所有符合条件的节点
find_previous():返回节点前面第一符合条件的节点
CSS选择器
利用select()方法来使用CSS选择器进行选择
html='''
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</li>
<li class="element">Bar</li>
</ul>
</div>
</div>
'''
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
print(soup.select(".panel .panel-heading"))
print(soup.select("ul li"))
print(soup.select('#list-2 .element'))
print(type(soup.select('ul')[0]))
[<div class="panel-heading">
<h4>Hello</h4>
</div>]
[<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay</li>, <li class="element">Foo</li>, <li class="element">Bar</li>]
[<li class="element">Foo</li>, <li class="element">Bar</li>]
<class 'bs4.element.Tag'>
1.嵌套选择
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
for ul in soup.select('ul'):
print(ul.select("li"))
[<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay</li>]
[<li class="element">Foo</li>, <li class="element">Bar</li>]
2.获取属性
from bs4 import BeautifulSoup
soup=BeautifulSoup(html,'lxml')
for ul in soup.select('ul'):
print(ul['id'])
print(ul.attrs['id'])
list-1
list-1
list-2
list-2
3.获取文本
from bs4 import BeautifulSoup
# 获取节点文本内容,除了属性string,还有get_text()
soup=BeautifulSoup(html,'lxml')
for li in soup.select('li'):
print('Get Text:',li.get_text())
print('String:',li.string)
Get Text: Foo
String: Foo
Get Text: Bar
String: Bar
Get Text: Jay
String: Jay
Get Text: Foo
String: Foo
Get Text: Bar
String: Bar
参考:崔庆才《python3网络爬虫开发实战》