BeautifulSoup库详解

最新推荐文章于 2024-08-19 09:22:32 发布

我想和你写代码

最新推荐文章于 2024-08-19 09:22:32 发布

阅读量237

点赞数

分类专栏：好记性不如烂博客文章标签： python 选择器标签 BeautifulSoup

本文链接：https://blog.csdn.net/l4642247/article/details/81060559

版权

好记性不如烂博客专栏收录该内容

17 篇文章 0 订阅

订阅专栏

基本使用

from bs4 import BeautifulSoup

html = '''
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title" name="dromouse"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1"><!-- Elsie --></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
'''
soup = BeautifulSoup(html,'lxml')
print(soup.prettify())
print(soup.title.string)

print(soup.title)        #out：<title>The Dormouse's story</title>
print(type(soup.title))  #out：<class 'bs4.element.Tag'>
print(soup.head)         #out：<head><title>The Dormouse's story</title></head>
print(soup.p)            #只输出了第一个p标签

获取属性

print(soup.p.attrs['name'])
print(soup.p['name'])

获取内容

print(soup.p.string)

嵌套选择

print(soup.head.title.string)

子节点和子孙节点

from bs4 import BeautifulSoup


html = '''
<html>
    <head>
        <title>The Dormouse's story</title>
    </head>
    <body>
        <p class="story">
            Once upon a time there were three little sisters; and their names were
            <a href="http://example.com/elsie" class="sister" id="link1">
                <span>Elsie</span>
            </a>
            <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> 
            and
            <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
            and they lived at the bottom of a well.
        </p>
        <p class="story">...</p>
'''
soup = BeautifulSoup(html,'lxml')
print(soup.p.contents)#集合形式
print(soup.p.children)#迭代器
for i,child in enumerate(soup.p.children):
    print(i, child)

print(soup.p.descendants)#子孙节点
for i,child in enumerate(soup.p.descendants):
    print(i, child)

[u'\n            Once upon a time there were three little sisters; and their names were\n            ', <a class="sister" href="http://example.com/elsie" id="link1">\n<span>Elsie</span>\n</a>, u'\n', <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>, u' \n            and\n            ', <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>, u'\n            and they lived at the bottom of a well.\n        ']
<listiterator object at 0x0000000002B51198>
(0, u'\n            Once upon a time there were three little sisters; and their names were\n            ')
(1, <a class="sister" href="http://example.com/elsie" id="link1">\n<span>Elsie</span>\n</a>)
(2, u'\n')
(3, <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>)
(4, u' \n            and\n            ')
(5, <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>)
(6, u'\n            and they lived at the bottom of a well.\n        ')

<generator object descendants at 0x0000000002B59C18>
(0, u'\n            Once upon a time there were three little sisters; and their names were\n            ')
(1, <a class="sister" href="http://example.com/elsie" id="link1">\n<span>Elsie</span>\n</a>)
(2, u'\n')
(3, <span>Elsie</span>)
(4, u'Elsie')
(5, u'\n')
(6, u'\n')
(7, <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>)
(8, u'Lacie')
(9, u' \n            and\n            ')
(10, <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>)
(11, u'Tillie')
(12, u'\n            and they lived at the bottom of a well.\n        ')

父节点和祖先节点还有兄弟节点

from bs4 import BeautifulSoup

html = '''
<html>
    <head>
        <title>The Dormouse's story</title>
    </head>
    <body>
        <p class="story">
            Once upon a time there were three little sisters; and their names were
            <a href="http://example.com/elsie" class="sister" id="link1">
                <span>Elsie</span>
            </a>
            <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> 
            and
            <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
            and they lived at the bottom of a well.
        </p>
        <p class="story">...</p>
'''
soup = BeautifulSoup(html,'lxml')
print(soup.p.contents)
print(list(enumerate(soup.p.parents)))
print(list(enumerate(soup.a.next_siblings)))
print(list(enumerate(soup.a.previous_siblings)))

标准选择器

find_all

from bs4 import BeautifulSoup

html = '''
<div class="panel">
    <div class="panel-heading">
        <h4>Hello</h4>
    </div>
    <div class="panel-body">
        <ul class="list" id="list-1">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
            <li class="element">Jay</li>
        </ul>
        <ul class="list list-small" id="list-2">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
        </ul>
    </div>
</div>
'''
soup = BeautifulSoup(html,'lxml')
print(soup.find_all('ul'))
print(type(soup.find_all('ul')[0]))

attrs

soup = BeautifulSoup(html,'lxml')
print(soup.find_all(attrs={'id':'list-1'}))
print(soup.find_all(attrs={'name':'elements'}))
#简写
soup = BeautifulSoup(html,'lxml')
print(soup.find_all(id='list-1'))
print(soup.find_all(class_='element'))

text

print(soup.find_all(text='Foo'))

find

find_parents
find_next_siblings
find_previous_siblings
...

CSS选择器

from bs4 import BeautifulSoup

html = '''
<div class="panel">
    <div class="panel-heading">
        <h4>Hello</h4>
    </div>
    <div class="panel-body">
        <ul class="list" id="list-1" name='elements'>
            <li class="element">Foo</li>
            <li class="element">Bar</li>
            <li class="element">Jay</li>
        </ul>
        <ul class="list list-small" id="list-2">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
        </ul>
    </div>
</div>
'''
soup = BeautifulSoup(html,'lxml')
print(soup.select('.panel .panel-heading'))
print(soup.select('ul li'))
print(soup.select('#list-2 .element'))

获取属性

print(type(soup.select('ul')[0]))
for ul in soup.select('ul'):
    print ul['id'],'---'
    print ul.attrs['id']

获取内容

for li in soup.select('li'):
    print li.get_text()