Python爬虫入门（二）BeautifulSoup基础

一、BeautifulSoup基础

from bs4 import BeautifulSoup
import requests
''' bs4入门 '''
# !!!find_all找到所有，返回列表  find找到第一个


def test():
    #  BeautifulSoup对象对应一个html文件
    #  BeautifulSoup类基本元素
    #  tag tag.name tag.attrs tag.string Comment
    r = requests.get("http://python123.io/ws/demo.html")
    r.encoding = r.apparent_encoding
    demo = r.text
    soup = BeautifulSoup(demo, "html.parser")  # 对html解释，用html.parser解释器
    # print(soup.prettify())
    print(soup.title)
    # 返回第一个a标签
    tag = soup.a
    print(tag)
    print(tag.name)
    print(tag.parent.name)
    print(tag.attrs)
    print(tag.string)
    print(soup.p.string)


# test()


def traverse():
    #  HTML内容下行遍历
    r = requests.get("http://python123.io/ws/demo.html")
    r.encoding = r.apparent_encoding
    demo = r.text
    soup = BeautifulSoup(demo, "html.parser")
    head = soup.head
    print(head)
    print(head.contents)  # 返回列表，代表儿子标签
    print(soup.body.contents)  # 不仅包括标签结点，也包括字符串结点
    # tap.children 迭代孩子  tap.descendants 迭代子孙

    # 上行遍历
    print(soup.a.parent)
    print(type(soup.a.parents))
    for parent in soup.a.parents:
        if parent is None:
            print(parent)
        else:
            print(parent.name)

    # 平行遍历 同一个父节点 后一个结点不一定是标签，可能是string
    print(soup.a.next_sibling)
    print(type(soup.a.previous_siblings))  # 不是列表类型，是迭代类型
    for sibling in soup.a.previous_siblings:
        print(sibling)


# traverse()


def format_code_html():
    # html格式化和编码 bs4库会将编码默认utf8
    r = requests.get("http://python123.io/ws/demo.html")
    r.encoding = r.apparent_encoding
    demo = r.text
    soup = BeautifulSoup(demo, "html.parser")  # 对html解释，用html.parser解释器
    # print(soup.prettify())
    print(soup.a.prettify())


# format_code_html()


def test_type():
    # 测试标签之间文本的数据类型
    r = requests.get("http://python123.io/ws/demo.html")
    r.encoding = r.apparent_encoding
    demo = r.text
    soup = BeautifulSoup(demo, "html.parser")
    print(soup.find('a'))
    print(type(soup.find('a')))
    print(soup.find('a').text)
    print(type(soup.find('a').text))
    print(soup.find_all('a'))
    print(soup.find_all('a')[0])
    print(type(soup.find_all('a')[0]))


test_type()

二、信息组织与提取方法

from bs4 import BeautifulSoup
import requests
import re

'''信息组织与提取方法'''

# 信息标记的三种形式
# XML JSON YAML
r = requests.get("http://python123.io/ws/demo.html")
r.encoding = r.apparent_encoding
demo = r.text
soup = BeautifulSoup(demo, "html.parser")


def get_information_test():
    # 信息提取: 形式解析 搜索 融合
    for link in soup.find_all('a'):
        print(link.get('href'))  # href为属性名
    for tag in soup.find_all(['a', 'b']):  # 返回值为list
        print(tag)
    print(len(soup.find_all(['a', 'b'])), type(soup.find_all(['a', 'b'])), soup.find_all(['a', 'b'])[0])
    for tag in soup.find_all(True):  # 返回所有
        print(tag.name)
    for tag in soup.find_all(re.compile('b')):
        print(tag.name)
    print(soup.find_all('p', 'course'))  # 第二个参数是属性，又course属性的p标签
    print(soup.find_all(id='link1'))   # id域为link1
    print(soup.find_all(id=re.compile('link')))  # 以link开头
    print(soup.find_all(string="Basic Python"))  # 标签之间的内容为string参数
    print(soup.find_all(string=re.compile("python")))  # 标签之间内容包含python
    print(soup.find_all(string=re.compile("Python")))
    print(soup(True))  # soup()等价于soup.find_all()
    print("------------------")
    print(type(soup.find('a')))
    print("------------------")
    print(type(soup.find_all('a')[0]))


get_information_test()