BeautifulSoup库及实例

BeautifulSoup库的用法

#soup.name 返回的是文档中第一次出现name对应的标签
#soup.find(‘name’) 等同于soup.name
#soup.find(‘name’,class_ = ‘class’) 还可以实现class名为class的标签定位
#soup.findall(‘name’) 返回name标签所有的内容
#soup.select(‘某种选择器’) class/id等等
#soup.select(’.name > ul > li > a’) >表示一个层级 空格可以表示多个层级
#–获取标签间文本或者属性值–soup.a.text/string/get_text()
#获取文本内容–text/get_text 可以获取某一个标签下所有的文本内容 --string只能获取该标签下直系的文本内容
#获取属性值 --soup.a[‘属性类型 比如class’]

实例爬取三国演义小说

import requests
from bs4 import BeautifulSoup
if __name__ == "__main__":
    url = 'https://sanguo.5000yan.com/';
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
    }
    fp = open("./sanguo.text",'w',encoding = 'utf-8');
    #1.实例化一个BeautifulSoup对象
    page_text = requests.get(url = url , headers = headers).text;
    #解决乱码问题encode编码 decode解码
    page_text = page_text.encode("ISO-8859-1")
    page_text = page_text.decode("utf-8")
    soup = BeautifulSoup(page_text,'lxml');
    li_list = soup.select('.sidamingzhu-list-mulu > ul > li')
    for li in li_list:
        title = li.a.string;
        src = li.a['href'];
        detail_data = requests.get(url = src , headers = headers).text;
        detail_data = detail_data.encode("ISO-8859-1");
        detail_data = detail_data.decode("utf-8");
        detail_soup = BeautifulSoup(detail_data,'lxml');
        div = detail_soup.find('div',class_='grap');
        detail_content = div.text;
        detail_content = detail_content.rstrip();
        fp.write(title+':'+detail_content+'\n');