bs4解析器——爬取三国演义目录和内容

'''
爬取三国演义的目录和内容
'''
import requests
from bs4 import BeautifulSoup

if __name__ == '__main__':
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36'
    }
    url = "http://www.shicimingju.com/book/sanguoyanyi.html"
    response = requests.get(url=url,headers = headers)
    page_text = response.text
    #构建soup对象
    soup = BeautifulSoup(page_text,'lxml')
    #层级选择器，找到li标签
    li_list = soup.select(".book-mulu > ul >li")
    fp = open('三国演义.txt', 'w', encoding='utf-8')
    for li in li_list:
        title = li.a.string#获取a标签内的直接文本内容
        detail_url = "http://www.shicimingju.com"+ li.a['href']#获取a标签的href属性值
        detail_page = requests.get(url = detail_url,headers=headers).text
        detail_soup = BeautifulSoup(detail_page,'lxml')
        div_tag = detail_soup.find("div",class_='chapter_content')#查找第一个类名为xx的div
        content = div_tag.text#获取div标签内的全部文本内容
        fp.write(title+':'+content+'\n')
        print(title,'爬取成功')