python爬虫爬取文本（以免费网站的小说为例）

# -*- codeing = utf-8 -*-
from bs4 import BeautifulSoup  # 网页解析，获取数据
import re  # 正则表达式，进行文字匹配
import urllib.request, urllib.error  # 制定URL，获取网页数据

prestr="<div class=\"content\">"
poststr="\\n<!-- /content -->\\n<div class=\"dede_pages\">"

def main():
    baseurl = "https://www.520yanqingwang.com/xiandaiyanqing/2018/0312/13570.html"
    datastr = getData(baseurl)
    savepath = "hhh.txt"
    saveData(datastr,savepath) 

    for i in range(1,113):#113 从0到112
        baseurl = "https://www.520yanqingwang.com/xiandaiyanqing/2018/0312/13570_"  #要爬取的网页链接
        baseurl += str(i+1)
        baseurl += ".html"
        #print(baseurl)
        # 1.爬取网页
        datastr = getData(baseurl)
        savepath = "hhh.txt"  
        # 3.保存数据
        saveData(datastr,savepath)  

# 爬取网页
def getData(baseurl):
    mydata = []  #用来存储需要的网页信息
    datalist = []  #用来存储爬取的网页信息
    url = baseurl
    html = askURL(url)  # 保存获取到的网页源码
    # 2.逐一解析数据
    soup = BeautifulSoup(html, "html.parser")
    for item in soup.find_all('div'):  # 查找符合要求的字符串
        data = []  # 保存一份符合条件的信息
        item = str(item)
        data.append(item)
        datalist.append(data)          
    datastr=str(datalist) 
    prepos = datastr.find(prestr)
    postpos = datastr.find(poststr)
    datastr = datastr[prepos:postpos]
    #print(datastr)
    return datastr


# 得到指定一个URL的网页内容
def askURL(url):
    head = {  # 模拟浏览器头部信息，向服务器发送消息
        "User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122  Safari / 537.36"
    }
    # 用户代理，表示告诉服务器，我们是什么类型的机器、浏览器（本质上是告诉浏览器，我们可以接收什么水平的文件内容）

    request = urllib.request.Request(url, headers=head)
    html = ""
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode(encoding="GB18030",errors='ignore')#从utf-8改为gbk
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)
    return html


# 保存数据到txt
def saveData(datastr,savepath):
    print("save.......")
    fh = open(savepath, 'a', encoding='utf-8')
    strlist = datastr.split("<div>")
    for item in strlist:
        if item.find("<div class=\"content\">")!=-1:
            continue
        item = item.replace('\n',"")
        item = item.replace("\\n<div>\\r\\n\\t","")
        item = item.replace("\\xa0","")
        item = item.replace("\\n","")
        item = item.replace("\\r","")
        item = item.replace("\\t","")
        item = item.replace("<div class=\"content\">","")
        item = item.replace("<table width=\"100%\">","")
        item = item.replace("<tr>","")
        item = item.replace("</tr>","")
        item = item.replace("</table>","")
        item = item.replace("</div>","")
        print(item)
        fh.write(item+'\n')
    fh.close()


if __name__ == "__main__":  # 当程序执行时
    # 调用函数
    main()
    print("爬取完毕！")