爬虫笔记01

PS:仅为本人笔记

使用urllib 爬取免费小说

导包 后续会用到正则表达式  Xpath提取文字
设置headers

from urllib import request
import re
from lxml import etree
headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'}

根据小说链接得到小说目录和对应URL,函数返回catelogs列表

def getCatelogs(url):
    # 小说目录页面
    req=request.Request(url=url,headers=headers)
    # 发送请求
    response=request.urlopen(req)
    # 返回数据
    result=[]
    if response.status==200:
        # 读取页面内容
        html=response.read().decode('gbk')
        # print(html)
        # 正则表达式得到<dd>节点列表
        aList=re.findall('<dd>.*</dd>',html)
        # print(aList)
        # 获取每个<dd>节点的url和小说章节标题
        for a in aList:
            # 过滤出URL和小说章节标题
            g=re.search('<a href ="(.*?)">(.*?)</a>',a)
            # print(g)
            if g!=None:
                # 拼接完整URL
                url='http://www.dpxsw.com'+g.group(1)
                # 获取小说章节标题
                title=g.group(2)
                # 创建字典存放URL和小说章节标题
                chapter={'title':title,'url':url}
                # print(chapter)
                # 添加到列表
                result.append(chapter)
    # print(result)
    return result

根据章节目录,抓取目录对应的URL指定的小说内容

def getChapterContent(chapters):
    for chapter in chapters:
        # print(chapter)
        req=request.Request(url=chapter['url'],headers=headers)
        response=request.urlopen(req)
        # print(response.read().decode('gbk'))
        if response.status==200:
            f=open('D:/study/shenxu/'+chapter['title']+'.txt','wb')
            # contents=re.findall('&nbsp;(.*?)<br>',response.read().decode('gbk'))
            r=response.read().decode('gbk')
            # 使用xpath提取文本
            contents = etree.HTML(r).xpath('//*[@id="content"]/text()')
            # print(contents)
            str1 = ''
            n = 0
            for i in contents:
                str1 = str1 + contents[n]
                n += 1
                # print(str1)
            f.write(str1.encode())
            f.close()
            print(chapter['title'],chapter['url'])

调用函数

if __name__ == '__main__':
    #该例为斗破小说网,url输入对应小说目录url
    getChapterContent(getCatelogs('url'))